From d91c5e803c2521834e5861928ce9327184bae63a Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:02:31 +0200 Subject: [PATCH 01/23] Add an immersive OpenXR goggle mode Presents the stream as a head locked compositor quad layer instead of a 2D panel that the headset resamples. The decoder writes straight into an Android surface swapchain (XR_KHR_android_surface_swapchain), so the video path loses the view hierarchy, SurfaceFlinger and the panel's system-chosen resolution. There is no GL blit and no readback: the EGL context only exists because OpenXR requires a graphics binding. New module :app:xr - XrGoggleSession (native): loader init, instance, session, LOCAL + VIEW reference spaces, the android surface swapchain, action set, frame loop. - Head locked by default (quad in VIEW space); world locked mode anchors a yaw-only pose in LOCAL space and can be recentered. - XR_FB_composition_layer_settings quality sharpening on the video layer. - XR_FB_passthrough as an underlay so the pilot can see the radio, a spotter and the ground without taking the headset off. - XR_FB_display_refresh_rate, "auto" picks the highest the headset offers. - Touch controller bindings: A recenter, B passthrough, X record, Y head lock toggle, right stick screen size, left stick distance. Simple controller bindings as a fallback. - Controller haptics on link degradation - a warning a goggle cannot give. - Every optional extension is probed; a runtime without them still flies. - OpenXR loader comes from org.khronos.openxr:openxr_loader_for_android via prefab, so no vendor SDK is vendored into the tree. New XrVideoActivity - Resolves USB permission while still flat (a permission dialog is much easier to deal with before the compositor takes over), then starts the session. - Sizes the swapchain from the last seen stream resolution so the compositor never resamples twice; updates the layer's imageRect live. - DVR toggle on X using the already granted folder; picking a folder still needs flat mode. - No teardown in onPause/onStop: the runtime pauses and resumes an immersive session for its own reasons, and dropping USB there costs seconds of black. - Any failure falls back to VideoActivity with the reason. Supporting changes - WfbLinkManager reported status through ActivityVideoBinding, which only the flat activity has. It now takes a LinkStatusView, implemented by both. - DVR file naming/creation moved from VideoActivity into DvrFiles so both activities share one implementation. - Settings -> VR goggles (OpenXR): enter, head lock, passthrough, sharpening, refresh rate. The whole submenu is hidden unless the device looks like a headset, so nothing changes on a phone. --- app/build.gradle | 1 + app/src/main/AndroidManifest.xml | 24 + .../java/com/openipc/pixelpilot/DvrFiles.java | 98 ++ .../openipc/pixelpilot/LinkStatusView.java | 13 + .../com/openipc/pixelpilot/VideoActivity.java | 144 ++- .../openipc/pixelpilot/WfbLinkManager.java | 27 +- .../openipc/pixelpilot/XrVideoActivity.java | 422 +++++++ app/src/main/res/values/strings.xml | 4 + app/xr/build.gradle | 53 + app/xr/src/main/cpp/CMakeLists.txt | 21 + app/xr/src/main/cpp/XrGoggleSession.cpp | 1043 +++++++++++++++++ app/xr/src/main/cpp/XrGoggleSession.h | 185 +++ app/xr/src/main/cpp/jni_bindings.cpp | 179 +++ .../java/com/openipc/xr/XrGoggleSession.java | 301 +++++ gradle/libs.versions.toml | 4 + settings.gradle | 1 + 16 files changed, 2460 insertions(+), 60 deletions(-) create mode 100644 app/src/main/java/com/openipc/pixelpilot/DvrFiles.java create mode 100644 app/src/main/java/com/openipc/pixelpilot/LinkStatusView.java create mode 100644 app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java create mode 100644 app/xr/build.gradle create mode 100644 app/xr/src/main/cpp/CMakeLists.txt create mode 100644 app/xr/src/main/cpp/XrGoggleSession.cpp create mode 100644 app/xr/src/main/cpp/XrGoggleSession.h create mode 100644 app/xr/src/main/cpp/jni_bindings.cpp create mode 100644 app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java diff --git a/app/build.gradle b/app/build.gradle index 66bbd35b..656a8089 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -59,6 +59,7 @@ dependencies { implementation(project(":app:mavlink")) implementation(project(":app:videonative")) implementation(project(":app:wfbngrtl8812")) + implementation(project(":app:xr")) implementation(libs.appcompat) implementation(libs.material) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 1ad6bc61..d756783a 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -7,6 +7,11 @@ + + @@ -76,5 +81,24 @@ + + + + + + + + + diff --git a/app/src/main/java/com/openipc/pixelpilot/DvrFiles.java b/app/src/main/java/com/openipc/pixelpilot/DvrFiles.java new file mode 100644 index 00000000..3db7e67b --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/DvrFiles.java @@ -0,0 +1,98 @@ +package com.openipc.pixelpilot; + +import android.content.Context; +import android.net.Uri; +import android.util.Log; + +import androidx.documentfile.provider.DocumentFile; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * DVR file naming and creation, shared by the flat and the immersive activity. + * + *

Everything here works off the already granted {@code dvr_folder_} tree URI. Picking a + * folder needs an Activity result and stays with the flat activity. + */ +public final class DvrFiles { + private static final String TAG = "pixelpilot"; + private static final String PREF_DVR_FILENAME = "dvr_filename"; + private static final String DEFAULT_TEMPLATE = "pixelpilot_[yyyyMMdd-HHmmss]"; + private static final Pattern TEMPLATE_PATTERN = Pattern.compile("\\[([^\\]]*)\\]"); + + private DvrFiles() { + } + + public static String template(Context context) { + return context.getSharedPreferences("general", Context.MODE_PRIVATE) + .getString(PREF_DVR_FILENAME, DEFAULT_TEMPLATE); + } + + public static void setTemplate(Context context, String template) { + context.getSharedPreferences("general", Context.MODE_PRIVATE) + .edit().putString(PREF_DVR_FILENAME, template).apply(); + } + + public static boolean fragmentedMp4(Context context) { + return context.getSharedPreferences("general", Context.MODE_PRIVATE) + .getBoolean("dvr_fmp4", true); + } + + /** Expands the {@code [pattern]} placeholder in a file name template. */ + public static String fileName(String template, LocalDateTime time) { + Matcher matcher = TEMPLATE_PATTERN.matcher(template); + String fallbackTime = time.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")); + + if (matcher.find()) { + String prefix = template.substring(0, matcher.start()); + String pattern = matcher.group(1); + String suffix = template.substring(matcher.end()); + + try { + String timePart = time.format(DateTimeFormatter.ofPattern(pattern)); + return prefix + timePart + suffix; + } catch (IllegalArgumentException e) { + return prefix + fallbackTime + suffix; + } + } + return "pixelpilot_" + fallbackTime; + } + + public static boolean hasRecordingFolder(Context context) { + return !context.getSharedPreferences("general", Context.MODE_PRIVATE) + .getString("dvr_folder_", "").isEmpty(); + } + + /** + * Creates the next recording file in the configured folder. + * + * @return its URI, or null if no folder is configured or it is not writable + */ + public static Uri createRecording(Context context) { + String dvrFolder = context.getSharedPreferences("general", Context.MODE_PRIVATE) + .getString("dvr_folder_", ""); + if (dvrFolder.isEmpty()) { + Log.e(TAG, "dvrFolder is empty"); + return null; + } + DocumentFile pickedDir = DocumentFile.fromTreeUri(context, Uri.parse(dvrFolder)); + if (pickedDir == null || !pickedDir.canWrite()) { + return null; + } + String name = fileName(template(context), LocalDateTime.now()) + ".mp4"; + DocumentFile newFile = pickedDir.createFile("video/mp4", name); + if (newFile == null) { + Log.e(TAG, "dvr newFile null"); + return null; + } + return newFile.getUri(); + } + + /** File name the next {@link #createRecording} would use, for UI previews. */ + public static String nextFileName(Context context) { + return fileName(template(context), LocalDateTime.now()) + ".mp4"; + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/LinkStatusView.java b/app/src/main/java/com/openipc/pixelpilot/LinkStatusView.java new file mode 100644 index 00000000..3d74a53b --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/LinkStatusView.java @@ -0,0 +1,13 @@ +package com.openipc.pixelpilot; + +/** + * Where {@link WfbLinkManager} reports adapter/link status to. The flat activity draws it + * into its overlay; the immersive activity has no view hierarchy and logs it instead. + */ +public interface LinkStatusView { + /** Adapter or link state the pilot needs to see (starting, no permission, ...). */ + void showLinkMessage(String message); + + /** Shown when no compatible adapter is attached: where to push a stream instead. */ + void showLocalStreamHint(String url); +} diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index e126fa2e..fb6edca8 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -65,6 +65,7 @@ import com.openipc.wfbngrtl8812.WfbNGStats; import com.openipc.wfbngrtl8812.WfbNGStatsChanged; import com.openipc.wfbngrtl8812.WfbNgLink; +import com.openipc.xr.XrGoggleSession; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; @@ -101,7 +102,7 @@ // Most basic implementation of an activity that uses VideoNative to stream a video // Into an Android Surface View public class VideoActivity extends AppCompatActivity implements IVideoParamsChanged, - WfbNGStatsChanged, MavlinkUpdate, SettingsChanged { + WfbNGStatsChanged, MavlinkUpdate, SettingsChanged, LinkStatusView { private static final String TAG = "pixelpilot"; private static final int PICK_KEY_REQUEST_CODE = 1; private static final int PICK_DVR_REQUEST_CODE = 2; @@ -348,7 +349,7 @@ private void initializeWfbNg() { copyGSKey(); wfbLink = new WfbNgLink(this); wfbLink.SetWfbNGStatsChanged(this); - wfbLinkManager = new WfbLinkManager(this, binding, wfbLink); + wfbLinkManager = new WfbLinkManager(this, this, wfbLink); } // ---------------------------------------------------------------------------- @@ -606,6 +607,9 @@ private void showSettingsMenu(View anchor) { // VR submenu setupVRSubMenu(popup); + // Immersive OpenXR submenu, headsets only + setupXrSubMenu(popup); + // Channel submenu setupChannelSubMenu(popup); @@ -659,6 +663,77 @@ private void setupVRSubMenu(PopupMenu popup) { }); } + /** + * Submenu for the immersive OpenXR mode. Only offered on a device that looks like a + * headset - on a phone there is no runtime to talk to, so the entry stays hidden. + */ + private void setupXrSubMenu(PopupMenu popup) { + if (!XrGoggleSession.isSupportedDevice(this)) { + return; + } + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + SubMenu xrMenu = popup.getMenu().addSubMenu("VR goggles (OpenXR)"); + + xrMenu.add("Enter VR").setOnMenuItemClickListener(item -> { + startActivity(new Intent(this, XrVideoActivity.class)); + return true; + }); + + MenuItem headLocked = xrMenu.add("Head locked"); + headLocked.setCheckable(true); + headLocked.setChecked(prefs.getBoolean(XrVideoActivity.PREF_HEAD_LOCKED, true)); + headLocked.setOnMenuItemClickListener(item -> { + boolean enabled = !item.isChecked(); + item.setChecked(enabled); + prefs.edit().putBoolean(XrVideoActivity.PREF_HEAD_LOCKED, enabled).apply(); + item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW); + item.setActionView(new View(this)); + return false; + }); + + MenuItem passthrough = xrMenu.add("Passthrough"); + passthrough.setCheckable(true); + passthrough.setChecked(prefs.getBoolean(XrVideoActivity.PREF_PASSTHROUGH, false)); + passthrough.setOnMenuItemClickListener(item -> { + boolean enabled = !item.isChecked(); + item.setChecked(enabled); + prefs.edit().putBoolean(XrVideoActivity.PREF_PASSTHROUGH, enabled).apply(); + item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW); + item.setActionView(new View(this)); + return false; + }); + + MenuItem sharpening = xrMenu.add("Layer sharpening"); + sharpening.setCheckable(true); + sharpening.setChecked(prefs.getBoolean(XrVideoActivity.PREF_SHARPENING, true)); + sharpening.setOnMenuItemClickListener(item -> { + boolean enabled = !item.isChecked(); + item.setChecked(enabled); + prefs.edit().putBoolean(XrVideoActivity.PREF_SHARPENING, enabled).apply(); + item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW); + item.setActionView(new View(this)); + return false; + }); + + SubMenu refresh = xrMenu.addSubMenu("Display refresh rate"); + float currentHz = prefs.getFloat(XrVideoActivity.PREF_REFRESH_HZ, 0f); + final float[] options = {0f, 72f, 90f, 120f}; + for (float hz : options) { + String label = hz <= 0f ? "Auto (highest)" : String.format(Locale.US, "%.0f Hz", hz); + MenuItem entry = refresh.add(label); + entry.setCheckable(true); + entry.setChecked(Math.abs(currentHz - hz) < 0.5f); + final float selected = hz; + entry.setOnMenuItemClickListener(item -> { + prefs.edit().putFloat(XrVideoActivity.PREF_REFRESH_HZ, selected).apply(); + return true; + }); + } + + MenuItem help = xrMenu.add("Controls: A recenter, B passthrough, X record, Y lock"); + help.setEnabled(false); + } + /** * Submenu that lists available channels and allows the user to select one. */ @@ -1267,24 +1342,12 @@ private void startVpnService() { } private Uri openDvrFile() { - String dvrFolder = getSharedPreferences("general", - Context.MODE_PRIVATE).getString("dvr_folder_", ""); - if (dvrFolder.isEmpty()) { - Log.e(TAG, "dvrFolder is empty"); - return null; - } - Uri uri = Uri.parse(dvrFolder); - DocumentFile pickedDir = DocumentFile.fromTreeUri(this, uri); - if (pickedDir != null && pickedDir.canWrite()) { - LocalDateTime now = LocalDateTime.now(); - String filename = getDvrFileName(getDvrFileNameTemplate(), now) + ".mp4"; - DocumentFile newFile = pickedDir.createFile("video/mp4", filename); + String filename = DvrFiles.nextFileName(this); + Uri target = DvrFiles.createRecording(this); + if (target != null) { Toast.makeText(this, "Recording to " + filename, Toast.LENGTH_SHORT).show(); - if (newFile == null) - Log.e(TAG, "dvr newFile null"); - return newFile != null ? newFile.getUri() : null; } - return null; + return target; } private void startStopDvr() { @@ -1463,7 +1526,7 @@ public void setGsKey(InputStream inputStream) throws IOException { } public boolean getDvrMP4() { - return getSharedPreferences("general", Context.MODE_PRIVATE).getBoolean("dvr_fmp4", true); + return DvrFiles.fragmentedMp4(this); } public void setDvrMP4(boolean enabled) { @@ -1704,6 +1767,22 @@ public void onWfbNgStatsChanged(WfbNGStats data) { }); } + @Override + public void showLinkMessage(String message) { + runOnUiThread(() -> { + binding.tvMessage.setVisibility(View.VISIBLE); + binding.tvMessage.setText(message); + }); + } + + @Override + public void showLocalStreamHint(String url) { + runOnUiThread(() -> { + binding.wifiMessage.setText(url); + binding.wifiMessage.setVisibility(View.VISIBLE); + }); + } + @Override public void onNewMavlinkData(MavlinkData data) { runOnUiThread(() -> osdManager.render(data)); @@ -1832,22 +1911,7 @@ public void afterTextChanged(Editable editable) {} // pixelpilot_[yyyyMMdd-HHmmss] // ".mp4" will append later private String getDvrFileName(String template, LocalDateTime time) { - Matcher matcher = Pattern.compile("\\[([^\\]]*)\\]").matcher(template); - String fallbackTime = time.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")); - - if (matcher.find()) { - String prefix = template.substring(0, matcher.start()); - String pattern = matcher.group(1); - String suffix = template.substring(matcher.end()); - - try { - String timePart = time.format(DateTimeFormatter.ofPattern(pattern)); - return prefix + timePart + suffix; - } catch (IllegalArgumentException e) { - return prefix + fallbackTime + suffix; - } - } - return "pixelpilot_" + fallbackTime; + return DvrFiles.fileName(template, time); } // Helper method to retrieve the drone username @@ -1856,9 +1920,8 @@ private String getDroneUsername() { return getSharedPreferences("general", Context.MODE_PRIVATE).getString(PREF_DRONE_USERNAME, "root"); } - private String getDvrFileNameTemplate() - { - return getSharedPreferences("general", Context.MODE_PRIVATE).getString(PREF_DVR_FILENAME, "pixelpilot_[yyyyMMdd-HHmmss]"); + private String getDvrFileNameTemplate() { + return DvrFiles.template(this); } // Helper method to save the drone username @@ -1870,10 +1933,7 @@ private void setDroneUsername(String username) { } private void setDvrFileName(String fileName) { - SharedPreferences prefs = getSharedPreferences("general", Context.MODE_PRIVATE); - SharedPreferences.Editor editor = prefs.edit(); - editor.putString(PREF_DVR_FILENAME, fileName); - editor.apply(); + DvrFiles.setTemplate(this, fileName); } // Helper method to retrieve the drone password diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java index 0c0390e0..aa73c37d 100644 --- a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java +++ b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java @@ -7,9 +7,7 @@ import android.hardware.usb.UsbDevice; import android.hardware.usb.UsbManager; import android.util.Log; -import android.view.View; -import com.openipc.pixelpilot.databinding.ActivityVideoBinding; import com.openipc.wfbngrtl8812.WfbNgLink; import java.util.HashMap; @@ -22,7 +20,7 @@ public class WfbLinkManager extends BroadcastReceiver { private static final String TAG = "pixelpilot"; static Map activeWifiAdapters = new HashMap<>(); private final WfbNgLink wfbLink; - private final ActivityVideoBinding binding; + private final LinkStatusView status; private final Context context; private int wifiChannel; private Bandwidth bandWidth; @@ -42,8 +40,8 @@ public int getValue() { } } - public WfbLinkManager(Context context, ActivityVideoBinding binding, WfbNgLink wfbNgLink) { - this.binding = binding; + public WfbLinkManager(Context context, LinkStatusView status, WfbNgLink wfbNgLink) { + this.status = status; this.context = context; this.wfbLink = wfbNgLink; } @@ -130,8 +128,7 @@ public synchronized void refreshAdapters() { (android.hardware.usb.UsbManager) context.getSystemService(Context.USB_SERVICE); for (Map.Entry entry : attachedAdapters.entrySet()) { if (!usbManager.hasPermission(entry.getValue())) { - binding.tvMessage.setVisibility(View.VISIBLE); - binding.tvMessage.setText("No permission for wifi adapter(s) " + entry.getValue().getDeviceName()); + status.showLinkMessage("No permission for wifi adapter(s) " + entry.getValue().getDeviceName()); // Android 14 refuses to deliver a PendingIntent built from an implicit // intent to a runtime registered receiver, so the permission result never // arrives unless the package is set explicitly. @@ -172,15 +169,11 @@ public synchronized void refreshAdapters() { } if (activeWifiAdapters.isEmpty()) { - String text = "No compatible wifi adapter found."; - binding.tvMessage.setText(text); - binding.tvMessage.setVisibility(View.VISIBLE); + status.showLinkMessage("No compatible wifi adapter found."); String wifi = VideoActivity.wirelessInfo(); if (wifi != null) { - String local = "udp://" + wifi + ":5600"; - binding.wifiMessage.setText(local); - binding.wifiMessage.setVisibility(View.VISIBLE); + status.showLocalStreamHint("udp://" + wifi + ":5600"); } } } @@ -212,12 +205,10 @@ public synchronized void startAdapters() { } public synchronized boolean startAdapter(UsbDevice dev) { - binding.tvMessage.setVisibility(View.VISIBLE); - String text = "Starting wfb-ng channel " + wifiChannel + " with " + String.format( - "[%04X", dev.getVendorId()) + ":" + String.format("%04X]", dev.getProductId()); - binding.tvMessage.setText(text); + status.showLinkMessage("Starting wfb-ng channel " + wifiChannel + " with " + String.format( + "[%04X", dev.getVendorId()) + ":" + String.format("%04X]", dev.getProductId())); if (!wfbLink.start(wifiChannel, bandWidth.getValue(), dev)) { - binding.tvMessage.setText("Could not open wifi adapter " + dev.getDeviceName()); + status.showLinkMessage("Could not open wifi adapter " + dev.getDeviceName()); return false; } return true; diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java new file mode 100644 index 00000000..1e32fb88 --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -0,0 +1,422 @@ +package com.openipc.pixelpilot; + +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.content.SharedPreferences; +import android.graphics.Color; +import android.hardware.usb.UsbDevice; +import android.hardware.usb.UsbManager; +import android.net.Uri; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.os.ParcelFileDescriptor; +import android.util.Log; +import android.view.Gravity; +import android.view.Surface; +import android.view.View; +import android.view.WindowManager; +import android.widget.TextView; +import android.widget.Toast; + +import androidx.annotation.Nullable; +import androidx.appcompat.app.AppCompatActivity; + +import com.openipc.videonative.DecodingInfo; +import com.openipc.videonative.IVideoParamsChanged; +import com.openipc.videonative.VideoPlayer; +import com.openipc.wfbngrtl8812.WfbNGStats; +import com.openipc.wfbngrtl8812.WfbNGStatsChanged; +import com.openipc.wfbngrtl8812.WfbNgLink; +import com.openipc.xr.XrGoggleSession; + +import java.io.IOException; +import java.util.Map; + +/** + * Immersive ground station. + * + *

Instead of a 2D panel that the headset compositor resamples onto a quad, the decoder + * writes straight into a compositor swapchain + * ({@code xrCreateSwapchainAndroidSurfaceKHR}) which is presented as a head locked quad + * layer. That removes the view hierarchy, SurfaceFlinger and the panel's fixed resolution + * from the video path. + * + *

There is no in-headset menu yet: channel, bandwidth and the DVR folder are still + * configured in {@link VideoActivity} before putting the headset on. Controllers cover + * what matters in flight. + */ +public class XrVideoActivity extends AppCompatActivity + implements XrGoggleSession.Listener, IVideoParamsChanged, WfbNGStatsChanged, LinkStatusView { + + private static final String TAG = "pixelpilot-xr"; + + // Preference keys, all in the shared "general" store so the flat activity's settings + // menu can write them. + static final String PREF_HEAD_LOCKED = "xr_head_locked"; + static final String PREF_PASSTHROUGH = "xr_passthrough"; + static final String PREF_SHARPENING = "xr_sharpening"; + static final String PREF_REFRESH_HZ = "xr_refresh_hz"; + static final String PREF_QUAD_WIDTH = "xr_quad_width"; + static final String PREF_QUAD_DISTANCE = "xr_quad_distance"; + static final String PREF_LAST_VIDEO_W = "xr_last_video_width"; + static final String PREF_LAST_VIDEO_H = "xr_last_video_height"; + + private final Handler handler = new Handler(Looper.getMainLooper()); + + private XrGoggleSession xr; + private VideoPlayer videoPlayer; + private WfbNgLink wfbLink; + private WfbLinkManager wfbLinkManager; + private TextView statusView; + + private boolean xrStarting; + private boolean linkStarted; + private ParcelFileDescriptor dvrFd; + + // Link quality haptics + private long lastHapticAt; + private boolean linkWasBad; + + private final BroadcastReceiver usbReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + // Permission granted / adapter (re)appeared - re-evaluate whether we can go + // immersive now. + handler.post(XrVideoActivity.this::tryBringUpLink); + } + }; + + // ------------------------------------------------------------------------------ + // lifecycle + // ------------------------------------------------------------------------------ + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + + // Only shown while still flat: USB permission, errors, "waiting for adapter". + statusView = new TextView(this); + statusView.setTextColor(Color.WHITE); + statusView.setBackgroundColor(Color.BLACK); + statusView.setTextSize(22f); + statusView.setGravity(Gravity.CENTER); + statusView.setText(R.string.xr_starting); + setContentView(statusView); + + if (!XrGoggleSession.isLibraryLoaded()) { + failToFlat("This build has no OpenXR support"); + return; + } + + videoPlayer = new VideoPlayer(this); + videoPlayer.setIVideoParamsChanged(this); + videoPlayer.setLowLatency(VideoActivity.getLowLatencySetting(this)); + + wfbLink = new WfbNgLink(this); + wfbLink.SetWfbNGStatsChanged(this); + wfbLinkManager = new WfbLinkManager(this, this, wfbLink); + + xr = new XrGoggleSession(this, this); + SharedPreferences prefs = prefs(); + // Match the swapchain to the stream so the compositor never resamples twice. The + // resolution from the last session is a good guess; 1080p until we have seen one. + xr.setSwapchainSize(prefs.getInt(PREF_LAST_VIDEO_W, 1920), prefs.getInt(PREF_LAST_VIDEO_H, 1080)); + xr.setHeadLocked(prefs.getBoolean(PREF_HEAD_LOCKED, true)); + xr.setPassthrough(prefs.getBoolean(PREF_PASSTHROUGH, false)); + xr.setSharpening(prefs.getBoolean(PREF_SHARPENING, true)); + xr.setQuadWidth(prefs.getFloat(PREF_QUAD_WIDTH, 2.2f)); + xr.setQuadDistance(prefs.getFloat(PREF_QUAD_DISTANCE, 1.6f)); + } + + @Override + protected void onResume() { + super.onResume(); + if (xr == null) { + return; + } + registerUsbReceiver(); + tryBringUpLink(); + } + + // Deliberately no teardown in onPause/onStop. In an immersive session the runtime + // pauses and resumes us for its own reasons (system menu, proximity sensor), and + // dropping the USB link there costs seconds of black screen on the way back. + + @Override + protected void onDestroy() { + persistGeometry(); + try { + unregisterReceiver(usbReceiver); + } catch (IllegalArgumentException ignored) { + // never registered + } + stopDvr(); + if (xr != null) { + xr.release(); + xr = null; + } + if (wfbLinkManager != null) { + wfbLinkManager.stopAdapters(); + } + if (videoPlayer != null) { + videoPlayer.stopAudio(); + videoPlayer.stop(); + } + super.onDestroy(); + } + + private SharedPreferences prefs() { + return getSharedPreferences("general", Context.MODE_PRIVATE); + } + + private void registerUsbReceiver() { + IntentFilter filter = new IntentFilter(); + filter.addAction(WfbLinkManager.ACTION_USB_PERMISSION); + filter.addAction(UsbManager.ACTION_USB_DEVICE_ATTACHED); + filter.addAction(UsbManager.ACTION_USB_DEVICE_DETACHED); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + registerReceiver(usbReceiver, filter, Context.RECEIVER_NOT_EXPORTED); + } else { + registerReceiver(usbReceiver, filter); + } + } + + // ------------------------------------------------------------------------------ + // bring-up + // ------------------------------------------------------------------------------ + + /** + * Resolves USB permission while still on a flat panel - a permission dialog is far + * easier to deal with before the compositor takes over - and only then starts the + * immersive session. + */ + private void tryBringUpLink() { + if (xr == null) { + return; + } + wfbLinkManager.setChannel(VideoActivity.getChannel(this)); + wfbLinkManager.setBandwidth(VideoActivity.getBandwidth(this)); + wfbLinkManager.refreshAdapters(); + + if (!adaptersReady()) { + // refreshAdapters() has asked for permission; the broadcast brings us back. + return; + } + if (!xrStarting) { + xrStarting = true; + statusView.setText(R.string.xr_entering); + xr.start(); + } + } + + /** True when every attached compatible adapter is usable (or there is none at all). */ + private boolean adaptersReady() { + Map adapters = wfbLinkManager.getAttachedAdapters(); + if (adapters == null || adapters.isEmpty()) { + // Nothing attached: still go immersive, the stream may arrive over UDP. + return true; + } + UsbManager usbManager = (UsbManager) getSystemService(Context.USB_SERVICE); + for (UsbDevice device : adapters.values()) { + if (!usbManager.hasPermission(device)) { + return false; + } + } + return true; + } + + @Override + public void onXrReady(Surface videoSurface) { + Log.i(TAG, "immersive session up, handing the swapchain surface to the decoder"); + // The decoder writes into the compositor swapchain from here on. + videoPlayer.addAndStartDecoderReceiver(videoSurface, 0); + videoPlayer.start(); + videoPlayer.startAudio(); + applyRefreshRatePreference(); + statusView.setVisibility(View.GONE); + linkStarted = true; + wfbLinkManager.startAdapters(); + } + + private void applyRefreshRatePreference() { + float wanted = prefs().getFloat(PREF_REFRESH_HZ, 0f); + if (wanted <= 0f) { + // "Auto" means the highest the headset offers: at 60 fps video a higher + // display rate only makes the cadence more even. + for (float hz : xr.refreshRates()) { + wanted = Math.max(wanted, hz); + } + } + if (wanted > 0f) { + xr.requestRefreshRate(wanted); + } + } + + @Override + public void onXrButton(int button) { + switch (button) { + case XrGoggleSession.BUTTON_RECENTER: + // Applied natively already, nothing to persist. + break; + case XrGoggleSession.BUTTON_PASSTHROUGH: + prefs().edit().putBoolean(PREF_PASSTHROUGH, xr.isPassthroughEnabled()).apply(); + break; + case XrGoggleSession.BUTTON_LOCK_MODE: + prefs().edit().putBoolean(PREF_HEAD_LOCKED, xr.isHeadLocked()).apply(); + break; + case XrGoggleSession.BUTTON_RECORD: + toggleDvr(); + break; + default: + break; + } + } + + @Override + public void onXrStopped(@Nullable String error) { + if (error != null) { + Log.e(TAG, "immersive session failed: " + error); + failToFlat(error); + } else { + finish(); + } + } + + /** Immersive mode is not usable here - say why and go back to the flat activity. */ + private void failToFlat(String reason) { + Toast.makeText(this, "VR mode unavailable: " + reason, Toast.LENGTH_LONG).show(); + if (statusView != null) { + statusView.setVisibility(View.VISIBLE); + statusView.setText(getString(R.string.xr_failed, reason)); + } + handler.postDelayed(() -> { + startActivity(new Intent(this, VideoActivity.class)); + finish(); + }, 2500); + } + + private void persistGeometry() { + if (xr == null) { + return; + } + prefs().edit() + .putFloat(PREF_QUAD_WIDTH, xr.quadWidth()) + .putFloat(PREF_QUAD_DISTANCE, xr.quadDistance()) + .putBoolean(PREF_HEAD_LOCKED, xr.isHeadLocked()) + .putBoolean(PREF_PASSTHROUGH, xr.isPassthroughEnabled()) + .apply(); + } + + // ------------------------------------------------------------------------------ + // DVR + // ------------------------------------------------------------------------------ + + private void toggleDvr() { + if (dvrFd != null) { + stopDvr(); + xr.haptic(0.7f, 60); + Toast.makeText(this, "Recording stopped", Toast.LENGTH_SHORT).show(); + return; + } + if (!DvrFiles.hasRecordingFolder(this)) { + // Picking a folder needs the flat activity; refuse rather than drop out of VR. + xr.haptic(1.0f, 250); + Toast.makeText(this, "Pick a DVR folder in flat mode first", Toast.LENGTH_LONG).show(); + return; + } + Uri target = DvrFiles.createRecording(this); + if (target == null) { + xr.haptic(1.0f, 250); + Toast.makeText(this, "Could not create the recording file", Toast.LENGTH_LONG).show(); + return; + } + try { + dvrFd = getContentResolver().openFileDescriptor(target, "rw"); + videoPlayer.startDvr(dvrFd.getFd(), DvrFiles.fragmentedMp4(this)); + xr.haptic(0.7f, 120); + Toast.makeText(this, "Recording", Toast.LENGTH_SHORT).show(); + } catch (IOException e) { + Log.e(TAG, "Failed to open dvr file", e); + dvrFd = null; + xr.haptic(1.0f, 250); + } + } + + private void stopDvr() { + if (dvrFd == null) { + return; + } + videoPlayer.stopDvr(); + try { + dvrFd.close(); + } catch (IOException e) { + Log.e(TAG, "Failed to close dvr file", e); + } + dvrFd = null; + } + + // ------------------------------------------------------------------------------ + // stream + link callbacks + // ------------------------------------------------------------------------------ + + @Override + public void onVideoRatioChanged(int videoW, int videoH) { + if (videoW <= 0 || videoH <= 0) { + return; + } + Log.i(TAG, "stream resolution " + videoW + "x" + videoH); + if (xr != null) { + xr.setVideoResolution(videoW, videoH); + } + // So the next session can size its swapchain exactly. + prefs().edit().putInt(PREF_LAST_VIDEO_W, videoW).putInt(PREF_LAST_VIDEO_H, videoH).apply(); + } + + @Override + public void onDecodingInfoChanged(DecodingInfo decodingInfo) { + // Nothing to draw in immersive mode yet. + } + + @Override + public void onWfbNgStatsChanged(WfbNGStats data) { + if (xr == null || !linkStarted || data.count_p_all <= 0) { + return; + } + // A headset can do something goggles cannot: warn you through the controllers, so + // a degrading link does not depend on you reading an OSD number mid-manoeuvre. + final int lost = data.count_p_lost; + final int recovered = data.count_p_fec_recovered; + final boolean bad = lost > 0 || recovered > data.count_p_all / 8; + + final long now = System.currentTimeMillis(); + if (bad && (!linkWasBad || now - lastHapticAt > 2000)) { + lastHapticAt = now; + xr.haptic(lost > 0 ? 1.0f : 0.5f, lost > 0 ? 180 : 70); + } + linkWasBad = bad; + } + + // ------------------------------------------------------------------------------ + // LinkStatusView + // ------------------------------------------------------------------------------ + + @Override + public void showLinkMessage(String message) { + Log.i(TAG, message); + runOnUiThread(() -> { + if (statusView != null && !linkStarted) { + statusView.setText(message); + } + }); + } + + @Override + public void showLocalStreamHint(String url) { + Log.i(TAG, "listening on " + url); + } +} diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 9ebd0632..d3c6705e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1,5 +1,9 @@ PixelPilot + PixelPilot VR + Waiting for the wifi adapter… + Entering VR… + VR mode unavailable: %1$s — falling back to flat mode… 1 2 diff --git a/app/xr/build.gradle b/app/xr/build.gradle new file mode 100644 index 00000000..20e81cc1 --- /dev/null +++ b/app/xr/build.gradle @@ -0,0 +1,53 @@ +plugins { + alias(libs.plugins.androidLibrary) +} + +android { + namespace = "com.openipc.xr" + compileSdk = 34 + + defaultConfig { + minSdk = 26 + } + + buildTypes { + release { + ndk { + abiFilters.add("arm64-v8a") + abiFilters.add("armeabi-v7a") + } + } + debug { + ndk { + abiFilters.add("arm64-v8a") + abiFilters.add("armeabi-v7a") + } + } + } + + externalNativeBuild { + cmake { + path("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } + + // The OpenXR loader is consumed as a prefab package (OpenXR::openxr_loader, + // OpenXR::headers) out of the Khronos AAR. + buildFeatures { + prefab = true + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } +} + +dependencies { + implementation(libs.annotation) + + // api, not implementation: the loader's .so and its manifest additions + // (OpenXR broker queries/permissions) have to reach the application module. + api(libs.openxr.loader) +} diff --git a/app/xr/src/main/cpp/CMakeLists.txt b/app/xr/src/main/cpp/CMakeLists.txt new file mode 100644 index 00000000..15cc16e2 --- /dev/null +++ b/app/xr/src/main/cpp/CMakeLists.txt @@ -0,0 +1,21 @@ + +cmake_minimum_required(VERSION 3.22.1) + +project("XrGoggle") + +find_package(OpenXR REQUIRED CONFIG) + +add_library(${CMAKE_PROJECT_NAME} SHARED + XrGoggleSession.cpp + jni_bindings.cpp) + +target_link_libraries(${CMAKE_PROJECT_NAME} + OpenXR::openxr_loader + OpenXR::headers + android + EGL + GLESv3 + log) + +set_property(TARGET ${CMAKE_PROJECT_NAME} PROPERTY CXX_STANDARD 20) +target_compile_options(${CMAKE_PROJECT_NAME} PRIVATE -fno-omit-frame-pointer) diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp new file mode 100644 index 00000000..aa58f8d9 --- /dev/null +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -0,0 +1,1043 @@ +#include "XrGoggleSession.h" + +#include +#include + +#include +#include +#include + +#define TAG "pixelpilot-xr" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) +#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) + +namespace +{ +constexpr float kStickDeadzone = 0.30f; +constexpr float kDistancePerSec = 0.9f; // meters per second of full stick deflection +constexpr float kWidthPerSec = 1.4f; +constexpr float kMinDistance = 0.4f; +constexpr float kMaxDistance = 8.0f; +constexpr float kMinWidth = 0.4f; +constexpr float kMaxWidth = 12.0f; + +float clampf(float v, float lo, float hi) +{ + return v < lo ? lo : (v > hi ? hi : v); +} + +// Yaw-only quaternion, so the panel never ends up tilted when it is anchored. +XrQuaternionf quatFromYaw(float yaw) +{ + return XrQuaternionf{0.0f, std::sin(yaw * 0.5f), 0.0f, std::cos(yaw * 0.5f)}; +} + +float yawOf(const XrQuaternionf& q) +{ + return std::atan2(2.0f * (q.w * q.y + q.x * q.z), 1.0f - 2.0f * (q.y * q.y + q.z * q.z)); +} +} // namespace + +// --------------------------------------------------------------------------------- +// error handling +// --------------------------------------------------------------------------------- + +void XrGoggleSession::setError(const std::string& what) +{ + std::lock_guard lock(mErrorMutex); + if (mLastError.empty()) + { + mLastError = what; + } + LOGE("%s", what.c_str()); +} + +std::string XrGoggleSession::lastError() +{ + std::lock_guard lock(mErrorMutex); + return mLastError; +} + +bool XrGoggleSession::check(XrResult result, const char* what) +{ + if (XR_SUCCEEDED(result)) + { + return true; + } + char name[XR_MAX_RESULT_STRING_SIZE] = {0}; + if (mInstance != XR_NULL_HANDLE) + { + xrResultToString(mInstance, result, name); + } + else + { + snprintf(name, sizeof(name), "%d", (int) result); + } + setError(std::string(what) + " failed: " + name); + return false; +} + +XrPath XrGoggleSession::path(const char* str) +{ + XrPath p = XR_NULL_PATH; + XrResult r = xrStringToPath(mInstance, str, &p); + if (XR_FAILED(r)) + { + LOGW("xrStringToPath(%s) failed", str); + return XR_NULL_PATH; + } + return p; +} + +bool XrGoggleSession::extensionSupported(const char* name) const +{ + for (const auto& e : mAvailableExtensions) + { + if (e == name) return true; + } + return false; +} + +// --------------------------------------------------------------------------------- +// bring-up +// --------------------------------------------------------------------------------- + +bool XrGoggleSession::create(JNIEnv* env, jobject activity) +{ + if (!initLoader(env, activity)) return false; + if (!createInstance(env, activity)) return false; + if (!createEgl()) return false; + if (!createSession()) return false; + if (!createSpaces()) return false; + if (!createSwapchain(env)) return false; + if (!createActions()) return false; + // Passthrough is optional: a runtime without XR_FB_passthrough still flies. + if (mHasPassthrough && !createPassthrough()) + { + LOGW("passthrough unavailable, continuing without it"); + mHasPassthrough = false; + } + if (mHasRefreshRate) + { + uint32_t count = 0; + if (check(mXrEnumerateDisplayRefreshRatesFB(mSession, 0, &count, nullptr), + "xrEnumerateDisplayRefreshRatesFB") && + count > 0) + { + mRefreshRates.resize(count); + mXrEnumerateDisplayRefreshRatesFB(mSession, count, &count, mRefreshRates.data()); + for (float hz : mRefreshRates) + { + LOGI("display refresh rate available: %.1f Hz", hz); + } + } + } + return true; +} + +bool XrGoggleSession::initLoader(JNIEnv* env, jobject activity) +{ + PFN_xrInitializeLoaderKHR initializeLoader = nullptr; + XrResult r = xrGetInstanceProcAddr( + XR_NULL_HANDLE, "xrInitializeLoaderKHR", (PFN_xrVoidFunction*) &initializeLoader); + if (XR_FAILED(r) || initializeLoader == nullptr) + { + // Not fatal on every runtime, but on Android it usually is. + LOGW("xrInitializeLoaderKHR not exposed by the loader"); + return true; + } + + JavaVM* vm = nullptr; + env->GetJavaVM(&vm); + + XrLoaderInitInfoAndroidKHR init{XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR}; + init.applicationVM = vm; + init.applicationContext = env->NewGlobalRef(activity); + r = initializeLoader((const XrLoaderInitInfoBaseHeaderKHR*) &init); + if (XR_FAILED(r)) + { + setError("xrInitializeLoaderKHR failed"); + return false; + } + return true; +} + +bool XrGoggleSession::createInstance(JNIEnv* env, jobject activity) +{ + uint32_t extCount = 0; + if (!check(xrEnumerateInstanceExtensionProperties(nullptr, 0, &extCount, nullptr), + "xrEnumerateInstanceExtensionProperties")) + { + return false; + } + std::vector props(extCount, {XR_TYPE_EXTENSION_PROPERTIES}); + if (extCount > 0) + { + if (!check( + xrEnumerateInstanceExtensionProperties(nullptr, extCount, &extCount, props.data()), + "xrEnumerateInstanceExtensionProperties")) + { + return false; + } + } + mAvailableExtensions.clear(); + for (const auto& p : props) + { + mAvailableExtensions.emplace_back(p.extensionName); + } + + // Without these three there is no immersive video path at all. + const char* required[] = { + XR_KHR_ANDROID_CREATE_INSTANCE_EXTENSION_NAME, + XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME, + XR_KHR_ANDROID_SURFACE_SWAPCHAIN_EXTENSION_NAME, + }; + std::vector enabled; + for (const char* name : required) + { + if (!extensionSupported(name)) + { + setError(std::string("OpenXR runtime does not support ") + name); + return false; + } + enabled.push_back(name); + } + + mHasPassthrough = extensionSupported(XR_FB_PASSTHROUGH_EXTENSION_NAME); + mHasLayerSettings = extensionSupported(XR_FB_COMPOSITION_LAYER_SETTINGS_EXTENSION_NAME); + mHasRefreshRate = extensionSupported(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME); + if (mHasPassthrough) enabled.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME); + if (mHasLayerSettings) enabled.push_back(XR_FB_COMPOSITION_LAYER_SETTINGS_EXTENSION_NAME); + if (mHasRefreshRate) enabled.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME); + LOGI("optional extensions: passthrough=%d layerSettings=%d refreshRate=%d", + (int) mHasPassthrough, + (int) mHasLayerSettings, + (int) mHasRefreshRate); + + JavaVM* vm = nullptr; + env->GetJavaVM(&vm); + + XrInstanceCreateInfoAndroidKHR androidInfo{XR_TYPE_INSTANCE_CREATE_INFO_ANDROID_KHR}; + androidInfo.applicationVM = vm; + androidInfo.applicationActivity = env->NewGlobalRef(activity); + + XrInstanceCreateInfo info{XR_TYPE_INSTANCE_CREATE_INFO}; + info.next = &androidInfo; + std::strncpy(info.applicationInfo.applicationName, "PixelPilot", XR_MAX_APPLICATION_NAME_SIZE - 1); + std::strncpy(info.applicationInfo.engineName, "PixelPilot", XR_MAX_ENGINE_NAME_SIZE - 1); + info.applicationInfo.applicationVersion = 1; + info.applicationInfo.engineVersion = 1; + // Ask for 1.0: every shipping Android runtime implements it, and nothing here needs 1.1. + info.applicationInfo.apiVersion = XR_API_VERSION_1_0; + info.enabledExtensionCount = (uint32_t) enabled.size(); + info.enabledExtensionNames = enabled.data(); + + if (!check(xrCreateInstance(&info, &mInstance), "xrCreateInstance")) return false; + + XrSystemGetInfo systemInfo{XR_TYPE_SYSTEM_GET_INFO}; + systemInfo.formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY; + if (!check(xrGetSystem(mInstance, &systemInfo, &mSystemId), "xrGetSystem")) return false; + + xrGetInstanceProcAddr(mInstance, + "xrCreateSwapchainAndroidSurfaceKHR", + (PFN_xrVoidFunction*) &mXrCreateSwapchainAndroidSurfaceKHR); + if (mXrCreateSwapchainAndroidSurfaceKHR == nullptr) + { + setError("xrCreateSwapchainAndroidSurfaceKHR not available"); + return false; + } + if (mHasPassthrough) + { + xrGetInstanceProcAddr( + mInstance, "xrCreatePassthroughFB", (PFN_xrVoidFunction*) &mXrCreatePassthroughFB); + xrGetInstanceProcAddr( + mInstance, "xrDestroyPassthroughFB", (PFN_xrVoidFunction*) &mXrDestroyPassthroughFB); + xrGetInstanceProcAddr( + mInstance, "xrPassthroughStartFB", (PFN_xrVoidFunction*) &mXrPassthroughStartFB); + xrGetInstanceProcAddr( + mInstance, "xrPassthroughPauseFB", (PFN_xrVoidFunction*) &mXrPassthroughPauseFB); + xrGetInstanceProcAddr(mInstance, + "xrCreatePassthroughLayerFB", + (PFN_xrVoidFunction*) &mXrCreatePassthroughLayerFB); + xrGetInstanceProcAddr(mInstance, + "xrDestroyPassthroughLayerFB", + (PFN_xrVoidFunction*) &mXrDestroyPassthroughLayerFB); + xrGetInstanceProcAddr(mInstance, + "xrPassthroughLayerResumeFB", + (PFN_xrVoidFunction*) &mXrPassthroughLayerResumeFB); + xrGetInstanceProcAddr(mInstance, + "xrPassthroughLayerPauseFB", + (PFN_xrVoidFunction*) &mXrPassthroughLayerPauseFB); + xrGetInstanceProcAddr(mInstance, + "xrPassthroughLayerSetStyleFB", + (PFN_xrVoidFunction*) &mXrPassthroughLayerSetStyleFB); + mHasPassthrough = mXrCreatePassthroughFB != nullptr && mXrCreatePassthroughLayerFB != nullptr && + mXrPassthroughStartFB != nullptr; + } + if (mHasRefreshRate) + { + xrGetInstanceProcAddr(mInstance, + "xrEnumerateDisplayRefreshRatesFB", + (PFN_xrVoidFunction*) &mXrEnumerateDisplayRefreshRatesFB); + xrGetInstanceProcAddr(mInstance, + "xrRequestDisplayRefreshRateFB", + (PFN_xrVoidFunction*) &mXrRequestDisplayRefreshRateFB); + mHasRefreshRate = + mXrEnumerateDisplayRefreshRatesFB != nullptr && mXrRequestDisplayRefreshRateFB != nullptr; + } + return true; +} + +bool XrGoggleSession::createEgl() +{ + // Nothing is rendered through this context - OpenXR just requires a graphics binding, + // and the runtime needs a current context on the session thread. + mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (mEglDisplay == EGL_NO_DISPLAY) + { + setError("eglGetDisplay failed"); + return false; + } + if (eglInitialize(mEglDisplay, nullptr, nullptr) == EGL_FALSE) + { + setError("eglInitialize failed"); + return false; + } + + const EGLint configAttribs[] = {EGL_RENDERABLE_TYPE, + EGL_OPENGL_ES3_BIT_KHR, + EGL_SURFACE_TYPE, + EGL_PBUFFER_BIT, + EGL_RED_SIZE, + 8, + EGL_GREEN_SIZE, + 8, + EGL_BLUE_SIZE, + 8, + EGL_ALPHA_SIZE, + 8, + EGL_DEPTH_SIZE, + 0, + EGL_STENCIL_SIZE, + 0, + EGL_SAMPLES, + 0, + EGL_NONE}; + EGLint numConfigs = 0; + if (eglChooseConfig(mEglDisplay, configAttribs, &mEglConfig, 1, &numConfigs) == EGL_FALSE || + numConfigs < 1) + { + setError("eglChooseConfig found no usable config"); + return false; + } + + const EGLint contextAttribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE}; + mEglContext = eglCreateContext(mEglDisplay, mEglConfig, EGL_NO_CONTEXT, contextAttribs); + if (mEglContext == EGL_NO_CONTEXT) + { + setError("eglCreateContext failed"); + return false; + } + + const EGLint surfaceAttribs[] = {EGL_WIDTH, 16, EGL_HEIGHT, 16, EGL_NONE}; + mEglSurface = eglCreatePbufferSurface(mEglDisplay, mEglConfig, surfaceAttribs); + if (mEglSurface == EGL_NO_SURFACE) + { + setError("eglCreatePbufferSurface failed"); + return false; + } + if (eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, mEglContext) == EGL_FALSE) + { + setError("eglMakeCurrent failed"); + return false; + } + return true; +} + +bool XrGoggleSession::createSession() +{ + // Required by the spec before xrCreateSession, even though we never render. + PFN_xrGetOpenGLESGraphicsRequirementsKHR getRequirements = nullptr; + xrGetInstanceProcAddr(mInstance, + "xrGetOpenGLESGraphicsRequirementsKHR", + (PFN_xrVoidFunction*) &getRequirements); + if (getRequirements != nullptr) + { + XrGraphicsRequirementsOpenGLESKHR requirements{XR_TYPE_GRAPHICS_REQUIREMENTS_OPENGL_ES_KHR}; + getRequirements(mInstance, mSystemId, &requirements); + } + + XrGraphicsBindingOpenGLESAndroidKHR binding{XR_TYPE_GRAPHICS_BINDING_OPENGL_ES_ANDROID_KHR}; + binding.display = mEglDisplay; + binding.config = mEglConfig; + binding.context = mEglContext; + + XrSessionCreateInfo info{XR_TYPE_SESSION_CREATE_INFO}; + info.next = &binding; + info.systemId = mSystemId; + return check(xrCreateSession(mInstance, &info, &mSession), "xrCreateSession"); +} + +bool XrGoggleSession::createSpaces() +{ + XrReferenceSpaceCreateInfo viewInfo{XR_TYPE_REFERENCE_SPACE_CREATE_INFO}; + viewInfo.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW; + viewInfo.poseInReferenceSpace = XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; + if (!check(xrCreateReferenceSpace(mSession, &viewInfo, &mViewSpace), + "xrCreateReferenceSpace(VIEW)")) + { + return false; + } + + XrReferenceSpaceCreateInfo localInfo{XR_TYPE_REFERENCE_SPACE_CREATE_INFO}; + localInfo.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_LOCAL; + localInfo.poseInReferenceSpace = XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; + return check(xrCreateReferenceSpace(mSession, &localInfo, &mLocalSpace), + "xrCreateReferenceSpace(LOCAL)"); +} + +bool XrGoggleSession::createSwapchain(JNIEnv* env) +{ + XrSwapchainCreateInfo info{XR_TYPE_SWAPCHAIN_CREATE_INFO}; + // An Android surface swapchain is produced into by MediaCodec, so format and the + // image-layout fields are not ours to pick. + info.usageFlags = XR_SWAPCHAIN_USAGE_SAMPLED_BIT; + info.format = 0; + info.sampleCount = 1; + info.width = (uint32_t) mSwapchainWidth; + info.height = (uint32_t) mSwapchainHeight; + info.faceCount = 1; + info.arraySize = 1; + info.mipCount = 1; + + jobject surface = nullptr; + if (!check(mXrCreateSwapchainAndroidSurfaceKHR(mSession, &info, &mSwapchain, &surface), + "xrCreateSwapchainAndroidSurfaceKHR")) + { + return false; + } + if (surface == nullptr) + { + setError("xrCreateSwapchainAndroidSurfaceKHR returned a null Surface"); + return false; + } + mVideoSurface = env->NewGlobalRef(surface); + LOGI("android surface swapchain %dx%d created", mSwapchainWidth, mSwapchainHeight); + return true; +} + +XrAction XrGoggleSession::boolAction(const char* name, const char* localized) +{ + XrActionCreateInfo info{XR_TYPE_ACTION_CREATE_INFO}; + info.actionType = XR_ACTION_TYPE_BOOLEAN_INPUT; + std::strncpy(info.actionName, name, XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(info.localizedActionName, localized, XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + XrAction action = XR_NULL_HANDLE; + if (!check(xrCreateAction(mActionSet, &info, &action), "xrCreateAction")) + { + return XR_NULL_HANDLE; + } + return action; +} + +bool XrGoggleSession::createActions() +{ + XrActionSetCreateInfo setInfo{XR_TYPE_ACTION_SET_CREATE_INFO}; + std::strncpy(setInfo.actionSetName, "goggle", XR_MAX_ACTION_SET_NAME_SIZE - 1); + std::strncpy(setInfo.localizedActionSetName, "Goggle", XR_MAX_LOCALIZED_ACTION_SET_NAME_SIZE - 1); + setInfo.priority = 0; + if (!check(xrCreateActionSet(mInstance, &setInfo, &mActionSet), "xrCreateActionSet")) + { + return false; + } + + mActionRecenter = boolAction("recenter", "Recenter view"); + mActionPassthrough = boolAction("passthrough", "Toggle passthrough"); + mActionRecord = boolAction("record", "Toggle recording"); + mActionLockMode = boolAction("lock_mode", "Toggle head lock"); + + XrActionCreateInfo stick{XR_TYPE_ACTION_CREATE_INFO}; + stick.actionType = XR_ACTION_TYPE_VECTOR2F_INPUT; + std::strncpy(stick.actionName, "size", XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(stick.localizedActionName, "Screen size", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + if (!check(xrCreateAction(mActionSet, &stick, &mActionSizeStick), "xrCreateAction(size)")) + { + return false; + } + std::strncpy(stick.actionName, "place", XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(stick.localizedActionName, "Screen distance", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + if (!check(xrCreateAction(mActionSet, &stick, &mActionPlaceStick), "xrCreateAction(place)")) + { + return false; + } + + XrActionCreateInfo haptic{XR_TYPE_ACTION_CREATE_INFO}; + haptic.actionType = XR_ACTION_TYPE_VIBRATION_OUTPUT; + std::strncpy(haptic.actionName, "haptic", XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(haptic.localizedActionName, "Haptic feedback", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + if (!check(xrCreateAction(mActionSet, &haptic, &mActionHaptic), "xrCreateAction(haptic)")) + { + return false; + } + + // Touch controller: A/B live on the right hand, X/Y on the left. + const XrActionSuggestedBinding touchBindings[] = { + {mActionRecenter, path("/user/hand/right/input/a/click")}, + {mActionPassthrough, path("/user/hand/right/input/b/click")}, + {mActionRecord, path("/user/hand/left/input/x/click")}, + {mActionLockMode, path("/user/hand/left/input/y/click")}, + {mActionSizeStick, path("/user/hand/right/input/thumbstick")}, + {mActionPlaceStick, path("/user/hand/left/input/thumbstick")}, + {mActionHaptic, path("/user/hand/left/output/haptic")}, + {mActionHaptic, path("/user/hand/right/output/haptic")}, + }; + XrInteractionProfileSuggestedBinding touch{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING}; + touch.interactionProfile = path("/interaction_profiles/oculus/touch_controller"); + touch.suggestedBindings = touchBindings; + touch.countSuggestedBindings = (uint32_t) (sizeof(touchBindings) / sizeof(touchBindings[0])); + if (XR_FAILED(xrSuggestInteractionProfileBindings(mInstance, &touch))) + { + LOGW("touch controller bindings rejected, controls may be unavailable"); + } + + // Bare minimum so something still works on a runtime without Touch controllers. + const XrActionSuggestedBinding simpleBindings[] = { + {mActionRecenter, path("/user/hand/right/input/select/click")}, + {mActionPassthrough, path("/user/hand/left/input/select/click")}, + {mActionHaptic, path("/user/hand/left/output/haptic")}, + {mActionHaptic, path("/user/hand/right/output/haptic")}, + }; + XrInteractionProfileSuggestedBinding simple{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING}; + simple.interactionProfile = path("/interaction_profiles/khr/simple_controller"); + simple.suggestedBindings = simpleBindings; + simple.countSuggestedBindings = (uint32_t) (sizeof(simpleBindings) / sizeof(simpleBindings[0])); + if (XR_FAILED(xrSuggestInteractionProfileBindings(mInstance, &simple))) + { + LOGW("simple controller bindings rejected"); + } + + XrSessionActionSetsAttachInfo attach{XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO}; + attach.countActionSets = 1; + attach.actionSets = &mActionSet; + return check(xrAttachSessionActionSets(mSession, &attach), "xrAttachSessionActionSets"); +} + +bool XrGoggleSession::createPassthrough() +{ + XrPassthroughCreateInfoFB info{XR_TYPE_PASSTHROUGH_CREATE_INFO_FB}; + info.flags = 0; + if (!check(mXrCreatePassthroughFB(mSession, &info, &mPassthrough), "xrCreatePassthroughFB")) + { + return false; + } + + XrPassthroughLayerCreateInfoFB layerInfo{XR_TYPE_PASSTHROUGH_LAYER_CREATE_INFO_FB}; + layerInfo.passthrough = mPassthrough; + layerInfo.purpose = XR_PASSTHROUGH_LAYER_PURPOSE_RECONSTRUCTION_FB; + layerInfo.flags = 0; + if (!check(mXrCreatePassthroughLayerFB(mSession, &layerInfo, &mPassthroughLayer), + "xrCreatePassthroughLayerFB")) + { + return false; + } + + if (mXrPassthroughLayerSetStyleFB != nullptr) + { + XrPassthroughStyleFB style{XR_TYPE_PASSTHROUGH_STYLE_FB}; + style.textureOpacityFactor = 1.0f; + style.edgeColor = XrColor4f{0.0f, 0.0f, 0.0f, 0.0f}; + mXrPassthroughLayerSetStyleFB(mPassthroughLayer, &style); + } + return true; +} + +// --------------------------------------------------------------------------------- +// settings +// --------------------------------------------------------------------------------- + +void XrGoggleSession::setSwapchainSize(int width, int height) +{ + if (width >= 320 && height >= 240 && width <= 4096 && height <= 4096) + { + mSwapchainWidth = width; + mSwapchainHeight = height; + mVideoWidth = width; + mVideoHeight = height; + } +} + +void XrGoggleSession::setVideoResolution(int width, int height) +{ + if (width > 0 && height > 0) + { + mVideoWidth = width; + mVideoHeight = height; + } +} + +void XrGoggleSession::setQuadDistance(float meters) +{ + mQuadDistance = clampf(meters, kMinDistance, kMaxDistance); +} + +void XrGoggleSession::setQuadWidth(float meters) +{ + mQuadWidth = clampf(meters, kMinWidth, kMaxWidth); +} + +void XrGoggleSession::setPassthrough(bool enabled) +{ + mPassthroughWanted = enabled; +} + +void XrGoggleSession::setSharpening(bool enabled) +{ + mSharpening = enabled; +} + +void XrGoggleSession::setHeadLocked(bool enabled) +{ + mHeadLocked = enabled; + if (!enabled) + { + // Anchor where the pilot is looking right now instead of at the origin. + mRecenterRequested = true; + } +} + +void XrGoggleSession::requestRefreshRate(float hz) +{ + mWantedRefreshRate = hz; +} + +std::vector XrGoggleSession::refreshRates() +{ + return mRefreshRates; +} + +void XrGoggleSession::requestHaptic(float amplitude, int durationMs) +{ + mPendingHapticAmplitude = clampf(amplitude, 0.0f, 1.0f); + mPendingHapticMs = durationMs < 1 ? 1 : durationMs; +} + +// --------------------------------------------------------------------------------- +// frame loop +// --------------------------------------------------------------------------------- + +void XrGoggleSession::runLoop(JNIEnv* env, jobject listener) +{ + bool exitLoop = false; + while (!exitLoop && !mStopRequested) + { + pollEvents(env, listener, &exitLoop); + if (exitLoop) break; + + if (!mSessionRunning) + { + // Idle without burning a core while the runtime has us backgrounded. + struct timespec ts = {0, 20 * 1000 * 1000}; + nanosleep(&ts, nullptr); + continue; + } + + applyPendingRefreshRate(); + syncActions(env, listener); + renderFrame(env, listener); + } + + if (mSessionRunning && mSession != XR_NULL_HANDLE) + { + xrRequestExitSession(mSession); + // Drain until the runtime actually stops the session, so teardown is clean. + for (int i = 0; i < 200 && mSessionRunning; ++i) + { + bool ignored = false; + pollEvents(env, listener, &ignored); + struct timespec ts = {0, 10 * 1000 * 1000}; + nanosleep(&ts, nullptr); + } + } +} + +void XrGoggleSession::pollEvents(JNIEnv* env, jobject listener, bool* exitLoop) +{ + (void) env; + (void) listener; + XrEventDataBuffer event{XR_TYPE_EVENT_DATA_BUFFER}; + while (true) + { + event.type = XR_TYPE_EVENT_DATA_BUFFER; + XrResult r = xrPollEvent(mInstance, &event); + if (r != XR_SUCCESS) break; + + switch (event.type) + { + case XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED: + { + const auto* changed = reinterpret_cast(&event); + mSessionState = changed->state; + LOGI("session state -> %d", (int) mSessionState); + if (mSessionState == XR_SESSION_STATE_READY) + { + XrSessionBeginInfo begin{XR_TYPE_SESSION_BEGIN_INFO}; + begin.primaryViewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + if (check(xrBeginSession(mSession, &begin), "xrBeginSession")) + { + mSessionRunning = true; + if (mHasPassthrough && mXrPassthroughStartFB != nullptr && + !mPassthroughRunning) + { + if (XR_SUCCEEDED(mXrPassthroughStartFB(mPassthrough))) + { + mPassthroughRunning = true; + if (mXrPassthroughLayerResumeFB != nullptr) + { + mXrPassthroughLayerResumeFB(mPassthroughLayer); + } + } + } + } + } + else if (mSessionState == XR_SESSION_STATE_STOPPING) + { + mSessionRunning = false; + xrEndSession(mSession); + } + else if (mSessionState == XR_SESSION_STATE_EXITING || + mSessionState == XR_SESSION_STATE_LOSS_PENDING) + { + mSessionRunning = false; + *exitLoop = true; + } + break; + } + case XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING: + { + LOGW("instance loss pending"); + mSessionRunning = false; + *exitLoop = true; + break; + } + case XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING: + { + // The runtime recentered underneath us; re-anchor on the next frame. + if (!mHeadLocked) mRecenterRequested = true; + break; + } + default: + break; + } + } +} + +void XrGoggleSession::applyPendingRefreshRate() +{ + if (!mHasRefreshRate) return; + const float wanted = mWantedRefreshRate.load(); + if (wanted <= 0.0f || std::fabs(wanted - mAppliedRefreshRate.load()) < 0.01f) return; + + // Only ask for a rate the runtime actually offers, otherwise it returns an error. + bool supported = mRefreshRates.empty(); + for (float hz : mRefreshRates) + { + if (std::fabs(hz - wanted) < 0.51f) + { + supported = true; + break; + } + } + if (!supported) + { + LOGW("refresh rate %.1f Hz not offered by the runtime", wanted); + mAppliedRefreshRate = wanted; + return; + } + if (XR_SUCCEEDED(mXrRequestDisplayRefreshRateFB(mSession, wanted))) + { + LOGI("display refresh rate set to %.1f Hz", wanted); + } + else + { + LOGW("xrRequestDisplayRefreshRateFB(%.1f) failed", wanted); + } + mAppliedRefreshRate = wanted; +} + +void XrGoggleSession::applyPendingHaptic() +{ + const float amplitude = mPendingHapticAmplitude.exchange(0.0f); + const int ms = mPendingHapticMs.exchange(0); + if (amplitude <= 0.0f || ms <= 0 || mActionHaptic == XR_NULL_HANDLE) return; + + XrHapticVibration vibration{XR_TYPE_HAPTIC_VIBRATION}; + vibration.amplitude = amplitude; + vibration.frequency = XR_FREQUENCY_UNSPECIFIED; + vibration.duration = (XrDuration) ms * 1000000LL; + + XrHapticActionInfo info{XR_TYPE_HAPTIC_ACTION_INFO}; + info.action = mActionHaptic; + info.subactionPath = XR_NULL_PATH; + xrApplyHapticFeedback(mSession, &info, (const XrHapticBaseHeader*) &vibration); +} + +void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) +{ + if (mActionSet == XR_NULL_HANDLE) return; + + XrActiveActionSet active{mActionSet, XR_NULL_PATH}; + XrActionsSyncInfo sync{XR_TYPE_ACTIONS_SYNC_INFO}; + sync.countActiveActionSets = 1; + sync.activeActionSets = &active; + if (XR_FAILED(xrSyncActions(mSession, &sync))) return; + + jclass cls = env->GetObjectClass(listener); + jmethodID onButton = env->GetMethodID(cls, "onXrButton", "(I)V"); + + struct + { + XrAction action; + int button; + } buttons[] = { + {mActionRecenter, BUTTON_RECENTER}, + {mActionPassthrough, BUTTON_PASSTHROUGH}, + {mActionRecord, BUTTON_RECORD}, + {mActionLockMode, BUTTON_LOCK_MODE}, + }; + + for (const auto& b : buttons) + { + if (b.action == XR_NULL_HANDLE) continue; + XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO}; + get.action = b.action; + XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN}; + if (XR_FAILED(xrGetActionStateBoolean(mSession, &get, &state))) continue; + // Rising edge only. + if (state.isActive && state.changedSinceLastSync && state.currentState) + { + switch (b.button) + { + case BUTTON_RECENTER: + mRecenterRequested = true; + requestHaptic(0.4f, 30); + break; + case BUTTON_PASSTHROUGH: + mPassthroughWanted = !mPassthroughWanted.load(); + requestHaptic(0.4f, 30); + break; + case BUTTON_LOCK_MODE: + setHeadLocked(!mHeadLocked.load()); + requestHaptic(0.4f, 30); + break; + default: + break; + } + if (onButton != nullptr) + { + env->CallVoidMethod(listener, onButton, (jint) b.button); + if (env->ExceptionCheck()) env->ExceptionClear(); + } + } + } + + const float dt = 1.0f / 72.0f; // good enough for a manual adjustment ramp + + XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO}; + XrActionStateVector2f stickState{XR_TYPE_ACTION_STATE_VECTOR2F}; + if (mActionSizeStick != XR_NULL_HANDLE) + { + get.action = mActionSizeStick; + stickState = XrActionStateVector2f{XR_TYPE_ACTION_STATE_VECTOR2F}; + if (XR_SUCCEEDED(xrGetActionStateVector2f(mSession, &get, &stickState)) && stickState.isActive) + { + const float y = stickState.currentState.y; + if (std::fabs(y) > kStickDeadzone) + { + setQuadWidth(mQuadWidth.load() + y * kWidthPerSec * dt); + } + } + } + if (mActionPlaceStick != XR_NULL_HANDLE) + { + get.action = mActionPlaceStick; + stickState = XrActionStateVector2f{XR_TYPE_ACTION_STATE_VECTOR2F}; + if (XR_SUCCEEDED(xrGetActionStateVector2f(mSession, &get, &stickState)) && stickState.isActive) + { + const float y = stickState.currentState.y; + if (std::fabs(y) > kStickDeadzone) + { + setQuadDistance(mQuadDistance.load() - y * kDistancePerSec * dt); + } + } + } +} + +void XrGoggleSession::recenterAt(XrTime time) +{ + XrSpaceLocation location{XR_TYPE_SPACE_LOCATION}; + if (XR_FAILED(xrLocateSpace(mViewSpace, mLocalSpace, time, &location))) + { + return; + } + if ((location.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) == 0 || + (location.locationFlags & XR_SPACE_LOCATION_POSITION_VALID_BIT) == 0) + { + return; + } + + const float yaw = yawOf(location.pose.orientation); + const float distance = mQuadDistance.load(); + mAnchorPose.orientation = quatFromYaw(yaw); + mAnchorPose.position = XrVector3f{location.pose.position.x - std::sin(yaw) * distance, + location.pose.position.y, + location.pose.position.z - std::cos(yaw) * distance}; + mAnchorValid = true; +} + +void XrGoggleSession::renderFrame(JNIEnv* env, jobject listener) +{ + (void) env; + (void) listener; + + XrFrameWaitInfo waitInfo{XR_TYPE_FRAME_WAIT_INFO}; + XrFrameState frameState{XR_TYPE_FRAME_STATE}; + if (XR_FAILED(xrWaitFrame(mSession, &waitInfo, &frameState))) return; + + XrFrameBeginInfo beginInfo{XR_TYPE_FRAME_BEGIN_INFO}; + if (XR_FAILED(xrBeginFrame(mSession, &beginInfo))) return; + + applyPendingHaptic(); + + if (mRecenterRequested.exchange(false) || (!mHeadLocked.load() && !mAnchorValid)) + { + recenterAt(frameState.predictedDisplayTime); + } + + const bool headLocked = mHeadLocked.load(); + const bool wantPassthrough = + mHasPassthrough && mPassthroughRunning && mPassthroughWanted.load(); + + // Video quad. Width is in meters at the configured distance, height follows the + // stream's aspect ratio so nothing is stretched. + const int vw = mVideoWidth.load(); + const int vh = mVideoHeight.load(); + const float aspect = (vw > 0 && vh > 0) ? (float) vh / (float) vw : 9.0f / 16.0f; + const float widthM = mQuadWidth.load(); + + XrCompositionLayerSettingsFB layerSettings{XR_TYPE_COMPOSITION_LAYER_SETTINGS_FB}; + layerSettings.layerFlags = XR_COMPOSITION_LAYER_SETTINGS_QUALITY_SHARPENING_BIT_FB; + + XrCompositionLayerQuad quad{XR_TYPE_COMPOSITION_LAYER_QUAD}; + quad.next = (mHasLayerSettings && mSharpening.load()) ? &layerSettings : nullptr; + quad.layerFlags = 0; // opaque: the video has to cover passthrough behind it + quad.space = headLocked ? mViewSpace : mLocalSpace; + quad.eyeVisibility = XR_EYE_VISIBILITY_BOTH; + quad.subImage.swapchain = mSwapchain; + quad.subImage.imageArrayIndex = 0; + // Clamp to the swapchain the runtime handed us; MediaCodec resizes the buffers to the + // stream resolution, which is normally the size this was created with. + quad.subImage.imageRect.offset = XrOffset2Di{0, 0}; + quad.subImage.imageRect.extent = + XrExtent2Di{vw > 0 && vw <= mSwapchainWidth ? vw : mSwapchainWidth, + vh > 0 && vh <= mSwapchainHeight ? vh : mSwapchainHeight}; + if (headLocked) + { + quad.pose = XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, -mQuadDistance.load()}}; + } + else + { + quad.pose = mAnchorPose; + } + quad.size = XrExtent2Df{widthM, widthM * aspect}; + + XrCompositionLayerPassthroughFB passthroughLayer{XR_TYPE_COMPOSITION_LAYER_PASSTHROUGH_FB}; + passthroughLayer.flags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT; + passthroughLayer.space = XR_NULL_HANDLE; + passthroughLayer.layerHandle = mPassthroughLayer; + + const XrCompositionLayerBaseHeader* layers[2]; + uint32_t layerCount = 0; + if (wantPassthrough) + { + layers[layerCount++] = (const XrCompositionLayerBaseHeader*) &passthroughLayer; + } + layers[layerCount++] = (const XrCompositionLayerBaseHeader*) &quad; + + XrFrameEndInfo endInfo{XR_TYPE_FRAME_END_INFO}; + endInfo.displayTime = frameState.predictedDisplayTime; + endInfo.environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; + // shouldRender false means the runtime wants nothing on screen (e.g. system overlay + // in front of us); submitting zero layers keeps the frame loop pacing intact. + endInfo.layerCount = frameState.shouldRender ? layerCount : 0; + endInfo.layers = frameState.shouldRender ? layers : nullptr; + xrEndFrame(mSession, &endInfo); +} + +// --------------------------------------------------------------------------------- +// teardown +// --------------------------------------------------------------------------------- + +void XrGoggleSession::destroy(JNIEnv* env) +{ + if (mPassthroughLayer != XR_NULL_HANDLE && mXrDestroyPassthroughLayerFB != nullptr) + { + mXrDestroyPassthroughLayerFB(mPassthroughLayer); + mPassthroughLayer = XR_NULL_HANDLE; + } + if (mPassthrough != XR_NULL_HANDLE && mXrDestroyPassthroughFB != nullptr) + { + if (mPassthroughRunning && mXrPassthroughPauseFB != nullptr) + { + mXrPassthroughPauseFB(mPassthrough); + } + mXrDestroyPassthroughFB(mPassthrough); + mPassthrough = XR_NULL_HANDLE; + mPassthroughRunning = false; + } + if (mActionSet != XR_NULL_HANDLE) + { + xrDestroyActionSet(mActionSet); + mActionSet = XR_NULL_HANDLE; + } + if (mSwapchain != XR_NULL_HANDLE) + { + xrDestroySwapchain(mSwapchain); + mSwapchain = XR_NULL_HANDLE; + } + if (mVideoSurface != nullptr) + { + env->DeleteGlobalRef(mVideoSurface); + mVideoSurface = nullptr; + } + if (mViewSpace != XR_NULL_HANDLE) + { + xrDestroySpace(mViewSpace); + mViewSpace = XR_NULL_HANDLE; + } + if (mLocalSpace != XR_NULL_HANDLE) + { + xrDestroySpace(mLocalSpace); + mLocalSpace = XR_NULL_HANDLE; + } + if (mSession != XR_NULL_HANDLE) + { + xrDestroySession(mSession); + mSession = XR_NULL_HANDLE; + } + if (mEglDisplay != EGL_NO_DISPLAY) + { + eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); + if (mEglSurface != EGL_NO_SURFACE) eglDestroySurface(mEglDisplay, mEglSurface); + if (mEglContext != EGL_NO_CONTEXT) eglDestroyContext(mEglDisplay, mEglContext); + eglTerminate(mEglDisplay); + mEglSurface = EGL_NO_SURFACE; + mEglContext = EGL_NO_CONTEXT; + mEglDisplay = EGL_NO_DISPLAY; + } + if (mInstance != XR_NULL_HANDLE) + { + xrDestroyInstance(mInstance); + mInstance = XR_NULL_HANDLE; + } +} diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h new file mode 100644 index 00000000..1dc3dede --- /dev/null +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -0,0 +1,185 @@ +// Immersive OpenXR presentation of the decoded video stream. +// +// The video never touches this code: xrCreateSwapchainAndroidSurfaceKHR hands out an +// Android Surface that is the producer side of a compositor swapchain, MediaCodec +// decodes straight into it, and the runtime composites it as a quad layer. There is no +// GL blit and no readback in the video path - the only reason an EGL context exists at +// all is that OpenXR requires a graphics binding for the session. + +#ifndef PIXELPILOT_XRGOGGLESESSION_H +#define PIXELPILOT_XRGOGGLESESSION_H + +#include +#include +#include +#include + +#define XR_USE_PLATFORM_ANDROID +#define XR_USE_GRAPHICS_API_OPENGL_ES +#include +#include + +#include +#include +#include +#include + +class XrGoggleSession +{ + public: + // Keep in sync with XrGoggleSession.java + enum Button + { + BUTTON_RECENTER = 0, + BUTTON_PASSTHROUGH = 1, + BUTTON_RECORD = 2, + BUTTON_LOCK_MODE = 3, + }; + + XrGoggleSession() = default; + ~XrGoggleSession() = default; + + // Brings up loader, instance, system, EGL, session, swapchain and actions. + // Returns false and leaves lastError() set on any failure; the caller is expected to + // fall back to the flat activity. + bool create(JNIEnv* env, jobject activity); + + // Blocks until requestStop() or an unrecoverable error. Must be called from the same + // thread that called create(), and that thread must be attached to the JVM: button + // presses are dispatched to listener.onXrButton(int) from here. + void runLoop(JNIEnv* env, jobject listener); + + void destroy(JNIEnv* env); + + void requestStop() { mStopRequested = true; } + + // The producer Surface for MediaCodec. Global ref, valid between create() and + // destroy(). + jobject videoSurface() const { return mVideoSurface; } + + std::string lastError(); + + std::vector refreshRates(); + + // --- live settings, read by the frame loop ------------------------------------- + void setVideoResolution(int width, int height); + void setQuadDistance(float meters); + void setQuadWidth(float meters); + void setPassthrough(bool enabled); + void setSharpening(bool enabled); + void setHeadLocked(bool enabled); + void requestRefreshRate(float hz); + void requestRecenter() { mRecenterRequested = true; } + // amplitude 0..1, duration in milliseconds + void requestHaptic(float amplitude, int durationMs); + + float quadDistance() const { return mQuadDistance; } + float quadWidth() const { return mQuadWidth; } + bool headLocked() const { return mHeadLocked; } + bool passthroughEnabled() const { return mPassthroughWanted; } + + // Swapchain geometry is fixed at create() time; the caller seeds it from the last + // known stream resolution. + void setSwapchainSize(int width, int height); + + private: + bool initLoader(JNIEnv* env, jobject activity); + bool createInstance(JNIEnv* env, jobject activity); + bool createEgl(); + bool createSession(); + bool createSpaces(); + bool createSwapchain(JNIEnv* env); + bool createActions(); + bool createPassthrough(); + + void pollEvents(JNIEnv* env, jobject listener, bool* exitLoop); + void renderFrame(JNIEnv* env, jobject listener); + void syncActions(JNIEnv* env, jobject listener); + void applyPendingHaptic(); + void applyPendingRefreshRate(); + void recenterAt(XrTime time); + + bool extensionSupported(const char* name) const; + bool check(XrResult result, const char* what); + void setError(const std::string& what); + XrPath path(const char* str); + XrAction boolAction(const char* name, const char* localized); + + // --- OpenXR handles ----------------------------------------------------------- + XrInstance mInstance = XR_NULL_HANDLE; + XrSystemId mSystemId = XR_NULL_SYSTEM_ID; + XrSession mSession = XR_NULL_HANDLE; + XrSpace mViewSpace = XR_NULL_HANDLE; + XrSpace mLocalSpace = XR_NULL_HANDLE; + XrSwapchain mSwapchain = XR_NULL_HANDLE; + jobject mVideoSurface = nullptr; + + XrPassthroughFB mPassthrough = XR_NULL_HANDLE; + XrPassthroughLayerFB mPassthroughLayer = XR_NULL_HANDLE; + bool mPassthroughRunning = false; + + XrActionSet mActionSet = XR_NULL_HANDLE; + XrAction mActionRecenter = XR_NULL_HANDLE; + XrAction mActionPassthrough = XR_NULL_HANDLE; + XrAction mActionRecord = XR_NULL_HANDLE; + XrAction mActionLockMode = XR_NULL_HANDLE; + XrAction mActionSizeStick = XR_NULL_HANDLE; + XrAction mActionPlaceStick = XR_NULL_HANDLE; + XrAction mActionHaptic = XR_NULL_HANDLE; + + // --- EGL ---------------------------------------------------------------------- + EGLDisplay mEglDisplay = EGL_NO_DISPLAY; + EGLConfig mEglConfig = nullptr; + EGLSurface mEglSurface = EGL_NO_SURFACE; + EGLContext mEglContext = EGL_NO_CONTEXT; + + // --- extension entry points --------------------------------------------------- + PFN_xrCreateSwapchainAndroidSurfaceKHR mXrCreateSwapchainAndroidSurfaceKHR = nullptr; + PFN_xrCreatePassthroughFB mXrCreatePassthroughFB = nullptr; + PFN_xrDestroyPassthroughFB mXrDestroyPassthroughFB = nullptr; + PFN_xrPassthroughStartFB mXrPassthroughStartFB = nullptr; + PFN_xrPassthroughPauseFB mXrPassthroughPauseFB = nullptr; + PFN_xrCreatePassthroughLayerFB mXrCreatePassthroughLayerFB = nullptr; + PFN_xrDestroyPassthroughLayerFB mXrDestroyPassthroughLayerFB = nullptr; + PFN_xrPassthroughLayerResumeFB mXrPassthroughLayerResumeFB = nullptr; + PFN_xrPassthroughLayerPauseFB mXrPassthroughLayerPauseFB = nullptr; + PFN_xrPassthroughLayerSetStyleFB mXrPassthroughLayerSetStyleFB = nullptr; + PFN_xrEnumerateDisplayRefreshRatesFB mXrEnumerateDisplayRefreshRatesFB = nullptr; + PFN_xrRequestDisplayRefreshRateFB mXrRequestDisplayRefreshRateFB = nullptr; + + std::vector mAvailableExtensions; + bool mHasPassthrough = false; + bool mHasLayerSettings = false; + bool mHasRefreshRate = false; + + // --- state -------------------------------------------------------------------- + XrSessionState mSessionState = XR_SESSION_STATE_UNKNOWN; + bool mSessionRunning = false; + std::atomic mStopRequested{false}; + std::atomic mRecenterRequested{false}; + + std::atomic mVideoWidth{1920}; + std::atomic mVideoHeight{1080}; + int mSwapchainWidth = 1920; + int mSwapchainHeight = 1080; + std::atomic mQuadDistance{1.6f}; + std::atomic mQuadWidth{2.2f}; + std::atomic mHeadLocked{true}; + std::atomic mPassthroughWanted{false}; + std::atomic mSharpening{true}; + std::atomic mWantedRefreshRate{0.0f}; + std::atomic mAppliedRefreshRate{0.0f}; + + std::atomic mPendingHapticAmplitude{0.0f}; + std::atomic mPendingHapticMs{0}; + + // Anchor for world-locked mode, in the LOCAL reference space. + XrPosef mAnchorPose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; + bool mAnchorValid = false; + + std::vector mRefreshRates; + std::mutex mErrorMutex; + std::string mLastError; +}; + +#endif // PIXELPILOT_XRGOGGLESESSION_H diff --git a/app/xr/src/main/cpp/jni_bindings.cpp b/app/xr/src/main/cpp/jni_bindings.cpp new file mode 100644 index 00000000..07e02283 --- /dev/null +++ b/app/xr/src/main/cpp/jni_bindings.cpp @@ -0,0 +1,179 @@ +#include + +#include "XrGoggleSession.h" + +#define JNI_METHOD(return_type, method_name) \ + extern "C" JNIEXPORT return_type JNICALL Java_com_openipc_xr_XrGoggleSession_##method_name + +static inline XrGoggleSession* native(jlong handle) +{ + return reinterpret_cast(handle); +} + +JNI_METHOD(jlong, nativeAlloc)(JNIEnv* env, jclass clazz) +{ + (void) env; + (void) clazz; + return reinterpret_cast(new XrGoggleSession()); +} + +JNI_METHOD(void, nativeFree)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + delete native(handle); +} + +JNI_METHOD(void, nativeSetSwapchainSize) +(JNIEnv* env, jclass clazz, jlong handle, jint width, jint height) +{ + (void) env; + (void) clazz; + native(handle)->setSwapchainSize(width, height); +} + +JNI_METHOD(jboolean, nativeCreate)(JNIEnv* env, jclass clazz, jlong handle, jobject activity) +{ + (void) clazz; + return native(handle)->create(env, activity) ? JNI_TRUE : JNI_FALSE; +} + +JNI_METHOD(jobject, nativeGetVideoSurface)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + return native(handle)->videoSurface(); +} + +JNI_METHOD(void, nativeRunLoop)(JNIEnv* env, jclass clazz, jlong handle, jobject listener) +{ + (void) clazz; + native(handle)->runLoop(env, listener); +} + +JNI_METHOD(void, nativeRequestStop)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + native(handle)->requestStop(); +} + +JNI_METHOD(void, nativeDestroy)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) clazz; + native(handle)->destroy(env); +} + +JNI_METHOD(jstring, nativeLastError)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) clazz; + const std::string error = native(handle)->lastError(); + if (error.empty()) return nullptr; + return env->NewStringUTF(error.c_str()); +} + +JNI_METHOD(jfloatArray, nativeGetRefreshRates)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) clazz; + const std::vector rates = native(handle)->refreshRates(); + jfloatArray out = env->NewFloatArray((jsize) rates.size()); + if (out == nullptr) return nullptr; + if (!rates.empty()) + { + env->SetFloatArrayRegion(out, 0, (jsize) rates.size(), rates.data()); + } + return out; +} + +JNI_METHOD(void, nativeSetVideoResolution) +(JNIEnv* env, jclass clazz, jlong handle, jint width, jint height) +{ + (void) env; + (void) clazz; + native(handle)->setVideoResolution(width, height); +} + +JNI_METHOD(void, nativeSetQuadDistance)(JNIEnv* env, jclass clazz, jlong handle, jfloat meters) +{ + (void) env; + (void) clazz; + native(handle)->setQuadDistance(meters); +} + +JNI_METHOD(void, nativeSetQuadWidth)(JNIEnv* env, jclass clazz, jlong handle, jfloat meters) +{ + (void) env; + (void) clazz; + native(handle)->setQuadWidth(meters); +} + +JNI_METHOD(void, nativeSetPassthrough)(JNIEnv* env, jclass clazz, jlong handle, jboolean enabled) +{ + (void) env; + (void) clazz; + native(handle)->setPassthrough(enabled == JNI_TRUE); +} + +JNI_METHOD(void, nativeSetSharpening)(JNIEnv* env, jclass clazz, jlong handle, jboolean enabled) +{ + (void) env; + (void) clazz; + native(handle)->setSharpening(enabled == JNI_TRUE); +} + +JNI_METHOD(void, nativeSetHeadLocked)(JNIEnv* env, jclass clazz, jlong handle, jboolean enabled) +{ + (void) env; + (void) clazz; + native(handle)->setHeadLocked(enabled == JNI_TRUE); +} + +JNI_METHOD(void, nativeRequestRefreshRate)(JNIEnv* env, jclass clazz, jlong handle, jfloat hz) +{ + (void) env; + (void) clazz; + native(handle)->requestRefreshRate(hz); +} + +JNI_METHOD(void, nativeRequestRecenter)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + native(handle)->requestRecenter(); +} + +JNI_METHOD(void, nativeHaptic) +(JNIEnv* env, jclass clazz, jlong handle, jfloat amplitude, jint durationMs) +{ + (void) env; + (void) clazz; + native(handle)->requestHaptic(amplitude, durationMs); +} + +JNI_METHOD(jfloat, nativeGetQuadDistance)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + return native(handle)->quadDistance(); +} + +JNI_METHOD(jfloat, nativeGetQuadWidth)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + return native(handle)->quadWidth(); +} + +JNI_METHOD(jboolean, nativeIsHeadLocked)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + return native(handle)->headLocked() ? JNI_TRUE : JNI_FALSE; +} + +JNI_METHOD(jboolean, nativeIsPassthroughEnabled)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + return native(handle)->passthroughEnabled() ? JNI_TRUE : JNI_FALSE; +} diff --git a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java new file mode 100644 index 00000000..9423e23c --- /dev/null +++ b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java @@ -0,0 +1,301 @@ +package com.openipc.xr; + +import android.app.Activity; +import android.content.Context; +import android.content.pm.PackageManager; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; +import android.view.Surface; + +import androidx.annotation.Keep; +import androidx.annotation.Nullable; + +/** + * Immersive OpenXR presentation of the decoded video stream. + * + *

The video is not rendered by this class. {@code xrCreateSwapchainAndroidSurfaceKHR} + * hands out a {@link Surface} that is the producer side of a compositor swapchain; + * MediaCodec decodes straight into it and the runtime composites it as a head locked + * quad layer. No GL blit, no readback, no view hierarchy. + * + *

All native calls happen on a dedicated thread owned by this class. Listener + * callbacks are posted to the main looper. + */ +@Keep +public final class XrGoggleSession { + // Keep in sync with XrGoggleSession.h + public static final int BUTTON_RECENTER = 0; + public static final int BUTTON_PASSTHROUGH = 1; + public static final int BUTTON_RECORD = 2; + public static final int BUTTON_LOCK_MODE = 3; + + private static final String TAG = "pixelpilot-xr"; + + private static boolean libraryLoaded; + + static { + try { + System.loadLibrary("XrGoggle"); + libraryLoaded = true; + } catch (Throwable t) { + // A device without the OpenXR loader still has to be able to run the flat app. + Log.w(TAG, "libXrGoggle not available: " + t.getMessage()); + libraryLoaded = false; + } + } + + public interface Listener { + /** The compositor swapchain is ready; hand this Surface to the decoder. */ + void onXrReady(Surface videoSurface); + + /** One of the BUTTON_* constants was pressed on a controller. */ + void onXrButton(int button); + + /** Session ended. {@code error} is null for a normal stop. */ + void onXrStopped(@Nullable String error); + } + + private final Activity activity; + private final Listener listener; + private final Handler main = new Handler(Looper.getMainLooper()); + + private long handle; + private Thread thread; + private volatile boolean stopping; + + public XrGoggleSession(Activity activity, Listener listener) { + this.activity = activity; + this.listener = listener; + } + + /** + * Cheap pre-flight check used to decide whether to offer the mode at all. It says + * "this looks like a headset", not "a session will succeed" — only {@link #start()} + * can tell you that. + */ + public static boolean isSupportedDevice(Context context) { + if (!libraryLoaded) { + return false; + } + PackageManager pm = context.getPackageManager(); + if (pm.hasSystemFeature("android.hardware.vr.headtracking") + || pm.hasSystemFeature("android.software.xr.immersive") + || pm.hasSystemFeature("oculus.software.handtracking")) { + return true; + } + // Horizon OS shell, present on every Quest. + try { + pm.getPackageInfo("com.oculus.vrshell", 0); + return true; + } catch (PackageManager.NameNotFoundException ignored) { + return false; + } + } + + public static boolean isLibraryLoaded() { + return libraryLoaded; + } + + /** Seeds the swapchain geometry. Must be called before {@link #start()}. */ + public void setSwapchainSize(int width, int height) { + ensureHandle(); + if (handle != 0) { + nativeSetSwapchainSize(handle, width, height); + } + } + + /** + * Creates the session on a background thread and starts the frame loop. The listener + * gets either {@link Listener#onXrReady} or {@link Listener#onXrStopped} with an + * error. + */ + public void start() { + if (thread != null) { + return; + } + ensureHandle(); + if (handle == 0) { + main.post(() -> listener.onXrStopped("OpenXR native library is not available")); + return; + } + thread = new Thread(this::threadBody, "xr-goggle"); + // The frame loop drives the compositor; it must not lose to background work. + thread.setPriority(Thread.MAX_PRIORITY); + thread.start(); + } + + /** Asks the frame loop to exit. Returns immediately. */ + public void stop() { + stopping = true; + if (handle != 0) { + nativeRequestStop(handle); + } + } + + /** Stops and joins, then releases the native object. Safe to call twice. */ + public void release() { + stop(); + Thread t = thread; + if (t != null) { + try { + t.join(2000); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + thread = null; + } + if (handle != 0) { + nativeFree(handle); + handle = 0; + } + } + + private void ensureHandle() { + if (libraryLoaded && handle == 0) { + handle = nativeAlloc(); + } + } + + private void threadBody() { + final long h = handle; + if (!nativeCreate(h, activity)) { + final String error = errorOr("OpenXR session could not be created"); + main.post(() -> listener.onXrStopped(error)); + return; + } + final Surface surface = nativeGetVideoSurface(h); + if (surface == null) { + main.post(() -> listener.onXrStopped("OpenXR gave us no video surface")); + nativeDestroy(h); + return; + } + main.post(() -> listener.onXrReady(surface)); + + // Blocks until stop() or an unrecoverable runtime error. Button presses arrive + // through onXrButton() below, called from this thread. + nativeRunLoop(h, this); + + final String error = stopping ? null : errorOr("OpenXR session ended unexpectedly"); + nativeDestroy(h); + main.post(() -> listener.onXrStopped(error)); + } + + private String errorOr(String fallback) { + String error = handle != 0 ? nativeLastError(handle) : null; + return error != null ? error : fallback; + } + + /** Called from the XR frame loop thread. */ + @Keep + void onXrButton(int button) { + main.post(() -> listener.onXrButton(button)); + } + + // --- live settings --------------------------------------------------------------- + + public void setVideoResolution(int width, int height) { + if (handle != 0) nativeSetVideoResolution(handle, width, height); + } + + public void setQuadDistance(float meters) { + if (handle != 0) nativeSetQuadDistance(handle, meters); + } + + public void setQuadWidth(float meters) { + if (handle != 0) nativeSetQuadWidth(handle, meters); + } + + public void setPassthrough(boolean enabled) { + if (handle != 0) nativeSetPassthrough(handle, enabled); + } + + public void setSharpening(boolean enabled) { + if (handle != 0) nativeSetSharpening(handle, enabled); + } + + public void setHeadLocked(boolean enabled) { + if (handle != 0) nativeSetHeadLocked(handle, enabled); + } + + /** 0 leaves the runtime default alone. */ + public void requestRefreshRate(float hz) { + if (handle != 0) nativeRequestRefreshRate(handle, hz); + } + + public void recenter() { + if (handle != 0) nativeRequestRecenter(handle); + } + + public void haptic(float amplitude, int durationMs) { + if (handle != 0) nativeHaptic(handle, amplitude, durationMs); + } + + public float[] refreshRates() { + return handle != 0 ? nativeGetRefreshRates(handle) : new float[0]; + } + + public float quadDistance() { + return handle != 0 ? nativeGetQuadDistance(handle) : 0f; + } + + public float quadWidth() { + return handle != 0 ? nativeGetQuadWidth(handle) : 0f; + } + + public boolean isHeadLocked() { + return handle != 0 && nativeIsHeadLocked(handle); + } + + public boolean isPassthroughEnabled() { + return handle != 0 && nativeIsPassthroughEnabled(handle); + } + + // --- native --------------------------------------------------------------------- + + private static native long nativeAlloc(); + + private static native void nativeFree(long handle); + + private static native void nativeSetSwapchainSize(long handle, int width, int height); + + private static native boolean nativeCreate(long handle, Activity activity); + + private static native Surface nativeGetVideoSurface(long handle); + + private static native void nativeRunLoop(long handle, XrGoggleSession listener); + + private static native void nativeRequestStop(long handle); + + private static native void nativeDestroy(long handle); + + private static native String nativeLastError(long handle); + + private static native float[] nativeGetRefreshRates(long handle); + + private static native void nativeSetVideoResolution(long handle, int width, int height); + + private static native void nativeSetQuadDistance(long handle, float meters); + + private static native void nativeSetQuadWidth(long handle, float meters); + + private static native void nativeSetPassthrough(long handle, boolean enabled); + + private static native void nativeSetSharpening(long handle, boolean enabled); + + private static native void nativeSetHeadLocked(long handle, boolean enabled); + + private static native void nativeRequestRefreshRate(long handle, float hz); + + private static native void nativeRequestRecenter(long handle); + + private static native void nativeHaptic(long handle, float amplitude, int durationMs); + + private static native float nativeGetQuadDistance(long handle); + + private static native float nativeGetQuadWidth(long handle); + + private static native boolean nativeIsHeadLocked(long handle); + + private static native boolean nativeIsPassthroughEnabled(long handle); +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b1540ecf..d2075b43 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,18 +2,22 @@ agp = "8.5.2" coreKtx = "1.13.1" appcompat = "1.7.0" +annotation = "1.8.2" material = "1.12.0" constraintlayout = "2.1.4" mpandroidchartVersion = "3.1.0" mediapipe = "0.10.29" +openxr = "1.1.49" [libraries] core-ktx = { module = "androidx.core:core-ktx", version.ref = "coreKtx" } appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" } +annotation = { module = "androidx.annotation:annotation", version.ref = "annotation" } material = { group = "com.google.android.material", name = "material", version.ref = "material" } constraintlayout = { group = "androidx.constraintlayout", name = "constraintlayout", version.ref = "constraintlayout" } philjay-mpandroidchart = { module = "com.github.Philjay:mpandroidchart", version.ref = "mpandroidchartVersion" } mediapipe-tasks-vision = { module = "com.google.mediapipe:tasks-vision", version.ref = "mediapipe" } +openxr-loader = { module = "org.khronos.openxr:openxr_loader_for_android", version.ref = "openxr" } [plugins] androidApplication = { id = "com.android.application", version.ref = "agp" } diff --git a/settings.gradle b/settings.gradle index b383549c..12d4a68b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -27,4 +27,5 @@ rootProject.name = "PixelPilot" include(":app") include(":app:videonative") include(":app:wfbngrtl8812") +include(":app:xr") include(":app:mavlink") From df3b9a7b37acec21a1f7d9b028b15521dae2a801 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:09:42 +0200 Subject: [PATCH 02/23] Only run passthrough while it is on screen Passthrough was started at session begin and merely withheld from the frame's layer list when disabled, so the cameras and the reconstruction pipeline kept running for the whole flight. On a headset that is already thermally tight and on battery that is a real cost for nothing. xrPassthroughStartFB / xrPassthroughPauseFB now follow the setting, checked once per frame before the layers are submitted. --- app/xr/src/main/cpp/XrGoggleSession.cpp | 52 +++++++++++++++++++------ app/xr/src/main/cpp/XrGoggleSession.h | 1 + 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index aa58f8d9..729f784c 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -644,6 +644,7 @@ void XrGoggleSession::runLoop(JNIEnv* env, jobject listener) applyPendingRefreshRate(); syncActions(env, listener); + applyPassthroughState(); renderFrame(env, listener); } @@ -686,18 +687,6 @@ void XrGoggleSession::pollEvents(JNIEnv* env, jobject listener, bool* exitLoop) if (check(xrBeginSession(mSession, &begin), "xrBeginSession")) { mSessionRunning = true; - if (mHasPassthrough && mXrPassthroughStartFB != nullptr && - !mPassthroughRunning) - { - if (XR_SUCCEEDED(mXrPassthroughStartFB(mPassthrough))) - { - mPassthroughRunning = true; - if (mXrPassthroughLayerResumeFB != nullptr) - { - mXrPassthroughLayerResumeFB(mPassthroughLayer); - } - } - } } } else if (mSessionState == XR_SESSION_STATE_STOPPING) @@ -765,6 +754,45 @@ void XrGoggleSession::applyPendingRefreshRate() mAppliedRefreshRate = wanted; } +// Passthrough keeps the cameras and reconstruction running, so only pay for it while it +// is on screen. +void XrGoggleSession::applyPassthroughState() +{ + if (!mHasPassthrough || mPassthrough == XR_NULL_HANDLE) return; + const bool wanted = mPassthroughWanted.load(); + if (wanted == mPassthroughRunning) return; + + if (wanted) + { + if (mXrPassthroughStartFB == nullptr) return; + if (!XR_SUCCEEDED(mXrPassthroughStartFB(mPassthrough))) + { + LOGW("xrPassthroughStartFB failed"); + mPassthroughWanted = false; + return; + } + if (mXrPassthroughLayerResumeFB != nullptr) + { + mXrPassthroughLayerResumeFB(mPassthroughLayer); + } + mPassthroughRunning = true; + LOGI("passthrough started"); + } + else + { + if (mXrPassthroughLayerPauseFB != nullptr) + { + mXrPassthroughLayerPauseFB(mPassthroughLayer); + } + if (mXrPassthroughPauseFB != nullptr) + { + mXrPassthroughPauseFB(mPassthrough); + } + mPassthroughRunning = false; + LOGI("passthrough paused"); + } +} + void XrGoggleSession::applyPendingHaptic() { const float amplitude = mPendingHapticAmplitude.exchange(0.0f); diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index 1dc3dede..9297ce19 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -96,6 +96,7 @@ class XrGoggleSession void renderFrame(JNIEnv* env, jobject listener); void syncActions(JNIEnv* env, jobject listener); void applyPendingHaptic(); + void applyPassthroughState(); void applyPendingRefreshRate(); void recenterAt(XrTime time); From 0ac7161d2ef2333b7ea2935dfb4d2b4def3f38a7 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:26:23 +0200 Subject: [PATCH 03/23] Fix: NPE in wirelessInfo() outside VideoActivity wirelessInfo() is static and read the static WifiManager field, which only VideoActivity.initializeUI() ever assigns. WfbLinkManager calls it from the "no compatible wifi adapter found" branch, so any other host of WfbLinkManager - the immersive activity - crashed there with NullPointerException: WifiManager.getConnectionInfo() on a null object at VideoActivity.wirelessInfo(VideoActivity.java:183) at WfbLinkManager.refreshAdapters(WfbLinkManager.java:174) Confirmed on a Quest 3 (Horizon OS, Android 14): launching XrVideoActivity with no adapter attached killed the process in onResume before the OpenXR session could start. wirelessInfo() now takes a Context, fetches the service itself and null checks both the manager and the WifiInfo, and the static field is gone - so this cannot come back through a different entry point. --- .../com/openipc/pixelpilot/VideoActivity.java | 21 +++++++++++++++---- .../openipc/pixelpilot/WfbLinkManager.java | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index fb6edca8..8ad0939f 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -16,6 +16,7 @@ import android.hardware.usb.UsbManager; import android.net.Uri; import android.net.VpnService; +import android.net.wifi.WifiInfo; import android.net.wifi.WifiManager; import android.os.BatteryManager; import android.os.Build; @@ -115,7 +116,6 @@ public class VideoActivity extends AppCompatActivity implements IVideoParamsChan private static final long MODEL_LITE2_BYTES = 23096891L; private static final String PREF_OD_CUSTOM_MODEL_URI = "od_custom_model_uri"; private static final String PREF_OD_CUSTOM_MODEL_NAME = "od_custom_model_name"; - private static WifiManager wifiManager; final Handler handler = new Handler(Looper.getMainLooper()); final Runnable runnable = new Runnable() { public void run() { @@ -179,8 +179,22 @@ public static int getBandwidth(Context context) { Context.MODE_PRIVATE).getInt("bandwidth", 20); } - public static String wirelessInfo() { - int address = wifiManager.getConnectionInfo().getIpAddress(); + /** + * IPv4 address of the phone/headset on its own wifi, used to tell the pilot where to + * push a stream when no adapter is attached. Returns null when there is nothing to + * report - a headset in particular may have no WifiManager state to read at all. + */ + public static String wirelessInfo(Context context) { + WifiManager manager = + (WifiManager) context.getApplicationContext().getSystemService(Context.WIFI_SERVICE); + if (manager == null) { + return null; + } + WifiInfo info = manager.getConnectionInfo(); + if (info == null) { + return null; + } + int address = info.getIpAddress(); return (address == 0) ? null : Formatter.formatIpAddress(address); } @@ -333,7 +347,6 @@ private void initializeUI() { getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); getWindow().getDecorView().setSystemUiVisibility(View.SYSTEM_UI_FLAG_HIDE_NAVIGATION); - wifiManager = (WifiManager) getSystemService(WIFI_SERVICE); } // ---------------------------------------------------------------------------- diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java index aa73c37d..58c32789 100644 --- a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java +++ b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java @@ -171,7 +171,7 @@ public synchronized void refreshAdapters() { if (activeWifiAdapters.isEmpty()) { status.showLinkMessage("No compatible wifi adapter found."); - String wifi = VideoActivity.wirelessInfo(); + String wifi = VideoActivity.wirelessInfo(context); if (wifi != null) { status.showLocalStreamHint("udp://" + wifi + ":5600"); } From b96ed6f48958989955ff4243bbc45325bab64435 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:31:36 +0200 Subject: [PATCH 04/23] Let the immersive activity launch without Touch controllers Horizon OS routes an activity declared with com.oculus.intent.category.VR through RequiresControllersLaunchInterceptor and refuses to start it while the controllers are off: ActivityLaunchInterceptorController: RequiresControllersLaunchInterceptor: com.openipc.pixelpilot/.XrVideoActivity -> com.oculus.vrshell/...LaunchCheckControllerRequiredDialogActivity An app is treated as controller-required unless it advertises hand tracking. For a ground station that is the wrong default: the pilot is holding a radio, and video, link and DVR do not need a controller - only the optional in-flight shortcuts do. Declaring oculus.software.handtracking as required="false" lets the session start regardless, and the controller bindings still work when the controllers are on. --- app/src/main/AndroidManifest.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index d756783a..9f3c13b8 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -12,6 +12,14 @@ + + From f5cad101816c11e6b4c478667401707f50b217bc Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:33:46 +0200 Subject: [PATCH 05/23] Point the OpenXR loader at the Horizon OS runtime explicitly The generic Khronos loader cannot find a runtime on a Quest 3, verified on the device (Horizon OS, Android 14): - /system/etc/openxr/1/active_runtime.json does not exist - the Khronos broker authorities are registered but resolve to internal.horizonos.openxr.DisabledRuntimeBroker / DisabledRuntimeSystemBroker, and querying content://org.khronos.openxr.runtime_broker/openxr/1/abi/arm64-v8a/runtimes/active returns "No result found" - libopenxr_loader.so from openxr_loader_for_android has no reference to libopenxr_forwardloader.so at all The runtime is reached through libopenxr_forwardloader.so in the VrDriver APEX. Its lib directory is added to every app's linker namespace, and its only exports are xrNegotiateLoaderRuntimeInterface and xrInitializeLoaderKHR - so as far as a loader is concerned it *is* the runtime. resolveRuntime() therefore writes a runtime manifest into the app's files dir and sets XR_RUNTIME_JSON, trying in order: 1. nothing at all, so a device whose broker does work is left alone 2. library_path "libopenxr_forwardloader.so" (bare name = system search path) 3. the absolute APEX path, for a loader that insists on one Each candidate is probed by asking for the instance extension list and requiring XR_KHR_android_surface_swapchain to be in it, so "the call did not fail" is not mistaken for "a runtime answered". Which candidate won is logged. --- app/xr/src/main/cpp/XrGoggleSession.cpp | 123 ++++++++++++++++++ app/xr/src/main/cpp/XrGoggleSession.h | 8 ++ app/xr/src/main/cpp/jni_bindings.cpp | 12 ++ .../java/com/openipc/xr/XrGoggleSession.java | 4 + 4 files changed, 147 insertions(+) diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index 729f784c..c88fd1de 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -1,11 +1,13 @@ #include "XrGoggleSession.h" #include +#include #include #include #include #include +#include #define TAG "pixelpilot-xr" #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) @@ -106,6 +108,7 @@ bool XrGoggleSession::extensionSupported(const char* name) const bool XrGoggleSession::create(JNIEnv* env, jobject activity) { if (!initLoader(env, activity)) return false; + if (!resolveRuntime()) return false; if (!createInstance(env, activity)) return false; if (!createEgl()) return false; if (!createSession()) return false; @@ -136,6 +139,126 @@ bool XrGoggleSession::create(JNIEnv* env, jobject activity) return true; } +// --------------------------------------------------------------------------------- +// runtime discovery +// --------------------------------------------------------------------------------- + +namespace +{ +#if defined(__aarch64__) +constexpr const char* kAbi = "arm64-v8a"; +#elif defined(__arm__) +constexpr const char* kAbi = "armeabi-v7a"; +#elif defined(__x86_64__) +constexpr const char* kAbi = "x86_64"; +#else +constexpr const char* kAbi = "x86"; +#endif +} // namespace + +// True when a real runtime answered - a runtime-provided extension has to be in the list, +// not just "the call did not fail". +bool XrGoggleSession::runtimeAnswers() +{ + uint32_t count = 0; + if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, 0, &count, nullptr)) || count == 0) + { + return false; + } + std::vector props(count, {XR_TYPE_EXTENSION_PROPERTIES}); + if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, count, &count, props.data()))) + { + return false; + } + for (const auto& p : props) + { + if (std::strcmp(p.extensionName, XR_KHR_ANDROID_SURFACE_SWAPCHAIN_EXTENSION_NAME) == 0) + { + return true; + } + } + return false; +} + +// Drops a runtime manifest next to our files and points the loader at it. +bool XrGoggleSession::pointLoaderAt(const char* libraryPath) +{ + if (mManifestDir.empty()) return false; + const std::string manifest = mManifestDir + "/openxr_runtime.json"; + { + std::ofstream out(manifest, std::ios::trunc); + if (!out) return false; + out << "{\n" + << " \"file_format_version\": \"1.0.0\",\n" + << " \"runtime\": {\n" + << " \"name\": \"Android system OpenXR runtime\",\n" + << " \"library_path\": \"" << libraryPath << "\"\n" + << " }\n" + << "}\n"; + } + setenv("XR_RUNTIME_JSON", manifest.c_str(), 1); + return true; +} + +/* + * Horizon OS does not let the generic Khronos loader find its runtime on its own: + * + * - there is no /system/etc/openxr/1/active_runtime.json + * - the Khronos broker authorities exist but resolve to + * internal.horizonos.openxr.DisabledRuntimeBroker, which returns no rows + * - the loader binary has no knowledge of libopenxr_forwardloader.so + * + * The runtime is reached through libopenxr_forwardloader.so from the VrDriver APEX. That + * library is on our linker search path (VrDriver.apk!/lib/ is added to every app's + * namespace) and exports xrNegotiateLoaderRuntimeInterface, i.e. as far as a loader is + * concerned it *is* the runtime. So if nothing answers out of the box, hand the loader an + * explicit manifest pointing at it. + * + * Candidates are tried in order and the one that answers is kept, so a device where the + * broker does work is left alone. + */ +bool XrGoggleSession::resolveRuntime() +{ + if (runtimeAnswers()) + { + LOGI("runtime found without an override"); + mRuntimeLibrary = ""; + return true; + } + + const std::string apkPath = std::string("/apex/com.meta.xr/priv-app/VrDriver/VrDriver.apk!/lib/") + + kAbi + "/libopenxr_forwardloader.so"; + const char* candidates[] = { + // A bare file name means "use the system library search path", which is where the + // APEX library actually lives for us. + "libopenxr_forwardloader.so", + // Some loaders insist on an absolute path; Android's linker understands apk!/entry. + apkPath.c_str(), + }; + + for (const char* candidate : candidates) + { + if (!pointLoaderAt(candidate)) + { + LOGW("could not write a runtime manifest into %s", mManifestDir.c_str()); + break; + } + LOGI("trying runtime library_path=%s", candidate); + if (runtimeAnswers()) + { + LOGI("runtime resolved via %s", candidate); + mRuntimeLibrary = candidate; + return true; + } + } + + unsetenv("XR_RUNTIME_JSON"); + setError( + "No OpenXR runtime answered. Tried the system default, then " + "libopenxr_forwardloader.so by name and by absolute APEX path."); + return false; +} + bool XrGoggleSession::initLoader(JNIEnv* env, jobject activity) { PFN_xrInitializeLoaderKHR initializeLoader = nullptr; diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index 9297ce19..282cd799 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -82,8 +82,14 @@ class XrGoggleSession // known stream resolution. void setSwapchainSize(int width, int height); + // Writable directory used to drop a runtime manifest, see resolveRuntime(). + void setManifestDir(std::string dir) { mManifestDir = std::move(dir); } + private: bool initLoader(JNIEnv* env, jobject activity); + bool resolveRuntime(); + bool runtimeAnswers(); + bool pointLoaderAt(const char* libraryPath); bool createInstance(JNIEnv* env, jobject activity); bool createEgl(); bool createSession(); @@ -178,6 +184,8 @@ class XrGoggleSession XrPosef mAnchorPose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; bool mAnchorValid = false; + std::string mManifestDir; + std::string mRuntimeLibrary; std::vector mRefreshRates; std::mutex mErrorMutex; std::string mLastError; diff --git a/app/xr/src/main/cpp/jni_bindings.cpp b/app/xr/src/main/cpp/jni_bindings.cpp index 07e02283..d647d5fe 100644 --- a/app/xr/src/main/cpp/jni_bindings.cpp +++ b/app/xr/src/main/cpp/jni_bindings.cpp @@ -32,6 +32,18 @@ JNI_METHOD(void, nativeSetSwapchainSize) native(handle)->setSwapchainSize(width, height); } +JNI_METHOD(void, nativeSetManifestDir)(JNIEnv* env, jclass clazz, jlong handle, jstring dir) +{ + (void) clazz; + if (dir == nullptr) return; + const char* chars = env->GetStringUTFChars(dir, nullptr); + if (chars != nullptr) + { + native(handle)->setManifestDir(std::string(chars)); + env->ReleaseStringUTFChars(dir, chars); + } +} + JNI_METHOD(jboolean, nativeCreate)(JNIEnv* env, jclass clazz, jlong handle, jobject activity) { (void) clazz; diff --git a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java index 9423e23c..39bc22ee 100644 --- a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java +++ b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java @@ -154,6 +154,8 @@ public void release() { private void ensureHandle() { if (libraryLoaded && handle == 0) { handle = nativeAlloc(); + // Somewhere writable for the runtime manifest the loader may need. + nativeSetManifestDir(handle, activity.getFilesDir().getAbsolutePath()); } } @@ -259,6 +261,8 @@ public boolean isPassthroughEnabled() { private static native void nativeSetSwapchainSize(long handle, int width, int height); + private static native void nativeSetManifestDir(long handle, String dir); + private static native boolean nativeCreate(long handle, Activity activity); private static native Surface nativeGetVideoSurface(long handle); From df54e7be7d544c7ac0c30aaf8f037197c612b423 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:40:49 +0200 Subject: [PATCH 06/23] Provision gs.key and the wfb options from the immersive activity too Two things VideoActivity.onCreate() does that XrVideoActivity did not, both found on a Quest 3: 1. gs.key. The native aggregators open files/gs.key in their constructor and throw if it is missing. VideoActivity copies the bundled key there before constructing WfbNgLink; the immersive activity did not, so a fresh install aborted the process on launch: libc++abi: terminating due to uncaught exception of type std::runtime_error: Unable to open /data/user/0/com.openipc.pixelpilot/files/gs.key Fatal signal 6 (SIGABRT) ... XrVideoActivity.onCreate 2. Adaptive link, TX power and FEC/LDPC/STBC. The native side does not read these itself, so without initDefaultOptions() they were silently ignored in immersive mode. Both moved into GsKey and WfbOptions and used from both activities, so there is one implementation rather than a copy that can drift. Also hardened, because a missing key should never be fatal: WfbngLink::initAgg() catches the aggregator constructor's exception, logs the key path and leaves the aggregators null instead of letting it escape a JNI call and abort the process. nativeRefreshKey() re-runs initAgg(), so fixing the key recovers without a restart. And XrGoggleSession now logs the runtime's full instance extension list, which is the only way to tell from a bug report what a given headset offers. --- .../java/com/openipc/pixelpilot/GsKey.java | 87 +++++++++++++++++++ .../com/openipc/pixelpilot/VideoActivity.java | 73 ++-------------- .../com/openipc/pixelpilot/WfbOptions.java | 46 ++++++++++ .../openipc/pixelpilot/XrVideoActivity.java | 8 ++ app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp | 14 +++ app/xr/src/main/cpp/XrGoggleSession.cpp | 7 ++ 6 files changed, 168 insertions(+), 67 deletions(-) create mode 100644 app/src/main/java/com/openipc/pixelpilot/GsKey.java create mode 100644 app/src/main/java/com/openipc/pixelpilot/WfbOptions.java diff --git a/app/src/main/java/com/openipc/pixelpilot/GsKey.java b/app/src/main/java/com/openipc/pixelpilot/GsKey.java new file mode 100644 index 00000000..1383fbbc --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/GsKey.java @@ -0,0 +1,87 @@ +package com.openipc.pixelpilot; + +import android.content.Context; +import android.content.SharedPreferences; +import android.util.Base64; +import android.util.Log; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * The wfb-ng ground station key, shared by the flat and the immersive activity. + * + *

The key lives in preferences (so a user-picked one survives) and has to be + * materialised as {@code files/gs.key} before {@code WfbNgLink} is constructed: the native + * aggregators open that path in their constructor and throw if it is missing. + */ +public final class GsKey { + private static final String TAG = "pixelpilot"; + private static final String PREF_KEY = "gs.key"; + private static final String FILE_NAME = "gs.key"; + + private GsKey() { + } + + public static byte[] get(Context context) { + String pref = context.getSharedPreferences("general", Context.MODE_PRIVATE) + .getString(PREF_KEY, ""); + return Base64.decode(pref, Base64.DEFAULT); + } + + public static void set(Context context, InputStream inputStream) throws IOException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + result.write(buffer, 0, length); + } + SharedPreferences prefs = context.getSharedPreferences("general", Context.MODE_PRIVATE); + prefs.edit().putString(PREF_KEY, Base64.encodeToString(result.toByteArray(), Base64.DEFAULT)).apply(); + } + + /** Seeds preferences from the bundled asset the first time round. */ + public static void importDefaultIfMissing(Context context) { + if (get(context).length > 0) { + Log.d(TAG, "gs.key already saved in preferences."); + return; + } + try (InputStream inputStream = context.getAssets().open(FILE_NAME)) { + Log.d(TAG, "Importing default gs.key..."); + set(context, inputStream); + } catch (IOException e) { + Log.e(TAG, "Failed to import default gs.key", e); + } + } + + /** Writes the key from preferences to {@code files/gs.key}. */ + public static boolean writeToFilesDir(Context context) { + byte[] keyBytes = get(context); + if (keyBytes.length == 0) { + Log.e(TAG, "No gs.key available to write"); + return false; + } + File file = new File(context.getApplicationContext().getFilesDir(), FILE_NAME); + try (OutputStream out = new FileOutputStream(file)) { + out.write(keyBytes, 0, keyBytes.length); + Log.d(TAG, "Wrote gs.key to " + file.getAbsolutePath()); + return true; + } catch (IOException e) { + Log.e(TAG, "Failed to write gs.key", e); + return false; + } + } + + /** + * Everything that has to happen before {@code new WfbNgLink(...)}. Call from any + * activity that brings up the link. + */ + public static boolean ensure(Context context) { + importDefaultIfMissing(context); + return writeToFilesDir(context); + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index 8ad0939f..aa652d0e 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -1021,35 +1021,12 @@ private void showFecThresholdsDialog() { } void initDefaultOptions() { - SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); - boolean adaptiveEnabled = prefs.getBoolean("adaptive_link_enabled", true); - int adaptiveTxPower = prefs.getInt("adaptive_tx_power", 20); - wfbLink.nativeSetAdaptiveLinkEnabled(adaptiveEnabled); - wfbLink.nativeSetTxPower(adaptiveTxPower); - boolean fecEnabled = prefs.getBoolean("custom_fec_enabled", true); - wfbLink.nativeSetUseFec(fecEnabled ? 1 : 0); - - // LDPC and STBC default options - boolean ldpcEnabled = prefs.getBoolean("custom_ldpc_enabled", true); - wfbLink.nativeSetUseLdpc(ldpcEnabled ? 1 : 0); - - boolean stbcEnabled = prefs.getBoolean("custom_stbc_enabled", true); - wfbLink.nativeSetUseStbc(stbcEnabled ? 1 : 0); - - setFecThresholdsFromPrefs(); + WfbOptions.applyDefaults(this, wfbLink); } // Read FEC thresholds from prefs and call native method to apply private void setFecThresholdsFromPrefs() { - SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); - int lostTo5 = prefs.getInt("fec_lost_to_5", 2); - int recTo4 = prefs.getInt("fec_recovered_to_4", 30); - int recTo3 = prefs.getInt("fec_recovered_to_3", 24); - int recTo2 = prefs.getInt("fec_recovered_to_2", 14); - int recTo1 = prefs.getInt("fec_recovered_to_1", 8); - if (wfbLink != null) { - wfbLink.setFecThresholds(lostTo5, recTo4, recTo3, recTo2, recTo1); - } + WfbOptions.applyFecThresholds(this, wfbLink); } /** @@ -1506,36 +1483,15 @@ protected void onActivityResult(int requestCode, int resultCode, @Nullable Inten } public void setDefaultGsKey() { - if (getGsKey().length > 0) { - Log.d(TAG, "gs.key already saved in preferences."); - return; - } - try { - Log.d(TAG, "Importing default gs.key..."); - InputStream inputStream = getAssets().open("gs.key"); - setGsKey(inputStream); - inputStream.close(); - } catch (IOException e) { - Log.e(TAG, "Failed to import default gs.key"); - } + GsKey.importDefaultIfMissing(this); } public byte[] getGsKey() { - String pref = getSharedPreferences("general", Context.MODE_PRIVATE).getString("gs.key", ""); - return Base64.decode(pref, Base64.DEFAULT); + return GsKey.get(this); } public void setGsKey(InputStream inputStream) throws IOException { - ByteArrayOutputStream result = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int length; - while ((length = inputStream.read(buffer)) != -1) { - result.write(buffer, 0, length); - } - SharedPreferences prefs = getSharedPreferences("general", Context.MODE_PRIVATE); - SharedPreferences.Editor editor = prefs.edit(); - editor.putString("gs.key", Base64.encodeToString(result.toByteArray(), Base64.DEFAULT)); - editor.apply(); + GsKey.set(this, inputStream); } public boolean getDvrMP4() { @@ -1802,24 +1758,7 @@ public void onNewMavlinkData(MavlinkData data) { } private void copyGSKey() { - File file = new File(getApplicationContext().getFilesDir(), "gs.key"); - OutputStream out = null; - try { - byte[] keyBytes = getGsKey(); - Log.d(TAG, "Using gs.key:" + bytesToHex(keyBytes) + "; Copying to" + file.getAbsolutePath()); - out = new FileOutputStream(file); - out.write(keyBytes, 0, keyBytes.length); - } catch (IOException e) { - Log.e(TAG, "Failed to copy asset", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - // NOOP - } - } - } + GsKey.writeToFilesDir(this); } private void showLoginCredentialsDialog() { diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbOptions.java b/app/src/main/java/com/openipc/pixelpilot/WfbOptions.java new file mode 100644 index 00000000..a495587e --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/WfbOptions.java @@ -0,0 +1,46 @@ +package com.openipc.pixelpilot; + +import android.content.Context; +import android.content.SharedPreferences; + +import com.openipc.wfbngrtl8812.WfbNgLink; + +/** + * Pushes the persisted wfb-ng options into the native link. + * + *

These are not read by the native side on its own, so any activity that constructs a + * {@link WfbNgLink} has to apply them explicitly or the user's adaptive link, TX power and + * FEC settings are silently ignored. + */ +public final class WfbOptions { + private WfbOptions() { + } + + public static void applyDefaults(Context context, WfbNgLink wfbLink) { + if (wfbLink == null) { + return; + } + SharedPreferences prefs = context.getSharedPreferences("general", Context.MODE_PRIVATE); + + wfbLink.nativeSetAdaptiveLinkEnabled(prefs.getBoolean("adaptive_link_enabled", true)); + wfbLink.nativeSetTxPower(prefs.getInt("adaptive_tx_power", 20)); + wfbLink.nativeSetUseFec(prefs.getBoolean("custom_fec_enabled", true) ? 1 : 0); + wfbLink.nativeSetUseLdpc(prefs.getBoolean("custom_ldpc_enabled", true) ? 1 : 0); + wfbLink.nativeSetUseStbc(prefs.getBoolean("custom_stbc_enabled", true) ? 1 : 0); + + applyFecThresholds(context, wfbLink); + } + + public static void applyFecThresholds(Context context, WfbNgLink wfbLink) { + if (wfbLink == null) { + return; + } + SharedPreferences prefs = context.getSharedPreferences("general", Context.MODE_PRIVATE); + wfbLink.setFecThresholds( + prefs.getInt("fec_lost_to_5", 2), + prefs.getInt("fec_recovered_to_4", 30), + prefs.getInt("fec_recovered_to_3", 24), + prefs.getInt("fec_recovered_to_2", 14), + prefs.getInt("fec_recovered_to_1", 8)); + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index 1e32fb88..d030acf9 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -117,8 +117,16 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { videoPlayer.setIVideoParamsChanged(this); videoPlayer.setLowLatency(VideoActivity.getLowLatencySetting(this)); + // The native aggregators open files/gs.key in their constructor and throw if it + // is not there, so this has to happen before WfbNgLink is created. + if (!GsKey.ensure(this)) { + failToFlat("no usable gs.key"); + return; + } wfbLink = new WfbNgLink(this); wfbLink.SetWfbNGStatsChanged(this); + // Not read by the native side on its own: adaptive link, TX power, FEC/LDPC/STBC. + WfbOptions.applyDefaults(this, wfbLink); wfbLinkManager = new WfbLinkManager(this, this, wfbLink); xr = new XrGoggleSession(this, this); diff --git a/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp b/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp index a3046b07..ae365310 100644 --- a/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp +++ b/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp @@ -56,6 +56,7 @@ WfbngLink::WfbngLink(JNIEnv *env, jobject context) } void WfbngLink::initAgg() { + try { std::string client_addr = "127.0.0.1"; uint64_t epoch = 0; @@ -81,6 +82,19 @@ void WfbngLink::initAgg() { udp_aggregator = std::make_unique(client_addr, udp_client_port, keyPath, epoch, udp_channel_id_f, 0); + } catch (const std::exception &e) { + // The aggregators open the key in their constructor. Letting that escape a JNI call + // aborts the whole process, which is what a missing gs.key used to do. + __android_log_print(ANDROID_LOG_ERROR, + TAG, + "initAgg failed (%s) - check gs.key at %s. Link stays down until " + "the key is refreshed.", + e.what(), + keyPath); + video_aggregator.reset(); + mavlink_aggregator.reset(); + udp_aggregator.reset(); + } } int WfbngLink::run(JNIEnv *env, jobject context, jint wifiChannel, jint bw, jint fd) { diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index c88fd1de..9979c025 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -309,6 +309,13 @@ bool XrGoggleSession::createInstance(JNIEnv* env, jobject activity) { mAvailableExtensions.emplace_back(p.extensionName); } + // Worth the log lines: it is the only way to see from a bug report what the runtime on + // a given headset actually offers. + LOGI("runtime exposes %zu instance extensions:", mAvailableExtensions.size()); + for (const auto& name : mAvailableExtensions) + { + LOGI(" %s", name.c_str()); + } // Without these three there is no immersive video path at all. const char* required[] = { From afa0499bebbe131a6f235f5004e519654e59bfcd Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:48:29 +0200 Subject: [PATCH 07/23] Fix the Android surface swapchain creation, and ask for passthrough Two findings from a Quest 3 (Horizon OS, Android 14). 1. xrCreateSwapchainAndroidSurfaceKHR returned XR_ERROR_VALIDATION_FAILURE. The extension documents format/sampleCount/faceCount/arraySize/mipCount as ignored for an Android surface swapchain, but Meta's runtime validates them, and the obvious 1s are rejected. Rather than swap one guess for another, createSwapchain() now tries the plausible shapes - ignored fields zeroed, set to 1, with and without COLOR_ATTACHMENT, with and without an explicit RGBA8 - and logs which one the runtime accepted. If all fail it dumps xrEnumerateSwapchainFormats so the next attempt has data instead of guesses. 2. passthrough was missing from the instance extension list on a Quest 3. Horizon OS only exposes XR_FB_passthrough to an app that declares com.oculus.feature.PASSTHROUGH in its manifest. Added as required="false", so nothing changes for a phone. Also corrects the reasoning in resolveRuntime(): the Khronos loader *does* resolve the Horizon runtime on its own, through the org.khronos.openxr.runtime_broker provider, and the "change nothing" candidate wins on a Quest. The earlier conclusion came from querying that provider with `adb shell content query`, which returns no rows only because the shell does not hold org.khronos.openxr.permission.OPENXR. The fallback is kept for a device that offers neither the broker nor active_runtime.json, but it is a fallback, not a workaround for the Quest. --- app/src/main/AndroidManifest.xml | 5 + app/xr/src/main/cpp/XrGoggleSession.cpp | 122 +++++++++++++++++------- 2 files changed, 92 insertions(+), 35 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 9f3c13b8..ea9f08c9 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -20,6 +20,11 @@ + + diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index 9979c025..464a2ec5 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -201,21 +201,19 @@ bool XrGoggleSession::pointLoaderAt(const char* libraryPath) } /* - * Horizon OS does not let the generic Khronos loader find its runtime on its own: + * Fallback for a headset where the loader cannot find a runtime by itself. * - * - there is no /system/etc/openxr/1/active_runtime.json - * - the Khronos broker authorities exist but resolve to - * internal.horizonos.openxr.DisabledRuntimeBroker, which returns no rows - * - the loader binary has no knowledge of libopenxr_forwardloader.so + * On a Quest 3 this is not needed: the Khronos loader resolves the Horizon runtime through + * the org.khronos.openxr.runtime_broker content provider and the first candidate ("change + * nothing") wins. Querying that provider from `adb shell` returns no rows, but only because + * the shell does not hold org.khronos.openxr.permission.OPENXR - the app does. * - * The runtime is reached through libopenxr_forwardloader.so from the VrDriver APEX. That - * library is on our linker search path (VrDriver.apk!/lib/ is added to every app's - * namespace) and exports xrNegotiateLoaderRuntimeInterface, i.e. as far as a loader is - * concerned it *is* the runtime. So if nothing answers out of the box, hand the loader an - * explicit manifest pointing at it. - * - * Candidates are tried in order and the one that answers is kept, so a device where the - * broker does work is left alone. + * It is kept because the loader has exactly three discovery paths - the broker, + * active_runtime.json on disk, and XR_RUNTIME_JSON - and a device that offers neither of the + * first two would otherwise be dead in the water. Where that happens the runtime is still + * reachable as libopenxr_forwardloader.so from the VrDriver APEX: that library sits on every + * app's linker search path and exports xrNegotiateLoaderRuntimeInterface, so as far as a + * loader is concerned it *is* the runtime. */ bool XrGoggleSession::resolveRuntime() { @@ -527,34 +525,88 @@ bool XrGoggleSession::createSpaces() "xrCreateReferenceSpace(LOCAL)"); } +/* + * MediaCodec is the producer for an Android surface swapchain, so the image description + * fields are not really ours to choose. The extension documents them as ignored, but Meta's + * runtime validates them: passing the "obvious" 1s gets XR_ERROR_VALIDATION_FAILURE on a + * Quest 3. Rather than hardcode one guess, try the plausible shapes and log which one the + * runtime accepted. + */ bool XrGoggleSession::createSwapchain(JNIEnv* env) { - XrSwapchainCreateInfo info{XR_TYPE_SWAPCHAIN_CREATE_INFO}; - // An Android surface swapchain is produced into by MediaCodec, so format and the - // image-layout fields are not ours to pick. - info.usageFlags = XR_SWAPCHAIN_USAGE_SAMPLED_BIT; - info.format = 0; - info.sampleCount = 1; - info.width = (uint32_t) mSwapchainWidth; - info.height = (uint32_t) mSwapchainHeight; - info.faceCount = 1; - info.arraySize = 1; - info.mipCount = 1; - - jobject surface = nullptr; - if (!check(mXrCreateSwapchainAndroidSurfaceKHR(mSession, &info, &mSwapchain, &surface), - "xrCreateSwapchainAndroidSurfaceKHR")) + // GL_RGBA8, for the variant where the runtime wants a real format. + constexpr int64_t kGlRgba8 = 0x8058; + + struct Variant { - return false; + const char* what; + XrSwapchainUsageFlags usage; + int64_t format; + uint32_t counts; // sampleCount / faceCount / arraySize / mipCount + }; + const Variant variants[] = { + {"ignored fields zeroed", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, 0, 0}, + {"ignored fields set to 1", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, 0, 1}, + {"zeroed + color attachment", + XR_SWAPCHAIN_USAGE_SAMPLED_BIT | XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT, + 0, + 0}, + {"zeroed + explicit RGBA8", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, kGlRgba8, 0}, + {"1s + explicit RGBA8", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, kGlRgba8, 1}, + }; + + jobject surface = nullptr; + XrResult last = XR_ERROR_VALIDATION_FAILURE; + for (const auto& variant : variants) + { + XrSwapchainCreateInfo info{XR_TYPE_SWAPCHAIN_CREATE_INFO}; + info.usageFlags = variant.usage; + info.format = variant.format; + info.sampleCount = variant.counts; + info.width = (uint32_t) mSwapchainWidth; + info.height = (uint32_t) mSwapchainHeight; + info.faceCount = variant.counts; + info.arraySize = variant.counts; + info.mipCount = variant.counts; + + surface = nullptr; + last = mXrCreateSwapchainAndroidSurfaceKHR(mSession, &info, &mSwapchain, &surface); + if (XR_SUCCEEDED(last) && surface != nullptr) + { + LOGI("android surface swapchain %dx%d created (%s)", + mSwapchainWidth, + mSwapchainHeight, + variant.what); + mVideoSurface = env->NewGlobalRef(surface); + return true; + } + char name[XR_MAX_RESULT_STRING_SIZE] = {0}; + xrResultToString(mInstance, last, name); + LOGW("swapchain variant \'%s\' rejected: %s", variant.what, name); + if (XR_SUCCEEDED(last) && mSwapchain != XR_NULL_HANDLE) + { + // Succeeded but handed back no Surface - not usable, do not leak it. + xrDestroySwapchain(mSwapchain); + mSwapchain = XR_NULL_HANDLE; + } } - if (surface == nullptr) + + // Nothing worked. Dump what the runtime does offer so the next attempt is informed. + uint32_t formatCount = 0; + if (XR_SUCCEEDED(xrEnumerateSwapchainFormats(mSession, 0, &formatCount, nullptr)) && + formatCount > 0) { - setError("xrCreateSwapchainAndroidSurfaceKHR returned a null Surface"); - return false; + std::vector formats(formatCount); + if (XR_SUCCEEDED( + xrEnumerateSwapchainFormats(mSession, formatCount, &formatCount, formats.data()))) + { + for (int64_t f : formats) + { + LOGI("runtime swapchain format 0x%llx", (unsigned long long) f); + } + } } - mVideoSurface = env->NewGlobalRef(surface); - LOGI("android surface swapchain %dx%d created", mSwapchainWidth, mSwapchainHeight); - return true; + return check(last, "xrCreateSwapchainAndroidSurfaceKHR"); } XrAction XrGoggleSession::boolAction(const char* name, const char* localized) From 7bcdb8213b80e67e808b20b3f94ee4678908de2d Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:53:25 +0200 Subject: [PATCH 08/23] Drive the in-flight actions from hand tracking as well Flying with hand tracking only and no controllers is a normal way to use a headset as a ground station, and the Quest 3 runtime offers XR_EXT_hand_interaction (confirmed in the instance extension list on Horizon OS 14, alongside XR_MSFT_hand_interaction and XR_META_hand_tracking_microgestures). The four boolean actions now also get bindings on /interaction_profiles/ext/hand_interaction_ext. A boolean action may be bound to a float source and the runtime does the thresholding, so no gesture code and no second input path is needed: right pinch -> recenter right grasp -> passthrough left pinch -> toggle DVR left grasp -> head lock Pinch is index-to-thumb and grasp is a whole-hand squeeze, far enough apart not to trigger each other. Controllers keep their A/B/X/Y and thumbstick bindings; whichever is active wins. Screen size and distance stay on the thumbsticks - there is no comfortable continuous hand equivalent. Also adds com.oculus.permission.HAND_TRACKING, which the handtracking feature needs before Horizon OS will actually hand the app tracked hands, and records the now-confirmed swapchain finding in the comment: Meta's runtime wants the "ignored" image description fields zeroed. --- app/src/main/AndroidManifest.xml | 3 ++ app/xr/src/main/cpp/XrGoggleSession.cpp | 44 ++++++++++++++++++++++--- app/xr/src/main/cpp/XrGoggleSession.h | 3 +- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index ea9f08c9..f742245f 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -37,6 +37,9 @@ + + GetJavaVM(&vm); @@ -528,9 +532,10 @@ bool XrGoggleSession::createSpaces() /* * MediaCodec is the producer for an Android surface swapchain, so the image description * fields are not really ours to choose. The extension documents them as ignored, but Meta's - * runtime validates them: passing the "obvious" 1s gets XR_ERROR_VALIDATION_FAILURE on a - * Quest 3. Rather than hardcode one guess, try the plausible shapes and log which one the - * runtime accepted. + * runtime validates them and wants them zeroed: passing the "obvious" 1s gets + * XR_ERROR_VALIDATION_FAILURE on a Quest 3 (Horizon OS, Android 14), while all-zero is + * accepted. The zeroed shape is therefore first; the rest stay as a fallback for a runtime + * that wants something else, and the accepted one is logged. */ bool XrGoggleSession::createSwapchain(JNIEnv* env) { @@ -699,6 +704,35 @@ bool XrGoggleSession::createActions() LOGW("simple controller bindings rejected"); } + // Hand tracking, for flying without controllers. A boolean action may be bound to a + // float source - the runtime does the thresholding - so the same four actions work. + // Pinch is index-to-thumb, grasp is a whole-hand squeeze, which are far enough apart + // not to trigger each other. + if (mHasHandInteraction) + { + const XrActionSuggestedBinding handBindings[] = { + {mActionRecenter, path("/user/hand/right/input/pinch_ext/value")}, + {mActionPassthrough, path("/user/hand/right/input/grasp_ext/value")}, + {mActionRecord, path("/user/hand/left/input/pinch_ext/value")}, + {mActionLockMode, path("/user/hand/left/input/grasp_ext/value")}, + }; + XrInteractionProfileSuggestedBinding hands{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING}; + hands.interactionProfile = path("/interaction_profiles/ext/hand_interaction_ext"); + hands.suggestedBindings = handBindings; + hands.countSuggestedBindings = (uint32_t) (sizeof(handBindings) / sizeof(handBindings[0])); + const XrResult r = xrSuggestInteractionProfileBindings(mInstance, &hands); + if (XR_FAILED(r)) + { + char name[XR_MAX_RESULT_STRING_SIZE] = {0}; + xrResultToString(mInstance, r, name); + LOGW("hand interaction bindings rejected: %s", name); + } + else + { + LOGI("hand interaction bindings suggested (pinch/grasp)"); + } + } + XrSessionActionSetsAttachInfo attach{XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO}; attach.countActionSets = 1; attach.actionSets = &mActionSet; diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index 282cd799..7bcc2b94 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -155,7 +155,8 @@ class XrGoggleSession PFN_xrRequestDisplayRefreshRateFB mXrRequestDisplayRefreshRateFB = nullptr; std::vector mAvailableExtensions; - bool mHasPassthrough = false; + bool mHasPassthrough = false; + bool mHasHandInteraction = false; bool mHasLayerSettings = false; bool mHasRefreshRate = false; From 098ca8ad6a235bd4e1ddc56b84ed51ea3c8dfcba Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:57:08 +0200 Subject: [PATCH 09/23] Show a status screen on the layer until video arrives With no stream the video quad is an opaque black rectangle, which from inside the headset is indistinguishable from "nothing is being composited at all" - exactly the situation on a first run with no adapter attached, and impossible to diagnose while wearing the headset. The swapchain's producer side is an ordinary Android Surface, so before the decoder takes it over the activity paints into it with lockHardwareCanvas(): a border and a centre cross, so the panel's real extent, centre, size and distance can be judged with nothing else on screen, plus the current link status, channel/bandwidth and the UDP address a stream can be pushed to. It redraws once a second and stops on the first decoded frame, since a buffer queue has a single producer and MediaCodec becomes it. If the canvas is refused it gives up quietly and leaves the layer to the decoder. --- .../openipc/pixelpilot/XrVideoActivity.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index d030acf9..6d65eb72 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -5,7 +5,9 @@ import android.content.Intent; import android.content.IntentFilter; import android.content.SharedPreferences; +import android.graphics.Canvas; import android.graphics.Color; +import android.graphics.Paint; import android.hardware.usb.UsbDevice; import android.hardware.usb.UsbManager; import android.net.Uri; @@ -75,6 +77,10 @@ public class XrVideoActivity extends AppCompatActivity private boolean xrStarting; private boolean linkStarted; + private Surface xrSurface; + private volatile String lastStatus = ""; + private boolean videoSeen; + private boolean statusScreenUsable = true; private ParcelFileDescriptor dvrFd; // Link quality haptics @@ -241,6 +247,11 @@ private boolean adaptersReady() { @Override public void onXrReady(Surface videoSurface) { Log.i(TAG, "immersive session up, handing the swapchain surface to the decoder"); + xrSurface = videoSurface; + // Put something on the layer before the decoder owns it. Without this the panel is + // an opaque black rectangle until the first frame arrives, which is + // indistinguishable from "nothing is being composited at all". + drawStatusScreen(); // The decoder writes into the compositor swapchain from here on. videoPlayer.addAndStartDecoderReceiver(videoSurface, 0); videoPlayer.start(); @@ -320,6 +331,88 @@ private void persistGeometry() { .apply(); } + // ------------------------------------------------------------------------------ + // "no signal" screen + // ------------------------------------------------------------------------------ + + /** + * Paints status straight into the compositor swapchain's Surface. + * + *

Only valid until MediaCodec becomes the producer for that Surface - a buffer queue + * has one producer - so this stops as soon as a frame has been decoded, and gives up + * quietly if the canvas is refused. + */ + private void drawStatusScreen() { + if (videoSeen || !statusScreenUsable || xrSurface == null || !xrSurface.isValid()) { + return; + } + Canvas canvas = null; + try { + canvas = xrSurface.lockHardwareCanvas(); + if (canvas == null) { + statusScreenUsable = false; + return; + } + paintStatus(canvas); + xrSurface.unlockCanvasAndPost(canvas); + } catch (Throwable t) { + // Most likely the decoder already connected as the producer. + Log.w(TAG, "status screen unavailable: " + t); + statusScreenUsable = false; + return; + } + handler.postDelayed(this::drawStatusScreen, 1000); + } + + private void paintStatus(Canvas canvas) { + final int w = canvas.getWidth(); + final int h = canvas.getHeight(); + + canvas.drawColor(Color.rgb(12, 12, 16)); + + Paint line = new Paint(); + line.setColor(Color.rgb(70, 80, 100)); + line.setStrokeWidth(Math.max(2f, h / 300f)); + line.setStyle(Paint.Style.STROKE); + // A border and a centre cross: makes the panel's real extent and centre visible, so + // size and distance can be judged with nothing else on screen. + float inset = h / 40f; + canvas.drawRect(inset, inset, w - inset, h - inset, line); + float cx = w / 2f, cy = h / 2f, arm = h / 20f; + canvas.drawLine(cx - arm, cy, cx + arm, cy, line); + canvas.drawLine(cx, cy - arm, cx, cy + arm, line); + + Paint title = new Paint(); + title.setAntiAlias(true); + title.setColor(Color.rgb(235, 235, 240)); + title.setTextSize(h / 14f); + title.setTextAlign(Paint.Align.CENTER); + canvas.drawText("No video", cx, cy - h / 8f, title); + + Paint body = new Paint(); + body.setAntiAlias(true); + body.setColor(Color.rgb(170, 180, 195)); + body.setTextSize(h / 28f); + body.setTextAlign(Paint.Align.CENTER); + + String status = lastStatus; + float y = cy + h / 7f; + if (!status.isEmpty()) { + canvas.drawText(status, cx, y, body); + y += h / 20f; + } + canvas.drawText("channel " + VideoActivity.getChannel(this) + + " @ " + VideoActivity.getBandwidth(this) + " MHz", + cx, y, body); + y += h / 20f; + String wifi = VideoActivity.wirelessInfo(this); + if (wifi != null) { + canvas.drawText("or push a stream to udp://" + wifi + ":5600", cx, y, body); + y += h / 20f; + } + canvas.drawText(xr != null && xr.isHeadLocked() ? "head locked" : "world locked", cx, y, body); + } + // ------------------------------------------------------------------------------ // DVR // ------------------------------------------------------------------------------ @@ -378,6 +471,7 @@ public void onVideoRatioChanged(int videoW, int videoH) { return; } Log.i(TAG, "stream resolution " + videoW + "x" + videoH); + videoSeen = true; if (xr != null) { xr.setVideoResolution(videoW, videoH); } @@ -416,6 +510,7 @@ public void onWfbNgStatsChanged(WfbNGStats data) { @Override public void showLinkMessage(String message) { Log.i(TAG, message); + lastStatus = message; runOnUiThread(() -> { if (statusView != null && !linkStarted) { statusView.setText(message); From 84b883d2658c0c378d1db6fd38c884c7928794cb Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:01:27 +0200 Subject: [PATCH 10/23] Fix: the layer was upside down Anything producing into an Android Surface - MediaCodec, and the Canvas used for the status screen - has its origin at the top left, while the compositor samples a swapchain image from the bottom left. Confirmed on a Quest 3: the layer composited correctly, in the right place and the right size, just vertically mirrored. XR_FB_composition_layer_image_layout exists for exactly this. The quad now carries XrCompositionLayerImageLayoutFB with VERTICAL_FLIP, chained together with the sharpening settings when both apply. Gated on the extension being present, like the others. --- app/xr/src/main/cpp/XrGoggleSession.cpp | 37 +++++++++++++++++++++---- app/xr/src/main/cpp/XrGoggleSession.h | 1 + 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index 7358ae03..e5909b0d 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -337,15 +337,21 @@ bool XrGoggleSession::createInstance(JNIEnv* env, jobject activity) mHasRefreshRate = extensionSupported(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME); // Lets the same actions be driven by a pinch when flying without controllers. mHasHandInteraction = extensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME); + // Needed to correct the origin of an Android-produced image, see renderFrame(). + mHasImageLayout = extensionSupported(XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME); if (mHasPassthrough) enabled.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME); if (mHasLayerSettings) enabled.push_back(XR_FB_COMPOSITION_LAYER_SETTINGS_EXTENSION_NAME); if (mHasRefreshRate) enabled.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME); if (mHasHandInteraction) enabled.push_back(XR_EXT_HAND_INTERACTION_EXTENSION_NAME); - LOGI("optional extensions: passthrough=%d layerSettings=%d refreshRate=%d handInteraction=%d", - (int) mHasPassthrough, - (int) mHasLayerSettings, - (int) mHasRefreshRate, - (int) mHasHandInteraction); + if (mHasImageLayout) enabled.push_back(XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME); + LOGI( + "optional extensions: passthrough=%d layerSettings=%d refreshRate=%d handInteraction=%d " + "imageLayout=%d", + (int) mHasPassthrough, + (int) mHasLayerSettings, + (int) mHasRefreshRate, + (int) mHasHandInteraction, + (int) mHasImageLayout); JavaVM* vm = nullptr; env->GetJavaVM(&vm); @@ -1172,8 +1178,27 @@ void XrGoggleSession::renderFrame(JNIEnv* env, jobject listener) XrCompositionLayerSettingsFB layerSettings{XR_TYPE_COMPOSITION_LAYER_SETTINGS_FB}; layerSettings.layerFlags = XR_COMPOSITION_LAYER_SETTINGS_QUALITY_SHARPENING_BIT_FB; + // Everything that produces into an Android Surface - MediaCodec, and Canvas for the + // status screen - has its origin at the top left, while the compositor samples a + // swapchain image from the bottom left. Without this the picture is upside down. + XrCompositionLayerImageLayoutFB imageLayout{XR_TYPE_COMPOSITION_LAYER_IMAGE_LAYOUT_FB}; + imageLayout.flags = XR_COMPOSITION_LAYER_IMAGE_LAYOUT_VERTICAL_FLIP_BIT_FB; + + // Chain whatever applies onto the quad. + const void* layerChain = nullptr; + if (mHasLayerSettings && mSharpening.load()) + { + layerSettings.next = layerChain; + layerChain = &layerSettings; + } + if (mHasImageLayout) + { + imageLayout.next = const_cast(layerChain); + layerChain = &imageLayout; + } + XrCompositionLayerQuad quad{XR_TYPE_COMPOSITION_LAYER_QUAD}; - quad.next = (mHasLayerSettings && mSharpening.load()) ? &layerSettings : nullptr; + quad.next = layerChain; quad.layerFlags = 0; // opaque: the video has to cover passthrough behind it quad.space = headLocked ? mViewSpace : mLocalSpace; quad.eyeVisibility = XR_EYE_VISIBILITY_BOTH; diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index 7bcc2b94..e4cf12df 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -157,6 +157,7 @@ class XrGoggleSession std::vector mAvailableExtensions; bool mHasPassthrough = false; bool mHasHandInteraction = false; + bool mHasImageLayout = false; bool mHasLayerSettings = false; bool mHasRefreshRate = false; From 1f76b13f4bc7f3b1861b1b55f6c792fd881d1a03 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:15:56 +0200 Subject: [PATCH 11/23] Let the panel be nudged up and down With a head-locked panel its height relative to your natural line of sight matters - too high or too low and you fly with your neck craned - and there was no way to change it. Pinch and grasp are already spent on recenter, passthrough, DVR and the head lock toggle, so the nudge goes on XR_META_hand_tracking_microgestures instead: a thumb swipe along the index finger, which is still an index-finger/thumb control but costs nothing that was already bound. right thumb swipe forward -> panel up right thumb swipe backward -> panel down That extension defines no structs or entry points, only binding paths, so it is suggested as part of the hand profile. Suggestions are atomic per profile, so a rejected microgesture path would take the working pinch/grasp bindings with it; if the call fails the microgesture entries are dropped and it is retried, and which variant took is logged. The controller mapping is reorganised so the same adjustment exists there and is intuitive - the left stick now moves the panel, the right stick sizes it: left stick up/down -> panel up/down (was: distance) left stick left/right -> nearer/farther right stick up/down -> panel size (unchanged) The offset is applied to the quad pose in both head-locked and world-locked mode and persisted as xr_quad_height. --- .../openipc/pixelpilot/XrVideoActivity.java | 7 ++ app/xr/src/main/cpp/XrGoggleSession.cpp | 80 ++++++++++++++++--- app/xr/src/main/cpp/XrGoggleSession.h | 8 ++ app/xr/src/main/cpp/jni_bindings.cpp | 14 ++++ .../java/com/openipc/xr/XrGoggleSession.java | 14 ++++ 5 files changed, 113 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index 6d65eb72..d80f8611 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -64,6 +64,7 @@ public class XrVideoActivity extends AppCompatActivity static final String PREF_REFRESH_HZ = "xr_refresh_hz"; static final String PREF_QUAD_WIDTH = "xr_quad_width"; static final String PREF_QUAD_DISTANCE = "xr_quad_distance"; + static final String PREF_QUAD_HEIGHT = "xr_quad_height"; static final String PREF_LAST_VIDEO_W = "xr_last_video_width"; static final String PREF_LAST_VIDEO_H = "xr_last_video_height"; @@ -145,6 +146,7 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { xr.setSharpening(prefs.getBoolean(PREF_SHARPENING, true)); xr.setQuadWidth(prefs.getFloat(PREF_QUAD_WIDTH, 2.2f)); xr.setQuadDistance(prefs.getFloat(PREF_QUAD_DISTANCE, 1.6f)); + xr.setQuadHeightOffset(prefs.getFloat(PREF_QUAD_HEIGHT, 0f)); } @Override @@ -291,6 +293,10 @@ public void onXrButton(int button) { case XrGoggleSession.BUTTON_RECORD: toggleDvr(); break; + case XrGoggleSession.BUTTON_RAISE: + case XrGoggleSession.BUTTON_LOWER: + prefs().edit().putFloat(PREF_QUAD_HEIGHT, xr.quadHeightOffset()).apply(); + break; default: break; } @@ -326,6 +332,7 @@ private void persistGeometry() { prefs().edit() .putFloat(PREF_QUAD_WIDTH, xr.quadWidth()) .putFloat(PREF_QUAD_DISTANCE, xr.quadDistance()) + .putFloat(PREF_QUAD_HEIGHT, xr.quadHeightOffset()) .putBoolean(PREF_HEAD_LOCKED, xr.isHeadLocked()) .putBoolean(PREF_PASSTHROUGH, xr.isPassthroughEnabled()) .apply(); diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index e5909b0d..0eec945d 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -23,6 +23,10 @@ constexpr float kMinDistance = 0.4f; constexpr float kMaxDistance = 8.0f; constexpr float kMinWidth = 0.4f; constexpr float kMaxWidth = 12.0f; +constexpr float kMaxHeightOffset = 1.5f; +// One nudge, about 1.8 degrees at the default distance - small enough to fine-tune. +constexpr float kHeightStep = 0.05f; +constexpr float kHeightPerSec = 0.7f; float clampf(float v, float lo, float hi) { @@ -339,19 +343,25 @@ bool XrGoggleSession::createInstance(JNIEnv* env, jobject activity) mHasHandInteraction = extensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME); // Needed to correct the origin of an Android-produced image, see renderFrame(). mHasImageLayout = extensionSupported(XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME); + // Thumb swipes along the index finger: a discrete control that does not occupy the + // hand, so pinch and grasp stay free for the flight actions. + mHasMicrogestures = mHasHandInteraction && + extensionSupported(XR_META_HAND_TRACKING_MICROGESTURES_EXTENSION_NAME); if (mHasPassthrough) enabled.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME); if (mHasLayerSettings) enabled.push_back(XR_FB_COMPOSITION_LAYER_SETTINGS_EXTENSION_NAME); if (mHasRefreshRate) enabled.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME); if (mHasHandInteraction) enabled.push_back(XR_EXT_HAND_INTERACTION_EXTENSION_NAME); if (mHasImageLayout) enabled.push_back(XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME); + if (mHasMicrogestures) enabled.push_back(XR_META_HAND_TRACKING_MICROGESTURES_EXTENSION_NAME); LOGI( "optional extensions: passthrough=%d layerSettings=%d refreshRate=%d handInteraction=%d " - "imageLayout=%d", + "imageLayout=%d microgestures=%d", (int) mHasPassthrough, (int) mHasLayerSettings, (int) mHasRefreshRate, (int) mHasHandInteraction, - (int) mHasImageLayout); + (int) mHasImageLayout, + (int) mHasMicrogestures); JavaVM* vm = nullptr; env->GetJavaVM(&vm); @@ -649,6 +659,8 @@ bool XrGoggleSession::createActions() mActionPassthrough = boolAction("passthrough", "Toggle passthrough"); mActionRecord = boolAction("record", "Toggle recording"); mActionLockMode = boolAction("lock_mode", "Toggle head lock"); + mActionRaise = boolAction("raise", "Move the screen up"); + mActionLower = boolAction("lower", "Move the screen down"); XrActionCreateInfo stick{XR_TYPE_ACTION_CREATE_INFO}; stick.actionType = XR_ACTION_TYPE_VECTOR2F_INPUT; @@ -674,7 +686,8 @@ bool XrGoggleSession::createActions() return false; } - // Touch controller: A/B live on the right hand, X/Y on the left. + // Touch controller: A/B live on the right hand, X/Y on the left. The left stick moves + // the panel (up/down and nearer/farther), the right stick sizes it. const XrActionSuggestedBinding touchBindings[] = { {mActionRecenter, path("/user/hand/right/input/a/click")}, {mActionPassthrough, path("/user/hand/right/input/b/click")}, @@ -716,17 +729,39 @@ bool XrGoggleSession::createActions() // not to trigger each other. if (mHasHandInteraction) { - const XrActionSuggestedBinding handBindings[] = { + std::vector handBindings = { {mActionRecenter, path("/user/hand/right/input/pinch_ext/value")}, {mActionPassthrough, path("/user/hand/right/input/grasp_ext/value")}, {mActionRecord, path("/user/hand/left/input/pinch_ext/value")}, {mActionLockMode, path("/user/hand/left/input/grasp_ext/value")}, }; + // Thumb swipes along the index finger, for nudging the panel up and down without + // spending pinch or grasp on it. + const size_t withoutMicrogestures = handBindings.size(); + if (mHasMicrogestures) + { + handBindings.push_back( + {mActionRaise, path("/user/hand/right/input/swipe_forward_meta")}); + handBindings.push_back( + {mActionLower, path("/user/hand/right/input/swipe_backward_meta")}); + } + XrInteractionProfileSuggestedBinding hands{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING}; hands.interactionProfile = path("/interaction_profiles/ext/hand_interaction_ext"); - hands.suggestedBindings = handBindings; - hands.countSuggestedBindings = (uint32_t) (sizeof(handBindings) / sizeof(handBindings[0])); - const XrResult r = xrSuggestInteractionProfileBindings(mInstance, &hands); + hands.suggestedBindings = handBindings.data(); + hands.countSuggestedBindings = (uint32_t) handBindings.size(); + XrResult r = xrSuggestInteractionProfileBindings(mInstance, &hands); + if (XR_FAILED(r) && handBindings.size() != withoutMicrogestures) + { + // Suggestions are atomic per profile, so one bad path would cost the working + // pinch/grasp bindings too. Drop the microgestures and keep those. + char name[XR_MAX_RESULT_STRING_SIZE] = {0}; + xrResultToString(mInstance, r, name); + LOGW("microgesture bindings rejected (%s), retrying without them", name); + mHasMicrogestures = false; + hands.countSuggestedBindings = (uint32_t) withoutMicrogestures; + r = xrSuggestInteractionProfileBindings(mInstance, &hands); + } if (XR_FAILED(r)) { char name[XR_MAX_RESULT_STRING_SIZE] = {0}; @@ -735,7 +770,8 @@ bool XrGoggleSession::createActions() } else { - LOGI("hand interaction bindings suggested (pinch/grasp)"); + LOGI("hand interaction bindings suggested (pinch/grasp%s)", + mHasMicrogestures ? " + microgesture height" : ""); } } @@ -808,6 +844,11 @@ void XrGoggleSession::setQuadWidth(float meters) mQuadWidth = clampf(meters, kMinWidth, kMaxWidth); } +void XrGoggleSession::setQuadHeightOffset(float meters) +{ + mQuadHeightOffset = clampf(meters, -kMaxHeightOffset, kMaxHeightOffset); +} + void XrGoggleSession::setPassthrough(bool enabled) { mPassthroughWanted = enabled; @@ -1054,6 +1095,8 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) {mActionPassthrough, BUTTON_PASSTHROUGH}, {mActionRecord, BUTTON_RECORD}, {mActionLockMode, BUTTON_LOCK_MODE}, + {mActionRaise, BUTTON_RAISE}, + {mActionLower, BUTTON_LOWER}, }; for (const auto& b : buttons) @@ -1080,6 +1123,14 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) setHeadLocked(!mHeadLocked.load()); requestHaptic(0.4f, 30); break; + case BUTTON_RAISE: + setQuadHeightOffset(mQuadHeightOffset.load() + kHeightStep); + requestHaptic(0.25f, 20); + break; + case BUTTON_LOWER: + setQuadHeightOffset(mQuadHeightOffset.load() - kHeightStep); + requestHaptic(0.25f, 20); + break; default: break; } @@ -1114,10 +1165,16 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) stickState = XrActionStateVector2f{XR_TYPE_ACTION_STATE_VECTOR2F}; if (XR_SUCCEEDED(xrGetActionStateVector2f(mSession, &get, &stickState)) && stickState.isActive) { + // Left stick moves the panel: up/down raises it, left/right pulls it in and out. const float y = stickState.currentState.y; if (std::fabs(y) > kStickDeadzone) { - setQuadDistance(mQuadDistance.load() - y * kDistancePerSec * dt); + setQuadHeightOffset(mQuadHeightOffset.load() + y * kHeightPerSec * dt); + } + const float x = stickState.currentState.x; + if (std::fabs(x) > kStickDeadzone) + { + setQuadDistance(mQuadDistance.load() + x * kDistancePerSec * dt); } } } @@ -1210,13 +1267,16 @@ void XrGoggleSession::renderFrame(JNIEnv* env, jobject listener) quad.subImage.imageRect.extent = XrExtent2Di{vw > 0 && vw <= mSwapchainWidth ? vw : mSwapchainWidth, vh > 0 && vh <= mSwapchainHeight ? vh : mSwapchainHeight}; + const float heightOffset = mQuadHeightOffset.load(); if (headLocked) { - quad.pose = XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, -mQuadDistance.load()}}; + quad.pose = + XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, heightOffset, -mQuadDistance.load()}}; } else { quad.pose = mAnchorPose; + quad.pose.position.y += heightOffset; } quad.size = XrExtent2Df{widthM, widthM * aspect}; diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index e4cf12df..14d4cf1c 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -34,6 +34,8 @@ class XrGoggleSession BUTTON_PASSTHROUGH = 1, BUTTON_RECORD = 2, BUTTON_LOCK_MODE = 3, + BUTTON_RAISE = 4, + BUTTON_LOWER = 5, }; XrGoggleSession() = default; @@ -65,6 +67,7 @@ class XrGoggleSession void setVideoResolution(int width, int height); void setQuadDistance(float meters); void setQuadWidth(float meters); + void setQuadHeightOffset(float meters); void setPassthrough(bool enabled); void setSharpening(bool enabled); void setHeadLocked(bool enabled); @@ -75,6 +78,7 @@ class XrGoggleSession float quadDistance() const { return mQuadDistance; } float quadWidth() const { return mQuadWidth; } + float quadHeightOffset() const { return mQuadHeightOffset; } bool headLocked() const { return mHeadLocked; } bool passthroughEnabled() const { return mPassthroughWanted; } @@ -132,6 +136,8 @@ class XrGoggleSession XrAction mActionLockMode = XR_NULL_HANDLE; XrAction mActionSizeStick = XR_NULL_HANDLE; XrAction mActionPlaceStick = XR_NULL_HANDLE; + XrAction mActionRaise = XR_NULL_HANDLE; + XrAction mActionLower = XR_NULL_HANDLE; XrAction mActionHaptic = XR_NULL_HANDLE; // --- EGL ---------------------------------------------------------------------- @@ -158,6 +164,7 @@ class XrGoggleSession bool mHasPassthrough = false; bool mHasHandInteraction = false; bool mHasImageLayout = false; + bool mHasMicrogestures = false; bool mHasLayerSettings = false; bool mHasRefreshRate = false; @@ -173,6 +180,7 @@ class XrGoggleSession int mSwapchainHeight = 1080; std::atomic mQuadDistance{1.6f}; std::atomic mQuadWidth{2.2f}; + std::atomic mQuadHeightOffset{0.0f}; std::atomic mHeadLocked{true}; std::atomic mPassthroughWanted{false}; std::atomic mSharpening{true}; diff --git a/app/xr/src/main/cpp/jni_bindings.cpp b/app/xr/src/main/cpp/jni_bindings.cpp index d647d5fe..12e68b76 100644 --- a/app/xr/src/main/cpp/jni_bindings.cpp +++ b/app/xr/src/main/cpp/jni_bindings.cpp @@ -119,6 +119,20 @@ JNI_METHOD(void, nativeSetQuadWidth)(JNIEnv* env, jclass clazz, jlong handle, jf native(handle)->setQuadWidth(meters); } +JNI_METHOD(void, nativeSetQuadHeightOffset)(JNIEnv* env, jclass clazz, jlong handle, jfloat meters) +{ + (void) env; + (void) clazz; + native(handle)->setQuadHeightOffset(meters); +} + +JNI_METHOD(jfloat, nativeGetQuadHeightOffset)(JNIEnv* env, jclass clazz, jlong handle) +{ + (void) env; + (void) clazz; + return native(handle)->quadHeightOffset(); +} + JNI_METHOD(void, nativeSetPassthrough)(JNIEnv* env, jclass clazz, jlong handle, jboolean enabled) { (void) env; diff --git a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java index 39bc22ee..6d8ddf85 100644 --- a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java +++ b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java @@ -29,6 +29,8 @@ public final class XrGoggleSession { public static final int BUTTON_PASSTHROUGH = 1; public static final int BUTTON_RECORD = 2; public static final int BUTTON_LOCK_MODE = 3; + public static final int BUTTON_RAISE = 4; + public static final int BUTTON_LOWER = 5; private static final String TAG = "pixelpilot-xr"; @@ -208,6 +210,14 @@ public void setQuadWidth(float meters) { if (handle != 0) nativeSetQuadWidth(handle, meters); } + public void setQuadHeightOffset(float meters) { + if (handle != 0) nativeSetQuadHeightOffset(handle, meters); + } + + public float quadHeightOffset() { + return handle != 0 ? nativeGetQuadHeightOffset(handle) : 0f; + } + public void setPassthrough(boolean enabled) { if (handle != 0) nativeSetPassthrough(handle, enabled); } @@ -283,6 +293,10 @@ public boolean isPassthroughEnabled() { private static native void nativeSetQuadWidth(long handle, float meters); + private static native void nativeSetQuadHeightOffset(long handle, float meters); + + private static native float nativeGetQuadHeightOffset(long handle); + private static native void nativeSetPassthrough(long handle, boolean enabled); private static native void nativeSetSharpening(long handle, boolean enabled); From c739935b72dff0cfce521b7a0eefcabef161d0ab Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:38:38 +0200 Subject: [PATCH 12/23] Mirror the controller mapping and put distance on trigger/grip Reported from the headset: the hand gestures work but the right thumbstick does nothing, and the trigger/grip do not move the panel. The likely cause of the dead stick is that only one interaction profile is live at a time. With hands tracked the runtime picks the hand profile and the touch controller bindings never fire, which looks exactly like a broken stick. The runtime offers XR_META_simultaneous_hands_and_controllers, so that is enabled and xrResumeSimultaneousHandsAndControllersTrackingMETA is called once the session starts. To make this visible rather than guessed at, the active interaction profile per hand is now logged on XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED and at session start. The mapping is also mirrored so a single controller in either hand is enough, and the trigger/grip take over distance: thumbstick up/down screen up/down thumbstick left/right screen smaller/bigger trigger (analog) pull the screen closer grip (analog) push the screen away A / X (lower button) recenter B / Y (upper button) toggle passthrough thumbstick click toggle recording A/B and X/Y sit in the same places on their respective controllers, so binding both to one action gives the same thumb positions whichever controller was picked up. Trigger and grip are float actions, so the movement is proportional to how far they are pulled. Head lock loses its controller binding - there is no button left that exists on both controllers - and stays on the left-hand grasp gesture and in the flat settings menu, which is where a set-once option belongs. --- app/xr/src/main/cpp/XrGoggleSession.cpp | 158 +++++++++++++++++++----- app/xr/src/main/cpp/XrGoggleSession.h | 8 +- 2 files changed, 131 insertions(+), 35 deletions(-) diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index 0eec945d..006e1a71 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -17,6 +17,7 @@ namespace { constexpr float kStickDeadzone = 0.30f; +constexpr float kTriggerDeadzone = 0.15f; constexpr float kDistancePerSec = 0.9f; // meters per second of full stick deflection constexpr float kWidthPerSec = 1.4f; constexpr float kMinDistance = 0.4f; @@ -347,21 +348,28 @@ bool XrGoggleSession::createInstance(JNIEnv* env, jobject activity) // hand, so pinch and grasp stay free for the flight actions. mHasMicrogestures = mHasHandInteraction && extensionSupported(XR_META_HAND_TRACKING_MICROGESTURES_EXTENSION_NAME); + // Without this only one interaction profile is live at a time: with hands tracked the + // controller bindings never fire, which looks exactly like "the stick is broken". + mHasSimultaneous = + extensionSupported(XR_META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME); if (mHasPassthrough) enabled.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME); if (mHasLayerSettings) enabled.push_back(XR_FB_COMPOSITION_LAYER_SETTINGS_EXTENSION_NAME); if (mHasRefreshRate) enabled.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME); if (mHasHandInteraction) enabled.push_back(XR_EXT_HAND_INTERACTION_EXTENSION_NAME); if (mHasImageLayout) enabled.push_back(XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME); if (mHasMicrogestures) enabled.push_back(XR_META_HAND_TRACKING_MICROGESTURES_EXTENSION_NAME); + if (mHasSimultaneous) + enabled.push_back(XR_META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME); LOGI( "optional extensions: passthrough=%d layerSettings=%d refreshRate=%d handInteraction=%d " - "imageLayout=%d microgestures=%d", + "imageLayout=%d microgestures=%d simultaneousHandsControllers=%d", (int) mHasPassthrough, (int) mHasLayerSettings, (int) mHasRefreshRate, (int) mHasHandInteraction, (int) mHasImageLayout, - (int) mHasMicrogestures); + (int) mHasMicrogestures, + (int) mHasSimultaneous); JavaVM* vm = nullptr; env->GetJavaVM(&vm); @@ -423,6 +431,13 @@ bool XrGoggleSession::createInstance(JNIEnv* env, jobject activity) mHasPassthrough = mXrCreatePassthroughFB != nullptr && mXrCreatePassthroughLayerFB != nullptr && mXrPassthroughStartFB != nullptr; } + if (mHasSimultaneous) + { + xrGetInstanceProcAddr(mInstance, + "xrResumeSimultaneousHandsAndControllersTrackingMETA", + (PFN_xrVoidFunction*) &mXrResumeSimultaneous); + mHasSimultaneous = mXrResumeSimultaneous != nullptr; + } if (mHasRefreshRate) { xrGetInstanceProcAddr(mInstance, @@ -664,15 +679,25 @@ bool XrGoggleSession::createActions() XrActionCreateInfo stick{XR_TYPE_ACTION_CREATE_INFO}; stick.actionType = XR_ACTION_TYPE_VECTOR2F_INPUT; - std::strncpy(stick.actionName, "size", XR_MAX_ACTION_NAME_SIZE - 1); - std::strncpy(stick.localizedActionName, "Screen size", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); - if (!check(xrCreateAction(mActionSet, &stick, &mActionSizeStick), "xrCreateAction(size)")) + std::strncpy(stick.actionName, "place", XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(stick.localizedActionName, "Move the screen", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + if (!check(xrCreateAction(mActionSet, &stick, &mActionStick), "xrCreateAction(place)")) { return false; } - std::strncpy(stick.actionName, "place", XR_MAX_ACTION_NAME_SIZE - 1); - std::strncpy(stick.localizedActionName, "Screen distance", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); - if (!check(xrCreateAction(mActionSet, &stick, &mActionPlaceStick), "xrCreateAction(place)")) + + // Analog, so the trigger and grip pull the panel in and push it out at a rate. + XrActionCreateInfo axis{XR_TYPE_ACTION_CREATE_INFO}; + axis.actionType = XR_ACTION_TYPE_FLOAT_INPUT; + std::strncpy(axis.actionName, "nearer", XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(axis.localizedActionName, "Pull the screen closer", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + if (!check(xrCreateAction(mActionSet, &axis, &mActionNearer), "xrCreateAction(nearer)")) + { + return false; + } + std::strncpy(axis.actionName, "farther", XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(axis.localizedActionName, "Push the screen away", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + if (!check(xrCreateAction(mActionSet, &axis, &mActionFarther), "xrCreateAction(farther)")) { return false; } @@ -686,15 +711,33 @@ bool XrGoggleSession::createActions() return false; } - // Touch controller: A/B live on the right hand, X/Y on the left. The left stick moves - // the panel (up/down and nearer/farther), the right stick sizes it. + // Mirrored left and right, so a single controller in either hand is enough. A/B and + // X/Y sit in the same places on their respective controllers, so binding both to the + // same action gives the same thumb positions whichever one you picked up. + // + // thumbstick up/down screen up/down + // thumbstick left/right screen smaller/bigger + // trigger pull the screen closer + // grip push the screen away + // A / X (lower button) recenter + // B / Y (upper button) toggle passthrough + // thumbstick click toggle recording + // + // Head lock stays on a hand gesture and in the flat settings menu - it is set once, + // not adjusted in flight, and there is no button left that exists on both controllers. const XrActionSuggestedBinding touchBindings[] = { {mActionRecenter, path("/user/hand/right/input/a/click")}, + {mActionRecenter, path("/user/hand/left/input/x/click")}, {mActionPassthrough, path("/user/hand/right/input/b/click")}, - {mActionRecord, path("/user/hand/left/input/x/click")}, - {mActionLockMode, path("/user/hand/left/input/y/click")}, - {mActionSizeStick, path("/user/hand/right/input/thumbstick")}, - {mActionPlaceStick, path("/user/hand/left/input/thumbstick")}, + {mActionPassthrough, path("/user/hand/left/input/y/click")}, + {mActionRecord, path("/user/hand/right/input/thumbstick/click")}, + {mActionRecord, path("/user/hand/left/input/thumbstick/click")}, + {mActionStick, path("/user/hand/right/input/thumbstick")}, + {mActionStick, path("/user/hand/left/input/thumbstick")}, + {mActionNearer, path("/user/hand/right/input/trigger/value")}, + {mActionNearer, path("/user/hand/left/input/trigger/value")}, + {mActionFarther, path("/user/hand/right/input/squeeze/value")}, + {mActionFarther, path("/user/hand/left/input/squeeze/value")}, {mActionHaptic, path("/user/hand/left/output/haptic")}, {mActionHaptic, path("/user/hand/right/output/haptic")}, }; @@ -950,6 +993,20 @@ void XrGoggleSession::pollEvents(JNIEnv* env, jobject listener, bool* exitLoop) if (check(xrBeginSession(mSession, &begin), "xrBeginSession")) { mSessionRunning = true; + if (mHasSimultaneous) + { + XrSimultaneousHandsAndControllersTrackingResumeInfoMETA resume{ + XR_TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META}; + if (XR_SUCCEEDED(mXrResumeSimultaneous(mSession, &resume))) + { + LOGI("hands and controllers tracked simultaneously"); + } + else + { + LOGW("xrResumeSimultaneousHandsAndControllersTrackingMETA failed"); + } + } + logInteractionProfiles(); } } else if (mSessionState == XR_SESSION_STATE_STOPPING) @@ -972,6 +1029,12 @@ void XrGoggleSession::pollEvents(JNIEnv* env, jobject listener, bool* exitLoop) *exitLoop = true; break; } + case XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED: + { + // The only way to see which of the suggested profiles is actually live. + logInteractionProfiles(); + break; + } case XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING: { // The runtime recentered underneath us; re-anchor on the next frame. @@ -984,6 +1047,30 @@ void XrGoggleSession::pollEvents(JNIEnv* env, jobject listener, bool* exitLoop) } } +void XrGoggleSession::logInteractionProfiles() +{ + for (const char* hand : {"/user/hand/left", "/user/hand/right"}) + { + XrInteractionProfileState state{XR_TYPE_INTERACTION_PROFILE_STATE}; + if (XR_FAILED(xrGetCurrentInteractionProfile(mSession, path(hand), &state))) + { + continue; + } + if (state.interactionProfile == XR_NULL_PATH) + { + LOGI("%s: no interaction profile active", hand); + continue; + } + char name[XR_MAX_PATH_LENGTH] = {0}; + uint32_t written = 0; + if (XR_SUCCEEDED( + xrPathToString(mInstance, state.interactionProfile, sizeof(name), &written, name))) + { + LOGI("%s: %s", hand, name); + } + } +} + void XrGoggleSession::applyPendingRefreshRate() { if (!mHasRefreshRate) return; @@ -1145,27 +1232,13 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) const float dt = 1.0f / 72.0f; // good enough for a manual adjustment ramp XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO}; - XrActionStateVector2f stickState{XR_TYPE_ACTION_STATE_VECTOR2F}; - if (mActionSizeStick != XR_NULL_HANDLE) + if (mActionStick != XR_NULL_HANDLE) { - get.action = mActionSizeStick; - stickState = XrActionStateVector2f{XR_TYPE_ACTION_STATE_VECTOR2F}; + get.action = mActionStick; + XrActionStateVector2f stickState{XR_TYPE_ACTION_STATE_VECTOR2F}; if (XR_SUCCEEDED(xrGetActionStateVector2f(mSession, &get, &stickState)) && stickState.isActive) { - const float y = stickState.currentState.y; - if (std::fabs(y) > kStickDeadzone) - { - setQuadWidth(mQuadWidth.load() + y * kWidthPerSec * dt); - } - } - } - if (mActionPlaceStick != XR_NULL_HANDLE) - { - get.action = mActionPlaceStick; - stickState = XrActionStateVector2f{XR_TYPE_ACTION_STATE_VECTOR2F}; - if (XR_SUCCEEDED(xrGetActionStateVector2f(mSession, &get, &stickState)) && stickState.isActive) - { - // Left stick moves the panel: up/down raises it, left/right pulls it in and out. + // Either thumbstick: up/down raises the panel, left/right resizes it. const float y = stickState.currentState.y; if (std::fabs(y) > kStickDeadzone) { @@ -1174,10 +1247,29 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) const float x = stickState.currentState.x; if (std::fabs(x) > kStickDeadzone) { - setQuadDistance(mQuadDistance.load() + x * kDistancePerSec * dt); + setQuadWidth(mQuadWidth.load() + x * kWidthPerSec * dt); } } } + + // Trigger pulls the panel in, grip pushes it out, both analog. + float distanceRate = 0.0f; + for (int i = 0; i < 2; ++i) + { + const XrAction action = (i == 0) ? mActionNearer : mActionFarther; + if (action == XR_NULL_HANDLE) continue; + get.action = action; + XrActionStateFloat axisState{XR_TYPE_ACTION_STATE_FLOAT}; + if (XR_SUCCEEDED(xrGetActionStateFloat(mSession, &get, &axisState)) && axisState.isActive && + axisState.currentState > kTriggerDeadzone) + { + distanceRate += (i == 0 ? -1.0f : 1.0f) * axisState.currentState; + } + } + if (std::fabs(distanceRate) > 0.0f) + { + setQuadDistance(mQuadDistance.load() + distanceRate * kDistancePerSec * dt); + } } void XrGoggleSession::recenterAt(XrTime time) diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index 14d4cf1c..e0dd89c2 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -107,6 +107,7 @@ class XrGoggleSession void syncActions(JNIEnv* env, jobject listener); void applyPendingHaptic(); void applyPassthroughState(); + void logInteractionProfiles(); void applyPendingRefreshRate(); void recenterAt(XrTime time); @@ -134,8 +135,9 @@ class XrGoggleSession XrAction mActionPassthrough = XR_NULL_HANDLE; XrAction mActionRecord = XR_NULL_HANDLE; XrAction mActionLockMode = XR_NULL_HANDLE; - XrAction mActionSizeStick = XR_NULL_HANDLE; - XrAction mActionPlaceStick = XR_NULL_HANDLE; + XrAction mActionStick = XR_NULL_HANDLE; + XrAction mActionNearer = XR_NULL_HANDLE; + XrAction mActionFarther = XR_NULL_HANDLE; XrAction mActionRaise = XR_NULL_HANDLE; XrAction mActionLower = XR_NULL_HANDLE; XrAction mActionHaptic = XR_NULL_HANDLE; @@ -159,12 +161,14 @@ class XrGoggleSession PFN_xrPassthroughLayerSetStyleFB mXrPassthroughLayerSetStyleFB = nullptr; PFN_xrEnumerateDisplayRefreshRatesFB mXrEnumerateDisplayRefreshRatesFB = nullptr; PFN_xrRequestDisplayRefreshRateFB mXrRequestDisplayRefreshRateFB = nullptr; + PFN_xrResumeSimultaneousHandsAndControllersTrackingMETA mXrResumeSimultaneous = nullptr; std::vector mAvailableExtensions; bool mHasPassthrough = false; bool mHasHandInteraction = false; bool mHasImageLayout = false; bool mHasMicrogestures = false; + bool mHasSimultaneous = false; bool mHasLayerSettings = false; bool mHasRefreshRate = false; From 6cf71ba0c8debc4e73bf2c3c883875aabef2bc60 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:27:57 +0200 Subject: [PATCH 13/23] Stop hand gestures firing by accident, and keep the flat activity out of the way Three problems reported from a session with the antenna connected. 1. Gestures triggered by accident - picking up the drone turned passthrough on. A whole-hand squeeze is exactly what grabbing a drone or a radio looks like, so the grasp bindings are gone. Pinch survives but now has to be *held*: separate hand-only actions fire once after 700 ms, so a momentary squeeze while handling the aircraft does nothing. The controller buttons are deliberately left immediate - they cannot be pressed by accident - which is why the hand variants are separate actions rather than a hold on the shared ones. Losing passthrough to a stray grab mid-flight is worse than not having the shortcut at all. Head lock loses its last input binding as a consequence and is now only in the flat settings menu. It is a set-once option, not an in-flight one. 2. Replugging the antenna opened the flat activity next to the running immersive one. The USB_DEVICE_ATTACHED filter lives on VideoActivity, so the system starts it on every attach. Two activities then fight over one adapter and one decoder and neither gets video. VideoActivity now steps aside when an immersive session is active and the launch came from that intent; the immersive activity already handles the attach itself through its own receiver. 3. No video, in either mode, with no crash. Diagnosis needs data the log did not have, so the whole chain is now visible: gs.key logs the byte count it wrote, and the wfb counters (all / decOk / decErr / fecRecovered / lost / bad / rssi) are logged every two seconds and drawn on the status screen. That separates "nothing is arriving from the air" from "packets arrive but nothing decodes", which are completely different bugs. Note for that hunt: WfbngLink::initAgg() no longer aborts the process when gs.key is unusable, so what used to be a loud crash is now silent - the log line is the only signal. --- .../java/com/openipc/pixelpilot/GsKey.java | 2 +- .../com/openipc/pixelpilot/VideoActivity.java | 10 +++ .../openipc/pixelpilot/XrVideoActivity.java | 44 ++++++++++ app/xr/src/main/cpp/XrGoggleSession.cpp | 81 +++++++++++++++++-- app/xr/src/main/cpp/XrGoggleSession.h | 13 +++ 5 files changed, 144 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/openipc/pixelpilot/GsKey.java b/app/src/main/java/com/openipc/pixelpilot/GsKey.java index 1383fbbc..3f5d8e3c 100644 --- a/app/src/main/java/com/openipc/pixelpilot/GsKey.java +++ b/app/src/main/java/com/openipc/pixelpilot/GsKey.java @@ -68,7 +68,7 @@ public static boolean writeToFilesDir(Context context) { File file = new File(context.getApplicationContext().getFilesDir(), FILE_NAME); try (OutputStream out = new FileOutputStream(file)) { out.write(keyBytes, 0, keyBytes.length); - Log.d(TAG, "Wrote gs.key to " + file.getAbsolutePath()); + Log.d(TAG, "Wrote gs.key (" + keyBytes.length + " bytes) to " + file.getAbsolutePath()); return true; } catch (IOException e) { Log.e(TAG, "Failed to write gs.key", e); diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index aa652d0e..fd3dfa8d 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -299,6 +299,16 @@ protected void onCreate(Bundle savedInstanceState) { Log.d(TAG, "lifecycle onCreate"); super.onCreate(savedInstanceState); + // The USB_DEVICE_ATTACHED filter is on this activity, so replugging the adapter + // starts it even while the immersive activity owns the link. Two activities fighting + // over one adapter and one decoder means neither gets video, so step aside. + if (XrVideoActivity.isActive() + && UsbManager.ACTION_USB_DEVICE_ATTACHED.equals(getIntent().getAction())) { + Log.i(TAG, "adapter attached while the immersive session is running, staying out of it"); + finish(); + return; + } + // UI Setup initializeUI(); diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index d80f8611..4b9d2483 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -68,6 +68,17 @@ public class XrVideoActivity extends AppCompatActivity static final String PREF_LAST_VIDEO_W = "xr_last_video_width"; static final String PREF_LAST_VIDEO_H = "xr_last_video_height"; + /** + * Whether an immersive session owns the link right now. The USB_DEVICE_ATTACHED filter + * lives on VideoActivity, so replugging the adapter would otherwise start the flat + * activity alongside a running immersive session. + */ + private static volatile boolean active; + + static boolean isActive() { + return active; + } + private final Handler handler = new Handler(Looper.getMainLooper()); private XrGoggleSession xr; @@ -87,6 +98,8 @@ public class XrVideoActivity extends AppCompatActivity // Link quality haptics private long lastHapticAt; private boolean linkWasBad; + private volatile WfbNGStats lastStats; + private long lastStatsLogAt; private final BroadcastReceiver usbReceiver = new BroadcastReceiver() { @Override @@ -104,6 +117,7 @@ public void onReceive(Context context, Intent intent) { @Override protected void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); + active = true; getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); // Only shown while still flat: USB permission, errors, "waiting for adapter". @@ -165,6 +179,7 @@ protected void onResume() { @Override protected void onDestroy() { + active = false; persistGeometry(); try { unregisterReceiver(usbReceiver); @@ -408,6 +423,20 @@ private void paintStatus(Canvas canvas) { canvas.drawText(status, cx, y, body); y += h / 20f; } + // The link counters answer the first question a pilot has when the screen stays + // empty: is anything arriving at all? + WfbNGStats stats = lastStats; + if (stats != null) { + canvas.drawText("wfb packets " + stats.count_p_all + + " ok " + stats.count_p_dec_ok + + " lost " + stats.count_p_lost + + " err " + stats.count_p_dec_err, + cx, y, body); + y += h / 20f; + } else { + canvas.drawText("no wfb statistics yet", cx, y, body); + y += h / 20f; + } canvas.drawText("channel " + VideoActivity.getChannel(this) + " @ " + VideoActivity.getBandwidth(this) + " MHz", cx, y, body); @@ -493,6 +522,21 @@ public void onDecodingInfoChanged(DecodingInfo decodingInfo) { @Override public void onWfbNgStatsChanged(WfbNGStats data) { + lastStats = data; + // Without this there is no way to tell "the adapter is dead" from "the link is fine + // but nothing decodes" while wearing the headset. + final long nowMs = System.currentTimeMillis(); + if (nowMs - lastStatsLogAt > 2000) { + lastStatsLogAt = nowMs; + Log.i(TAG, "wfb: all=" + data.count_p_all + + " decOk=" + data.count_p_dec_ok + + " decErr=" + data.count_p_dec_err + + " fecRec=" + data.count_p_fec_recovered + + " lost=" + data.count_p_lost + + " bad=" + data.count_p_bad + + " rssi=" + data.avg_rssi + + " videoSeen=" + videoSeen); + } if (xr == null || !linkStarted || data.count_p_all <= 0) { return; } diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index 006e1a71..c0b892ce 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -18,6 +18,9 @@ namespace { constexpr float kStickDeadzone = 0.30f; constexpr float kTriggerDeadzone = 0.15f; +// A hand gesture has to be held this long before it counts. Grabbing a drone or a radio +// squeezes the hand for a moment; deliberately holding a pinch does not happen by accident. +constexpr int64_t kHandHoldNs = 700LL * 1000 * 1000; constexpr float kDistancePerSec = 0.9f; // meters per second of full stick deflection constexpr float kWidthPerSec = 1.4f; constexpr float kMinDistance = 0.4f; @@ -676,6 +679,10 @@ bool XrGoggleSession::createActions() mActionLockMode = boolAction("lock_mode", "Toggle head lock"); mActionRaise = boolAction("raise", "Move the screen up"); mActionLower = boolAction("lower", "Move the screen down"); + // Separate actions so the hold requirement applies to the hand only - a controller + // button should still act the moment it is pressed. + mActionHandRecenter = boolAction("hand_recenter", "Recenter view (hold)"); + mActionHandPassthrough = boolAction("hand_passthrough", "Toggle passthrough (hold)"); XrActionCreateInfo stick{XR_TYPE_ACTION_CREATE_INFO}; stick.actionType = XR_ACTION_TYPE_VECTOR2F_INPUT; @@ -772,11 +779,12 @@ bool XrGoggleSession::createActions() // not to trigger each other. if (mHasHandInteraction) { + // No grasp bindings: a whole-hand squeeze is what you do to pick up a drone, and + // having that flip passthrough mid-flight is worse than not having the shortcut. + // Pinch survives, but only when held - see heldActionFired(). std::vector handBindings = { - {mActionRecenter, path("/user/hand/right/input/pinch_ext/value")}, - {mActionPassthrough, path("/user/hand/right/input/grasp_ext/value")}, - {mActionRecord, path("/user/hand/left/input/pinch_ext/value")}, - {mActionLockMode, path("/user/hand/left/input/grasp_ext/value")}, + {mActionHandRecenter, path("/user/hand/right/input/pinch_ext/value")}, + {mActionHandPassthrough, path("/user/hand/left/input/pinch_ext/value")}, }; // Thumb swipes along the index finger, for nudging the panel up and down without // spending pinch or grasp on it. @@ -813,7 +821,7 @@ bool XrGoggleSession::createActions() } else { - LOGI("hand interaction bindings suggested (pinch/grasp%s)", + LOGI("hand interaction bindings suggested (held pinch%s)", mHasMicrogestures ? " + microgesture height" : ""); } } @@ -1047,6 +1055,42 @@ void XrGoggleSession::pollEvents(JNIEnv* env, jobject listener, bool* exitLoop) } } +// A hand action fires once, after being held for kHandHoldNs. Anything shorter is treated +// as an accident. +bool XrGoggleSession::heldActionFired(XrAction action, int slot, XrTime now) +{ + if (action == XR_NULL_HANDLE) return false; + HeldButton& held = mHeld[slot]; + + XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO}; + get.action = action; + XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN}; + if (XR_FAILED(xrGetActionStateBoolean(mSession, &get, &state)) || !state.isActive) + { + held = HeldButton{}; + return false; + } + + if (!state.currentState) + { + held = HeldButton{}; + return false; + } + if (!held.down) + { + held.down = true; + held.fired = false; + held.downTime = now; + return false; + } + if (!held.fired && (now - held.downTime) >= kHandHoldNs) + { + held.fired = true; + return true; + } + return false; +} + void XrGoggleSession::logInteractionProfiles() { for (const char* hand : {"/user/hand/left", "/user/hand/right"}) @@ -1210,6 +1254,8 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) setHeadLocked(!mHeadLocked.load()); requestHaptic(0.4f, 30); break; + // Recenter and passthrough are dispatched from the held-gesture path too, so + // they are handled where the press is detected rather than here. case BUTTON_RAISE: setQuadHeightOffset(mQuadHeightOffset.load() + kHeightStep); requestHaptic(0.25f, 20); @@ -1229,6 +1275,29 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) } } + // Hand gestures, which need a deliberate hold before they count. + const XrTime nowNs = mLastPredictedDisplayTime; + if (heldActionFired(mActionHandRecenter, 0, nowNs)) + { + mRecenterRequested = true; + requestHaptic(0.5f, 60); + if (onButton != nullptr) + { + env->CallVoidMethod(listener, onButton, (jint) BUTTON_RECENTER); + if (env->ExceptionCheck()) env->ExceptionClear(); + } + } + if (heldActionFired(mActionHandPassthrough, 1, nowNs)) + { + mPassthroughWanted = !mPassthroughWanted.load(); + requestHaptic(0.5f, 60); + if (onButton != nullptr) + { + env->CallVoidMethod(listener, onButton, (jint) BUTTON_PASSTHROUGH); + if (env->ExceptionCheck()) env->ExceptionClear(); + } + } + const float dt = 1.0f / 72.0f; // good enough for a manual adjustment ramp XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO}; @@ -1306,6 +1375,8 @@ void XrGoggleSession::renderFrame(JNIEnv* env, jobject listener) XrFrameBeginInfo beginInfo{XR_TYPE_FRAME_BEGIN_INFO}; if (XR_FAILED(xrBeginFrame(mSession, &beginInfo))) return; + mLastPredictedDisplayTime = frameState.predictedDisplayTime; + applyPendingHaptic(); if (mRecenterRequested.exchange(false) || (!mHeadLocked.load() && !mAnchorValid)) diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index e0dd89c2..c58e5683 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -107,6 +107,8 @@ class XrGoggleSession void syncActions(JNIEnv* env, jobject listener); void applyPendingHaptic(); void applyPassthroughState(); + // Returns true on the frame a held action should fire, once per press. + bool heldActionFired(XrAction action, int slot, XrTime now); void logInteractionProfiles(); void applyPendingRefreshRate(); void recenterAt(XrTime time); @@ -138,6 +140,8 @@ class XrGoggleSession XrAction mActionStick = XR_NULL_HANDLE; XrAction mActionNearer = XR_NULL_HANDLE; XrAction mActionFarther = XR_NULL_HANDLE; + XrAction mActionHandRecenter = XR_NULL_HANDLE; + XrAction mActionHandPassthrough = XR_NULL_HANDLE; XrAction mActionRaise = XR_NULL_HANDLE; XrAction mActionLower = XR_NULL_HANDLE; XrAction mActionHaptic = XR_NULL_HANDLE; @@ -173,6 +177,7 @@ class XrGoggleSession bool mHasRefreshRate = false; // --- state -------------------------------------------------------------------- + XrTime mLastPredictedDisplayTime = 0; XrSessionState mSessionState = XR_SESSION_STATE_UNKNOWN; bool mSessionRunning = false; std::atomic mStopRequested{false}; @@ -200,6 +205,14 @@ class XrGoggleSession std::string mManifestDir; std::string mRuntimeLibrary; + // Press bookkeeping for the hand actions, which need a deliberate hold. + struct HeldButton + { + bool down = false; + bool fired = false; + XrTime downTime = 0; + }; + HeldButton mHeld[2]; std::vector mRefreshRates; std::mutex mErrorMutex; std::string mLastError; From 3a213414dc7d3d6a21e81123ae2afdf8a63976c2 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:38:02 +0200 Subject: [PATCH 14/23] Fix: several RX loops on one adapter, and the ANR that came with it Diagnosed from a thread dump on a Quest 3 while the app was unresponsive: 3 S wfb-001/002 <- three live RX loops, one dongle 1 S nipc.pixelpilot <- main thread asleep in join() Two libusb handles on the same interface means neither gets usable video, which is why no picture arrived in either the flat or the immersive mode, and the unbounded join() in a lifecycle callback turned it into a five second ANR. How three threads happen: - activeWifiAdapters in WfbLinkManager was static, so the flat and the immersive activity shared it while each owned its own WfbNgLink. startAdapters() only guards on wfbLink.isRunning(), which is per instance, so both decided the adapter still needed starting. - linkThreads.put(usbDevice, t) overwrites the previous entry, so the older thread is orphaned and never joined, and stopAll() only ever joins the last. - nothing refused a second RX loop for a device that already had one. Fixes: - a process-wide claim on the device name, keyed on the owning thread, so a second RX loop is refused no matter which WfbNgLink instance asks. Keying on the thread means a claim left behind by a thread that died is ignored instead of locking the adapter out permanently; the claim is released in a finally block around nativeRun(). - a per-instance check for an already-live thread, so the common case does not even reach the claim. - joins are bounded to 1500 ms and log when a driver thread does not come back. These calls come from Activity lifecycle callbacks on the main thread, where an unbounded join is an ANR waiting to happen. - activeWifiAdapters is no longer static. The real answer is a single owner for the link - a foreground service - but this stops the immediate damage. --- .../openipc/pixelpilot/WfbLinkManager.java | 5 +- .../com/openipc/wfbngrtl8812/WfbNgLink.java | 84 ++++++++++++++++--- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java index 58c32789..7737f62c 100644 --- a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java +++ b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java @@ -18,7 +18,10 @@ public class WfbLinkManager extends BroadcastReceiver { public static final String ACTION_USB_PERMISSION = "com.openipc.pixelpilot.USB_PERMISSION"; private static final String TAG = "pixelpilot"; - static Map activeWifiAdapters = new HashMap<>(); + // Not static: it used to be, and with two activities each owning its own WfbNgLink + // that shared map let both of them decide the adapter still needed starting, so the + // same dongle ended up with several RX loops. + private final Map activeWifiAdapters = new HashMap<>(); private final WfbNgLink wfbLink; private final LinkStatusView status; private final Context context; diff --git a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java index 39dca17d..49a79522 100644 --- a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java +++ b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java @@ -30,6 +30,31 @@ public void setFecThresholds(int lostTo5, int recTo4, int recTo3, int recTo2, in System.loadLibrary("WfbngRtl8812"); } + /** + * Devices with a live RX loop anywhere in the process, keyed by device name. + * + *

Per-instance bookkeeping is not enough: each Activity owns its own WfbNgLink, so + * two of them both see an empty map and both claim the same dongle. Keyed on the thread + * so a claim left behind by a thread that died is simply ignored rather than locking the + * adapter out for good. + */ + private static final Map claimedDevices = new HashMap<>(); + + private static synchronized boolean claim(String deviceName, Thread t) { + Thread holder = claimedDevices.get(deviceName); + if (holder != null && holder.isAlive()) { + return false; + } + claimedDevices.put(deviceName, t); + return true; + } + + private static synchronized void release(String deviceName, Thread t) { + if (claimedDevices.get(deviceName) == t) { + claimedDevices.remove(deviceName); + } + } + private final long nativeWfbngLink; private final Timer timer; private final Context context; @@ -93,6 +118,15 @@ public void nativeSetUseStbc(int use) { } public synchronized boolean start(int wifiChannel, int bandWidth, UsbDevice usbDevice) { + // A second RX loop on the same adapter means two libusb handles on one interface: + // neither gets usable video, the old thread is orphaned because linkThreads.put() + // overwrites its entry, and stopAll() then joins only the last one - which is how + // three live "wfb-001/002" threads and a hung main thread happen. + Thread existing = linkThreads.get(usbDevice); + if (existing != null && existing.isAlive()) { + Log.w(TAG, "wfb-ng already running on " + usbDevice.getDeviceName() + ", not starting a second"); + return true; + } Log.d(TAG, "wfb-ng monitoring on " + usbDevice.getDeviceName() + " using wifi channel " + wifiChannel); UsbManager usbManager = (UsbManager) context.getSystemService(Context.USB_SERVICE); // Returns null when the permission was revoked or the device disappeared between @@ -108,12 +142,26 @@ public synchronized boolean start(int wifiChannel, int bandWidth, UsbDevice usbD usbDeviceConnection.close(); return false; } - Thread t = new Thread(() -> nativeRun(nativeWfbngLink, context, wifiChannel, bandWidth, fd)); + final String deviceName = usbDevice.getDeviceName(); + final Thread[] self = new Thread[1]; + Thread t = new Thread(() -> { + try { + nativeRun(nativeWfbngLink, context, wifiChannel, bandWidth, fd); + } finally { + release(deviceName, self[0]); + } + }); + self[0] = t; t.setName(threadNameFor(usbDevice)); + if (!claim(deviceName, t)) { + Log.w(TAG, "another wfb-ng RX loop already owns " + deviceName + ", not starting a second"); + usbDeviceConnection.close(); + return false; + } linkThreads.put(usbDevice, t); linkConns.put(usbDevice, usbDeviceConnection); t.start(); - Log.d(TAG, "wfb-ng thread on " + usbDevice.getDeviceName() + " started."); + Log.d(TAG, "wfb-ng thread on " + deviceName + " started."); return true; } @@ -123,19 +171,36 @@ private static String threadNameFor(UsbDevice usbDevice) { return "wfb-" + (parts.length > 1 ? parts[1] : name); } + /** + * The RX loop is joined so the USB interface is released before anything reopens it, but + * these calls come from Activity lifecycle callbacks on the main thread. An unbounded + * join there is a five second ANR waiting to happen, and it did: a driver thread that + * does not come back froze the UI. Wait, but give up and say so. + */ + private static final long JOIN_TIMEOUT_MS = 1500; + + private static void joinBounded(Thread t, String what) throws InterruptedException { + if (t == null) { + return; + } + t.join(JOIN_TIMEOUT_MS); + if (t.isAlive()) { + Log.e(TAG, "wfb-ng thread on " + what + " did not stop within " + JOIN_TIMEOUT_MS + + "ms, leaving it behind"); + } else { + Log.d(TAG, "wfb-ng thread on " + what + " done."); + } + } + public synchronized void stopAll() throws InterruptedException { for (Map.Entry entry : linkConns.entrySet()) { nativeStop(nativeWfbngLink, context, entry.getValue().getFileDescriptor()); } for (Map.Entry entry : linkConns.entrySet()) { - Thread t = linkThreads.get(entry.getKey()); - if (t != null) { - t.join(); - } + joinBounded(linkThreads.get(entry.getKey()), entry.getKey().getDeviceName()); // The connection holds a dup of the usbfs fd. Without close() every // attach/detach cycle leaks one, until the process runs out. entry.getValue().close(); - Log.d(TAG, "wfb-ng thread on " + entry.getKey().getDeviceName() + " done."); } linkThreads.clear(); linkConns.clear(); @@ -148,10 +213,7 @@ public synchronized void stop(UsbDevice dev) throws InterruptedException { } int fd = conn.getFileDescriptor(); nativeStop(nativeWfbngLink, context, fd); - Thread t = linkThreads.get(dev); - if (t != null) { - t.join(); - } + joinBounded(linkThreads.get(dev), dev.getDeviceName()); linkThreads.remove(dev); linkConns.remove(dev); conn.close(); From 6c435fb4d7a634adaafa98ee62be0747d069d4ad Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:46:55 +0200 Subject: [PATCH 15/23] Say which adapter problem it actually is, and log the ids "No compatible wifi adapter found." was shown whenever no adapter ended up running, which covers three unrelated problems: nothing plugged in, a dongle whose id is not in usb_device_filter.xml, and a dongle that was found but failed to start. Only the middle one is what the message claims, and the last one became reachable when #116 stopped recording an adapter as active unless it actually came up. - getAttachedAdapters() logs every attached device with VID:PID, manufacturer and product name, and whether the filter matched. That id is exactly what a bug report needs in order to add an adapter, and there is no other way to read it - sysfs is not accessible to the shell on Horizon OS and dumpsys usb does not print host devices there. - the message now distinguishes "none attached", "found but could not be started - see the log" and "waiting for permission". Also: joinBounded() now releases the device claim even when the driver thread outstays the timeout. Holding it would leave the adapter unopenable until the process restarts, and the user would be told there is no compatible adapter - a worse failure than the duplicate RX loop it guards against, which the per-instance check and the single-owner handoff already cover. --- .../openipc/pixelpilot/WfbLinkManager.java | 27 +++++++++++++++++-- .../com/openipc/wfbngrtl8812/WfbNgLink.java | 7 ++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java index 7737f62c..bc8f157b 100644 --- a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java +++ b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java @@ -103,7 +103,12 @@ public Map getAttachedAdapters() { } Map res = new HashMap<>(); - for (UsbDevice dev : manager.getDeviceList().values()) { + Map attached = manager.getDeviceList(); + // Without this there is no way to tell "nothing is plugged in" from "the dongle is + // there but its id is not in usb_device_filter.xml", and the id is what a bug report + // needs in order to add it. + Log.i(TAG, "usb devices attached: " + attached.size()); + for (UsbDevice dev : attached.values()) { boolean allowed = false; for (UsbDeviceFilter filter : filters) { if (filter.productId == dev.getProductId() && filter.vendorId == dev.getVendorId()) { @@ -111,6 +116,13 @@ public Map getAttachedAdapters() { break; } } + Log.i(TAG, String.format(" %s %04X:%04X %s %s -> %s", + dev.getDeviceName(), + dev.getVendorId(), + dev.getProductId(), + String.valueOf(dev.getManufacturerName()), + String.valueOf(dev.getProductName()), + allowed ? "supported" : "NOT in usb_device_filter.xml")); if (!allowed) { continue; } @@ -160,6 +172,7 @@ public synchronized void refreshAdapters() { } // Starts newly attached adapters. + boolean startFailed = false; for (Map.Entry entry : attachedAdapters.entrySet()) { if (activeWifiAdapters.containsKey(entry.getKey())) { continue; @@ -168,11 +181,21 @@ public synchronized void refreshAdapters() { // is never retried on the next refresh. if (startAdapter(entry.getValue())) { activeWifiAdapters.put(entry.getKey(), entry.getValue()); + } else { + startFailed = true; } } if (activeWifiAdapters.isEmpty()) { - status.showLinkMessage("No compatible wifi adapter found."); + // These are three different problems and telling the pilot the wrong one costs + // a lot of pointless searching. + if (startFailed) { + status.showLinkMessage("Wifi adapter found but could not be started - see the log."); + } else if (attachedAdapters.isEmpty()) { + status.showLinkMessage("No compatible wifi adapter found."); + } else { + status.showLinkMessage("Waiting for wifi adapter permission…"); + } String wifi = VideoActivity.wirelessInfo(context); if (wifi != null) { diff --git a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java index 49a79522..99993cea 100644 --- a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java +++ b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java @@ -186,10 +186,15 @@ private static void joinBounded(Thread t, String what) throws InterruptedExcepti t.join(JOIN_TIMEOUT_MS); if (t.isAlive()) { Log.e(TAG, "wfb-ng thread on " + what + " did not stop within " + JOIN_TIMEOUT_MS - + "ms, leaving it behind"); + + "ms, giving up the claim anyway"); } else { Log.d(TAG, "wfb-ng thread on " + what + " done."); } + // Drop the claim even if the thread outstayed its welcome. Holding it would leave the + // adapter unopenable until the process restarts, and the user would be told there is + // no compatible adapter - a worse failure than the duplicate this guards against, + // which the per-instance check and the single-owner handoff already prevent. + release(what, t); } public synchronized void stopAll() throws InterruptedException { From 35f87e6b3f6a5167cb64cd2ee42d92b16afd4082 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:53:30 +0200 Subject: [PATCH 16/23] Retry the adapter handoff into immersive mode instead of failing it once Flat mode has video again once USB permission is granted, but entering immersive mode then reported that the adapter could not be opened. The handoff is the problem. VideoActivity.onPause() calls stopAll(), which asks the driver to stop and waits for the thread. StopRxLoop() only breaks the receive loop - the thread then still has to stop the TX frame, stop the adaptive link, power the chip down, release the USB interface and exit libusb. That takes longer than the 1500 ms bound added with the ANR fix, so the wait gave up, the immersive activity tried to claim an interface that was still held, failed once, and gave up too. - the join bound goes to 3000 ms, which is about what the unwind needs, while still being bounded so a stuck driver cannot freeze the UI. - the immersive activity retries bringing the adapter up every 700 ms for up to 14 s instead of trying once, and logs each attempt. The session is started regardless, so the pilot gets the status screen with a reason rather than a silent flat activity. - onXrReady() no longer calls startAdapters(): refreshAdapters() already brings adapters up and now retries, and two paths racing for one dongle is what caused the duplicate RX loops in the first place. - WfbLinkManager gained hasActiveAdapter() so a caller can tell "up" from "not up yet" rather than inferring it. Also: VideoActivity.showLinkMessage() now logs. It only ever reached the overlay, so a bug report had no record of what the app said about the link - a gap that cost real time in this hunt. --- .../com/openipc/pixelpilot/VideoActivity.java | 3 +++ .../openipc/pixelpilot/WfbLinkManager.java | 5 +++++ .../openipc/pixelpilot/XrVideoActivity.java | 20 ++++++++++++++++++- .../com/openipc/wfbngrtl8812/WfbNgLink.java | 5 ++++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index fd3dfa8d..1d4f782d 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -1748,6 +1748,9 @@ public void onWfbNgStatsChanged(WfbNGStats data) { @Override public void showLinkMessage(String message) { + // Also to the log: this used to only reach the overlay, so a bug report had no + // record of what the app actually said about the link. + Log.i(TAG, "link: " + message); runOnUiThread(() -> { binding.tvMessage.setVisibility(View.VISIBLE); binding.tvMessage.setText(message); diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java index bc8f157b..e1fa0dae 100644 --- a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java +++ b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java @@ -204,6 +204,11 @@ public synchronized void refreshAdapters() { } } + /** True when at least one adapter is up, so a caller can retry instead of guessing. */ + public synchronized boolean hasActiveAdapter() { + return !activeWifiAdapters.isEmpty(); + } + public synchronized void stopAdapters() { try { wfbLink.stopAll(); diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index 4b9d2483..c06b537b 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -87,6 +87,12 @@ static boolean isActive() { private WfbLinkManager wfbLinkManager; private TextView statusView; + // The flat activity may still be releasing the USB interface when we get here, so the + // adapter is retried for a while rather than given up on after one attempt. + private static final int LINK_RETRY_INTERVAL_MS = 700; + private static final int LINK_RETRY_LIMIT = 20; + + private int linkRetries; private boolean xrStarting; private boolean linkStarted; private Surface xrSurface; @@ -238,11 +244,22 @@ private void tryBringUpLink() { // refreshAdapters() has asked for permission; the broadcast brings us back. return; } + + // Going immersive does not depend on the link, and the pilot is better off seeing the + // status screen with a reason than a flat activity that says nothing. if (!xrStarting) { xrStarting = true; statusView.setText(R.string.xr_entering); xr.start(); } + + if (!wfbLinkManager.hasActiveAdapter() && linkRetries < LINK_RETRY_LIMIT) { + linkRetries++; + Log.i(TAG, "adapter not up yet, retry " + linkRetries + "/" + LINK_RETRY_LIMIT); + handler.postDelayed(this::tryBringUpLink, LINK_RETRY_INTERVAL_MS); + } else if (wfbLinkManager.hasActiveAdapter()) { + linkRetries = 0; + } } /** True when every attached compatible adapter is usable (or there is none at all). */ @@ -276,7 +293,8 @@ public void onXrReady(Surface videoSurface) { applyRefreshRatePreference(); statusView.setVisibility(View.GONE); linkStarted = true; - wfbLinkManager.startAdapters(); + // Not startAdapters(): tryBringUpLink() -> refreshAdapters() already owns bringing the + // adapter up, and it retries. Two paths racing for one dongle is what broke this. } private void applyRefreshRatePreference() { diff --git a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java index 99993cea..7e4ae984 100644 --- a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java +++ b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java @@ -177,7 +177,10 @@ private static String threadNameFor(UsbDevice usbDevice) { * join there is a five second ANR waiting to happen, and it did: a driver thread that * does not come back froze the UI. Wait, but give up and say so. */ - private static final long JOIN_TIMEOUT_MS = 1500; + // StopRxLoop() only breaks the receive loop; the thread then still has to stop the TX + // frame, the adaptive link, power the chip down and release the USB interface. Too + // short a wait here and the next owner cannot claim the interface yet. + private static final long JOIN_TIMEOUT_MS = 3000; private static void joinBounded(Thread t, String what) throws InterruptedException { if (t == null) { From fa8ab4136d41427c3294110b5bdacbb5721e9b56 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:03:33 +0200 Subject: [PATCH 17/23] Fix: both mode activities stayed alive and the adapter ended up owned by nobody With video working in flat mode, switching to immersive mode and back left no video in either. The activity dump explains it: topResumedActivity: .VideoActivity t5249 topResumedActivity: .XrVideoActivity t5253 threads: xr-goggle (no wfb-* at all) Both activities were alive in separate tasks - Horizon OS keeps the immersive one in its own task, so its onDestroy(), and with it stopAdapters(), never ran - while the flat one had also given the adapter up. Each activity constructs its own WfbNgLink, so with two of them the adapter belongs to neither. ModeOwner makes that impossible: whoever starts last evicts the other, whose onDestroy() releases the adapter. A WeakReference so a finished activity is not retained. The real answer is a single owner outliving both - a foreground service, which would also fix the flat mode dropping the link in onPause - but this removes the failure without that rework. Also corrects a wrong reading in the previous commit: three threads named "wfb-001/002" is *normal* for one adapter, because the TX and adaptive-link threads inherit the RX thread's name - nothing renames them. It is not three RX loops. The duplicate guard is still correct, since two activities each with their own WfbNgLink genuinely can double-start, but the evidence quoted for it was misread. Confirmed on hardware in the same run: the MediaCodec low-latency keys from #113 are accepted by Meta's H.265 decoder - "low-latency: 1, vendor.qti-ext-dec-low-latency.enable: 1, priority: 0" followed by "AMediaCodec_configure: OK". --- .../com/openipc/pixelpilot/ModeOwner.java | 45 +++++++++++++++++++ .../com/openipc/pixelpilot/VideoActivity.java | 7 +++ .../openipc/pixelpilot/XrVideoActivity.java | 4 ++ .../com/openipc/wfbngrtl8812/WfbNgLink.java | 12 +++-- 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 app/src/main/java/com/openipc/pixelpilot/ModeOwner.java diff --git a/app/src/main/java/com/openipc/pixelpilot/ModeOwner.java b/app/src/main/java/com/openipc/pixelpilot/ModeOwner.java new file mode 100644 index 00000000..80443d10 --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/ModeOwner.java @@ -0,0 +1,45 @@ +package com.openipc.pixelpilot; + +import android.app.Activity; +import android.util.Log; + +import java.lang.ref.WeakReference; + +/** + * Guarantees that only one of the two presentation modes is alive at a time. + * + *

The flat and the immersive activity each construct their own {@code WfbNgLink}, and the + * wifi adapter can only belong to one of them. Horizon OS keeps the immersive activity in its + * own task, so switching modes left both of them resumed in separate tasks with the adapter + * owned by neither, and no video in either mode. + * + *

Rather than trying to make two owners cooperate, whoever starts last evicts the other + * and its {@code onDestroy()} releases the adapter. The proper answer is a single owner that + * outlives both - a foreground service - but this removes the failure without that rework. + */ +final class ModeOwner { + private static final String TAG = "pixelpilot"; + + private static WeakReference current = new WeakReference<>(null); + + private ModeOwner() { + } + + /** Registers {@code activity} as the mode owner and finishes whoever held it before. */ + static synchronized void claim(Activity activity) { + Activity previous = current.get(); + if (previous != null && previous != activity && !previous.isFinishing()) { + Log.i(TAG, "mode switch: finishing " + previous.getClass().getSimpleName() + + " so " + activity.getClass().getSimpleName() + " can own the adapter"); + previous.finish(); + } + current = new WeakReference<>(activity); + } + + /** Clears the registration if {@code activity} still holds it. */ + static synchronized void release(Activity activity) { + if (current.get() == activity) { + current = new WeakReference<>(null); + } + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index 1d4f782d..06e270ae 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -308,6 +308,7 @@ protected void onCreate(Bundle savedInstanceState) { finish(); return; } + ModeOwner.claim(this); // UI Setup initializeUI(); @@ -1562,6 +1563,12 @@ protected void onPause() { startService(intent); } + @Override + protected void onDestroy() { + ModeOwner.release(this); + super.onDestroy(); + } + @Override protected void onStop() { MavlinkNative.nativeStop(this); diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index c06b537b..2caefec6 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -124,6 +124,9 @@ public void onReceive(Context context, Intent intent) { protected void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); active = true; + // Evict the flat activity: it owns a WfbNgLink too, and the adapter can only belong + // to one of them. + ModeOwner.claim(this); getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); // Only shown while still flat: USB permission, errors, "waiting for adapter". @@ -186,6 +189,7 @@ protected void onResume() { @Override protected void onDestroy() { active = false; + ModeOwner.release(this); persistGeometry(); try { unregisterReceiver(usbReceiver); diff --git a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java index 7e4ae984..ce8e3e6a 100644 --- a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java +++ b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java @@ -118,10 +118,14 @@ public void nativeSetUseStbc(int use) { } public synchronized boolean start(int wifiChannel, int bandWidth, UsbDevice usbDevice) { - // A second RX loop on the same adapter means two libusb handles on one interface: - // neither gets usable video, the old thread is orphaned because linkThreads.put() - // overwrites its entry, and stopAll() then joins only the last one - which is how - // three live "wfb-001/002" threads and a hung main thread happen. + // A second RX loop on the same adapter means two libusb handles on one interface, + // and the old thread is orphaned because linkThreads.put() overwrites its entry, so + // stopAll() then joins only the last one. + // + // (Note: three threads named "wfb-001/002" is *normal* for one adapter - the TX and + // adaptive-link threads inherit the RX thread's name because nothing renames them. + // An earlier reading of that as three RX loops was wrong; the guard is still right, + // because two activities each with their own WfbNgLink really can double-start.) Thread existing = linkThreads.get(usbDevice); if (existing != null && existing.isAlive()) { Log.w(TAG, "wfb-ng already running on " + usbDevice.getDeviceName() + ", not starting a second"); From 58e026a2cac5e97d3e483bc0f3de57186f406cd1 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:11:45 +0200 Subject: [PATCH 18/23] Move the link into a foreground service so it has a single owner The adapter, the wfb-ng link and the decoder belonged to whichever Activity was on screen. That does not survive contact with a ground station: - the flat activity tore the link down in onPause(), so a system menu, a file picker or taking the headset off cost seconds of black screen and a fresh USB enumeration - with a second presentation mode there were two owners for one adapter. Both activities stayed alive in separate tasks (Horizon OS keeps the immersive one in its own task, so its onDestroy never ran) and the adapter ended up owned by neither - no video in either mode - every mode switch reopened the adapter, which is also what made the handoff racy enough to need a claim guard, a bounded join and a retry loop LinkService now owns all three and outlives both activities. Activities bind for their whole lifetime, register as a LinkService.Client and hand over a Surface. Switching modes only swaps the decoder's output surface: reception and the adapter are never touched, so there is no reconnect and no second USB permission prompt. It is a foreground service with a Stop action, because it has to keep running while the pilot is looking at something else. - LinkClientActivity: shared bind/unbind, bound in onCreate and unbound in onDestroy rather than around start/stop, which is the point. - VideoPlayer gained attachSurface()/detachSurface(): a surface change rebuilds the decoder without stopping the receivers. stopAndRemoveReceiverDecoder() stopped the receivers too and nothing restarted them, which is why a hot surface swap was not possible before. - VideoPlayer and WfbNgLink take a Context instead of an AppCompatActivity. A decoder and a radio link have no business requiring an Activity, and the service cannot provide one. - the USB attach/detach receiver moved to the service: WfbLinkManager is that receiver, and it belongs to whoever owns the adapter. Registered by an activity, plug events were missed whenever none was around. - the flat activity no longer stops the link in onPause/onStop, and no longer tears it down to show a folder picker. - the service replays link status, resolution and stats to a client on attach, so a freshly opened mode never starts out blank. ModeOwner stays: one mode at a time is still the right model, and two surfaces on index 0 would fight. --- app/src/main/AndroidManifest.xml | 7 + .../pixelpilot/LinkClientActivity.java | 98 +++++ .../com/openipc/pixelpilot/LinkService.java | 405 ++++++++++++++++++ .../com/openipc/pixelpilot/VideoActivity.java | 198 +++++---- .../openipc/pixelpilot/XrVideoActivity.java | 131 ++---- .../com/openipc/videonative/VideoPlayer.java | 18 +- .../com/openipc/wfbngrtl8812/WfbNgLink.java | 3 +- 7 files changed, 684 insertions(+), 176 deletions(-) create mode 100644 app/src/main/java/com/openipc/pixelpilot/LinkClientActivity.java create mode 100644 app/src/main/java/com/openipc/pixelpilot/LinkService.java diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f742245f..1f0fd554 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -50,6 +50,13 @@ android:roundIcon="@mipmap/ic_launcher_round" android:usesCleartextTraffic="true" android:supportsRtl="true"> + + + Bound in {@code onCreate} and unbound in {@code onDestroy} rather than around + * start/stop: the point of the service is that the link survives the activity being paused, + * so a system menu or taking the headset off no longer costs a reconnect. + */ +public abstract class LinkClientActivity extends AppCompatActivity implements LinkService.Client { + + private static final String TAG = "pixelpilot"; + + @Nullable + protected LinkService link; + private boolean bound; + + private final ServiceConnection connection = new ServiceConnection() { + @Override + public void onServiceConnected(ComponentName name, IBinder service) { + link = ((LinkService.LocalBinder) service).service(); + link.addClient(LinkClientActivity.this); + Log.i(TAG, "attached to the link service"); + onLinkServiceConnected(link); + } + + @Override + public void onServiceDisconnected(ComponentName name) { + // The process is going away; nothing useful to do but stop talking to it. + link = null; + } + }; + + /** Called on the main thread once the link is available. */ + protected abstract void onLinkServiceConnected(LinkService service); + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + Intent intent = new Intent(this, LinkService.class); + // Started as well as bound, so it keeps running while no activity is attached - that + // is what carries the link across a mode switch. + startForegroundService(intent); + bound = bindService(intent, connection, Context.BIND_AUTO_CREATE); + if (!bound) { + Log.e(TAG, "could not bind the link service"); + } + } + + @Override + protected void onDestroy() { + if (link != null) { + link.removeClient(this); + } + if (bound) { + unbindService(connection); + bound = false; + } + link = null; + super.onDestroy(); + } + + // Defaults so a subclass only overrides what it draws. + + @Override + public void onLinkStatus(String message) { + } + + @Override + public void onLocalStreamHint(String url) { + } + + @Override + public void onVideoResolution(int width, int height) { + } + + @Override + public void onDecodingInfo(DecodingInfo info) { + } + + @Override + public void onWfbStats(WfbNGStats stats) { + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/LinkService.java b/app/src/main/java/com/openipc/pixelpilot/LinkService.java new file mode 100644 index 00000000..72d81d0a --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/LinkService.java @@ -0,0 +1,405 @@ +package com.openipc.pixelpilot; + +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.app.PendingIntent; +import android.app.Service; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.hardware.usb.UsbManager; +import android.os.Binder; +import android.os.Build; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.util.Log; +import android.view.Surface; + +import androidx.annotation.Nullable; + +import com.openipc.videonative.DecodingInfo; +import com.openipc.videonative.IVideoParamsChanged; +import com.openipc.videonative.VideoPlayer; +import com.openipc.wfbngrtl8812.WfbNGStats; +import com.openipc.wfbngrtl8812.WfbNGStatsChanged; +import com.openipc.wfbngrtl8812.WfbNgLink; + +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Sole owner of the wifi adapter, the wfb-ng link and the decoder. + * + *

These used to belong to whichever Activity was on screen, which does not survive contact + * with a ground station: the flat activity tore the link down in {@code onPause()} (so + * glancing at a system menu cost seconds of black screen), and with a second presentation + * mode there were two owners for one adapter - both activities ended up alive in separate + * tasks with the adapter owned by neither, and no video in either mode. + * + *

Now the link outlives the activities. They bind, register as a {@link Client}, and hand + * over a Surface; switching modes only swaps the Surface, so the adapter is never reopened + * and the USB permission dialog is not asked again. The service is a foreground service + * because it must keep running while the pilot is looking at something else. + */ +public class LinkService extends Service implements LinkStatusView, IVideoParamsChanged, WfbNGStatsChanged { + + public static final String ACTION_STOP = "com.openipc.pixelpilot.LINK_STOP"; + + private static final String TAG = "pixelpilot"; + private static final String CHANNEL_ID = "link"; + private static final int NOTIFICATION_ID = 1; + + /** + * Grace period after the last client unbinds. A mode switch unbinds one activity and + * binds the next, and tearing the link down in between is exactly what this service + * exists to avoid. + */ + private static final long IDLE_SHUTDOWN_MS = 20_000; + + /** What an attached activity gets told. All callbacks arrive on the main thread. */ + public interface Client { + void onLinkStatus(String message); + + void onLocalStreamHint(String url); + + void onVideoResolution(int width, int height); + + void onDecodingInfo(DecodingInfo info); + + void onWfbStats(WfbNGStats stats); + } + + public class LocalBinder extends Binder { + public LinkService service() { + return LinkService.this; + } + } + + private final LocalBinder binder = new LocalBinder(); + private final Handler main = new Handler(Looper.getMainLooper()); + private final CopyOnWriteArrayList clients = new CopyOnWriteArrayList<>(); + + private WfbNgLink wfbLink; + private WfbLinkManager wfbLinkManager; + private VideoPlayer videoPlayer; + + private boolean ready; + private boolean usbReceiverRegistered; + private String lastStatus = ""; + private WfbNGStats lastStats; + private int videoWidth; + private int videoHeight; + + private final Runnable idleShutdown = () -> { + Log.i(TAG, "no client came back, stopping the link service"); + stopSelf(); + }; + + // ------------------------------------------------------------------------------ + // lifecycle + // ------------------------------------------------------------------------------ + + @Override + public void onCreate() { + super.onCreate(); + startForegroundNotification(); + + // The native aggregators open files/gs.key in their constructor and throw if it is + // missing, so this has to happen before WfbNgLink exists. + if (!GsKey.ensure(this)) { + Log.e(TAG, "no usable gs.key, the link cannot come up"); + } + + wfbLink = new WfbNgLink(this); + wfbLink.SetWfbNGStatsChanged(this); + // Not read by the native side on its own: adaptive link, TX power, FEC/LDPC/STBC. + WfbOptions.applyDefaults(this, wfbLink); + + wfbLinkManager = new WfbLinkManager(this, this, wfbLink); + + videoPlayer = new VideoPlayer(this); + videoPlayer.setIVideoParamsChanged(this); + videoPlayer.setLowLatency(VideoActivity.getLowLatencySetting(this)); + // Receivers come up once and stay up; only the output surface changes with the mode. + videoPlayer.start(); + videoPlayer.startAudio(); + + registerUsbReceiver(); + + ready = true; + Log.i(TAG, "link service up"); + } + + @Override + public int onStartCommand(Intent intent, int flags, int startId) { + if (intent != null && ACTION_STOP.equals(intent.getAction())) { + Log.i(TAG, "stop requested from the notification"); + stopSelf(); + return START_NOT_STICKY; + } + return START_STICKY; + } + + @Nullable + @Override + public IBinder onBind(Intent intent) { + main.removeCallbacks(idleShutdown); + return binder; + } + + @Override + public void onRebind(Intent intent) { + main.removeCallbacks(idleShutdown); + } + + @Override + public boolean onUnbind(Intent intent) { + if (clients.isEmpty()) { + main.postDelayed(idleShutdown, IDLE_SHUTDOWN_MS); + } + // Allow onRebind, which is how a mode switch keeps the link alive. + return true; + } + + @Override + public void onDestroy() { + Log.i(TAG, "link service going down"); + main.removeCallbacks(idleShutdown); + ready = false; + if (usbReceiverRegistered) { + try { + unregisterReceiver(wfbLinkManager); + } catch (IllegalArgumentException ignored) { + } + usbReceiverRegistered = false; + } + if (wfbLinkManager != null) { + wfbLinkManager.stopAdapters(); + } + if (videoPlayer != null) { + videoPlayer.stopAudio(); + videoPlayer.stop(); + } + super.onDestroy(); + } + + /** + * WfbLinkManager is itself the USB attach/detach receiver, and it has to be registered by + * whoever owns the adapter - previously an Activity, which meant plug events were missed + * whenever no activity was around. + */ + private void registerUsbReceiver() { + IntentFilter filter = new IntentFilter(); + filter.addAction(UsbManager.ACTION_USB_DEVICE_ATTACHED); + filter.addAction(UsbManager.ACTION_USB_DEVICE_DETACHED); + filter.addAction(WfbLinkManager.ACTION_USB_PERMISSION); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + registerReceiver(wfbLinkManager, filter, Context.RECEIVER_NOT_EXPORTED); + } else { + registerReceiver(wfbLinkManager, filter); + } + usbReceiverRegistered = true; + } + + private void startForegroundNotification() { + NotificationManager manager = getSystemService(NotificationManager.class); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && manager != null + && manager.getNotificationChannel(CHANNEL_ID) == null) { + NotificationChannel channel = new NotificationChannel( + CHANNEL_ID, "Video link", NotificationManager.IMPORTANCE_LOW); + channel.setShowBadge(false); + manager.createNotificationChannel(channel); + } + + Intent stop = new Intent(this, LinkService.class).setAction(ACTION_STOP); + PendingIntent stopIntent = PendingIntent.getService( + this, 0, stop, PendingIntent.FLAG_IMMUTABLE | PendingIntent.FLAG_UPDATE_CURRENT); + + Notification notification = new Notification.Builder(this, CHANNEL_ID) + .setContentTitle("PixelPilot link") + .setContentText("Holding the wifi adapter and decoder") + .setSmallIcon(R.drawable.baseline_settings_24) + .setOngoing(true) + .addAction(new Notification.Action.Builder(null, "Stop", stopIntent).build()) + .build(); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + startForeground(NOTIFICATION_ID, notification, + android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC); + } else { + startForeground(NOTIFICATION_ID, notification); + } + } + + // ------------------------------------------------------------------------------ + // client API + // ------------------------------------------------------------------------------ + + public boolean isReady() { + return ready; + } + + /** Registers a client and replays the state it missed, so it never starts out blank. */ + public void addClient(Client client) { + clients.addIfAbsent(client); + main.removeCallbacks(idleShutdown); + if (!lastStatus.isEmpty()) { + client.onLinkStatus(lastStatus); + } + if (videoWidth > 0 && videoHeight > 0) { + client.onVideoResolution(videoWidth, videoHeight); + } + WfbNGStats stats = lastStats; + if (stats != null) { + client.onWfbStats(stats); + } + } + + public void removeClient(Client client) { + clients.remove(client); + } + + /** Points the decoder at a new output surface. The link keeps running throughout. */ + public void attachSurface(Surface surface, int index) { + if (videoPlayer != null) { + videoPlayer.attachSurface(surface, index); + } + } + + public void detachSurface(int index) { + if (videoPlayer != null) { + videoPlayer.detachSurface(index); + } + } + + /** Brings the adapter up if it is not already, using the persisted channel/bandwidth. */ + public void ensureAdapters() { + if (wfbLinkManager == null) { + return; + } + wfbLinkManager.setChannel(VideoActivity.getChannel(this)); + wfbLinkManager.setBandwidth(VideoActivity.getBandwidth(this)); + wfbLinkManager.refreshAdapters(); + } + + public boolean hasActiveAdapter() { + return wfbLinkManager != null && wfbLinkManager.hasActiveAdapter(); + } + + public void setChannel(int channel) { + if (wfbLinkManager == null) { + return; + } + wfbLinkManager.stopAdapters(); + wfbLinkManager.setChannel(channel); + wfbLinkManager.startAdapters(); + } + + public void setBandwidth(int bandwidth) { + if (wfbLinkManager == null) { + return; + } + wfbLinkManager.stopAdapters(); + wfbLinkManager.setBandwidth(bandwidth); + wfbLinkManager.startAdapters(); + } + + public void refreshKey() { + if (wfbLinkManager != null) { + wfbLinkManager.refreshKey(); + } + } + + /** For the settings menus that drive the native link options directly. */ + @Nullable + public WfbNgLink link() { + return wfbLink; + } + + public void startDvr(int fd, boolean fragmentedMp4) { + if (videoPlayer != null) { + videoPlayer.startDvr(fd, fragmentedMp4); + } + } + + public void stopDvr() { + if (videoPlayer != null) { + videoPlayer.stopDvr(); + } + } + + public void setUdpForwarding(String ip, int port, boolean enabled) { + if (videoPlayer != null) { + videoPlayer.setUdpForwarding(ip, port, enabled); + } + } + + public void setLowLatency(boolean enabled) { + if (videoPlayer != null) { + videoPlayer.setLowLatency(enabled); + } + } + + public int videoWidth() { + return videoWidth; + } + + public int videoHeight() { + return videoHeight; + } + + // ------------------------------------------------------------------------------ + // fan-out to the attached activity + // ------------------------------------------------------------------------------ + + @Override + public void showLinkMessage(String message) { + Log.i(TAG, "link: " + message); + lastStatus = message; + main.post(() -> { + for (Client c : clients) { + c.onLinkStatus(message); + } + }); + } + + @Override + public void showLocalStreamHint(String url) { + main.post(() -> { + for (Client c : clients) { + c.onLocalStreamHint(url); + } + }); + } + + @Override + public void onVideoRatioChanged(int width, int height) { + videoWidth = width; + videoHeight = height; + main.post(() -> { + for (Client c : clients) { + c.onVideoResolution(width, height); + } + }); + } + + @Override + public void onDecodingInfoChanged(DecodingInfo decodingInfo) { + main.post(() -> { + for (Client c : clients) { + c.onDecodingInfo(decodingInfo); + } + }); + } + + @Override + public void onWfbNgStatsChanged(WfbNGStats stats) { + lastStats = stats; + main.post(() -> { + for (Client c : clients) { + c.onWfbStats(stats); + } + }); + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index 06e270ae..72744021 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -61,11 +61,7 @@ import com.openipc.pixelpilot.osd.OSDElement; import com.openipc.pixelpilot.osd.OSDManager; import com.openipc.videonative.DecodingInfo; -import com.openipc.videonative.IVideoParamsChanged; -import com.openipc.videonative.VideoPlayer; import com.openipc.wfbngrtl8812.WfbNGStats; -import com.openipc.wfbngrtl8812.WfbNGStatsChanged; -import com.openipc.wfbngrtl8812.WfbNgLink; import com.openipc.xr.XrGoggleSession; import java.io.BufferedReader; @@ -94,6 +90,7 @@ import android.os.SystemClock; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import android.view.SurfaceHolder; import android.view.TextureView; import android.graphics.SurfaceTexture; import android.view.Surface; @@ -102,8 +99,8 @@ // Most basic implementation of an activity that uses VideoNative to stream a video // Into an Android Surface View -public class VideoActivity extends AppCompatActivity implements IVideoParamsChanged, - WfbNGStatsChanged, MavlinkUpdate, SettingsChanged, LinkStatusView { +public class VideoActivity extends LinkClientActivity + implements MavlinkUpdate, SettingsChanged { private static final String TAG = "pixelpilot"; private static final int PICK_KEY_REQUEST_CODE = 1; private static final int PICK_DVR_REQUEST_CODE = 2; @@ -125,9 +122,7 @@ public void run() { }; protected DecodingInfo mDecodingInfo; int lastVideoW = 0, lastVideoH = 0, lastCodec = 1; - WfbLinkManager wfbLinkManager; BroadcastReceiver batteryReceiver; - VideoPlayer videoPlayer; private ActivityVideoBinding binding; private OSDManager osdManager; private ParcelFileDescriptor dvrFd = null; @@ -140,7 +135,6 @@ public void run() { private boolean videoUsesTextureView = false; private ConstraintLayout constraintLayout; private ConstraintSet constraintSet; - private WfbNgLink wfbLink; private ObjectDetectorHelper objectDetectorHelper; private ExecutorService objectDetectionExecutor; @@ -368,12 +362,13 @@ private void initializeUI() { * Initializes WFB-NG related logic such as setting default gs.key and linking * to WFB-NG stats changes. */ + /** + * The adapter, the link and the decoder live in {@link LinkService}; this only makes sure + * the key it needs exists before the service constructs the native side. + */ private void initializeWfbNg() { setDefaultGsKey(); copyGSKey(); - wfbLink = new WfbNgLink(this); - wfbLink.SetWfbNGStatsChanged(this); - wfbLinkManager = new WfbLinkManager(this, this, wfbLink); } // ---------------------------------------------------------------------------- @@ -384,10 +379,6 @@ private void initializeWfbNg() { * Initializes VideoPlayer and configures surfaces for VR or standard mode. */ private void initializeVideoPlayers() { - videoPlayer = new VideoPlayer(this); - videoPlayer.setIVideoParamsChanged(this); - videoPlayer.setLowLatency(getLowLatencySetting(this)); - isVRMode = getVRSetting(); if (isVRMode) { @@ -403,8 +394,8 @@ private void initializeVideoPlayers() { private void setupVRVideoPlayers() { binding.mainVideo.setVisibility(View.GONE); binding.mainVideoSurface.setVisibility(View.GONE); - binding.surfaceViewLeft.getHolder().addCallback(videoPlayer.configure1(0)); - binding.surfaceViewRight.getHolder().addCallback(videoPlayer.configure1(1)); + binding.surfaceViewLeft.getHolder().addCallback(surfaceCallback(0)); + binding.surfaceViewRight.getHolder().addCallback(surfaceCallback(1)); } /** @@ -423,14 +414,68 @@ private void setupStandardVideoPlayer() { if (videoUsesTextureView) { binding.mainVideoSurface.setVisibility(View.GONE); binding.mainVideo.setVisibility(View.VISIBLE); - binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0)); + binding.mainVideo.setSurfaceTextureListener(surfaceTextureListener(0)); } else { binding.mainVideo.setVisibility(View.GONE); binding.mainVideoSurface.setVisibility(View.VISIBLE); - binding.mainVideoSurface.getHolder().addCallback(videoPlayer.configure1(0)); + binding.mainVideoSurface.getHolder().addCallback(surfaceCallback(0)); } } + /** + * Hands a Surface to the service and takes it back again. Only the decoder's output + * changes; reception and the adapter are untouched, which is what makes switching modes + * and surviving a pause free of a reconnect. + */ + private SurfaceHolder.Callback surfaceCallback(final int index) { + return new SurfaceHolder.Callback() { + @Override + public void surfaceCreated(SurfaceHolder holder) { + if (link != null) { + link.attachSurface(holder.getSurface(), index); + } + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + if (link != null) { + link.detachSurface(index); + } + } + }; + } + + private TextureView.SurfaceTextureListener surfaceTextureListener(final int index) { + return new TextureView.SurfaceTextureListener() { + @Override + public void onSurfaceTextureAvailable(SurfaceTexture texture, int width, int height) { + if (link != null) { + link.attachSurface(new Surface(texture), index); + } + } + + @Override + public void onSurfaceTextureSizeChanged(SurfaceTexture texture, int width, int height) { + } + + @Override + public boolean onSurfaceTextureDestroyed(SurfaceTexture texture) { + if (link != null) { + link.detachSurface(index); + } + return true; + } + + @Override + public void onSurfaceTextureUpdated(SurfaceTexture texture) { + } + }; + } + // ---------------------------------------------------------------------------- // VR SEEK BARS (only in VR mode) // ---------------------------------------------------------------------------- @@ -815,7 +860,7 @@ private void setupVideoSubMenu(PopupMenu popup) { item.setChecked(enabled); getSharedPreferences("general", MODE_PRIVATE).edit() .putBoolean("low_latency_decoder", enabled).apply(); - videoPlayer.setLowLatency(enabled); + if (link != null) link.setLowLatency(enabled); Toast.makeText(this, "Low latency " + (enabled ? "enabled" : "disabled") + ", applies on next video start.", Toast.LENGTH_SHORT).show(); item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW); @@ -890,8 +935,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) { SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit(); editor.putBoolean("adaptive_link_enabled", newState); editor.apply(); - // Call instance method on the WfbNgLink instance via the wfbLinkManager. - wfbLink.nativeSetAdaptiveLinkEnabled(newState); + if (link != null && link.link() != null) link.link().nativeSetAdaptiveLinkEnabled(newState); return true; }); @@ -913,8 +957,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) { SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit(); editor.putInt("adaptive_tx_power", power); editor.apply(); - // Call instance method on the WfbNgLink instance via the wfbLinkManager. - wfbLink.nativeSetTxPower(power); + if (link != null && link.link() != null) link.link().nativeSetTxPower(power); return true; }); } @@ -931,8 +974,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) { SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit(); editor.putBoolean("custom_fec_enabled", newState); editor.apply(); - // Call instance method on the WfbNgLink instance via the wfbLinkManager. - wfbLink.nativeSetUseFec(newState ? 1 : 0); + if (link != null && link.link() != null) link.link().nativeSetUseFec(newState ? 1 : 0); return true; }); @@ -947,7 +989,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) { SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit(); editor.putBoolean("custom_ldpc_enabled", newState); editor.apply(); - wfbLink.nativeSetUseLdpc(newState ? 1 : 0); + if (link != null && link.link() != null) link.link().nativeSetUseLdpc(newState ? 1 : 0); return true; }); @@ -962,7 +1004,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) { SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit(); editor.putBoolean("custom_stbc_enabled", newState); editor.apply(); - wfbLink.nativeSetUseStbc(newState ? 1 : 0); + if (link != null && link.link() != null) link.link().nativeSetUseStbc(newState ? 1 : 0); return true; }); @@ -1031,13 +1073,18 @@ private void showFecThresholdsDialog() { .show(); } + /** The service applies these when it constructs the link; this re-applies after edits. */ void initDefaultOptions() { - WfbOptions.applyDefaults(this, wfbLink); + if (link != null) { + WfbOptions.applyDefaults(this, link.link()); + } } // Read FEC thresholds from prefs and call native method to apply private void setFecThresholdsFromPrefs() { - WfbOptions.applyFecThresholds(this, wfbLink); + if (link != null) { + WfbOptions.applyFecThresholds(this, link.link()); + } } /** @@ -1173,8 +1220,8 @@ private void updateUdpForwardingState() { String ip = prefs.getString("forward_udp_ip", "192.168.1.100"); int port = prefs.getInt("forward_udp_port", 5600); - if (videoPlayer != null) { - videoPlayer.setUdpForwarding(ip, port, enabled); + if (link != null) { + link.setUdpForwarding(ip, port, enabled); } } @@ -1357,10 +1404,7 @@ private void startStopDvr() { if (dvrUri != null) { startDvr(dvrUri); } else { - wfbLinkManager.stopAdapters(); - videoPlayer.stop(); - videoPlayer.stopAudio(); - + // No need to tear the link down for a file picker any more. Intent intent = new Intent(Intent.ACTION_OPEN_DOCUMENT_TREE); intent.addCategory(Intent.CATEGORY_DEFAULT); intent.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION | @@ -1379,7 +1423,7 @@ private void startDvr(Uri dvrUri) { } try { dvrFd = getContentResolver().openFileDescriptor(dvrUri, "rw"); - videoPlayer.startDvr(dvrFd.getFd(), getDvrMP4()); + if (link != null) link.startDvr(dvrFd.getFd(), getDvrMP4()); binding.imgBtnRecord.setImageResource(R.drawable.recording); } catch (IOException e) { Log.e(TAG, "Failed to open dvr file ", e); @@ -1416,7 +1460,7 @@ private void stopDvr() { } binding.imgRecIndicator.setVisibility(View.INVISIBLE); binding.imgBtnRecord.setImageResource(R.drawable.record); - videoPlayer.stopDvr(); + if (link != null) link.stopDvr(); if (recordTimer != null) { recordTimer.cancel(); recordTimer.purge(); @@ -1448,7 +1492,7 @@ protected void onActivityResult(int requestCode, int resultCode, @Nullable Inten InputStream inputStream = getContentResolver().openInputStream(uri); setGsKey(inputStream); copyGSKey(); - wfbLinkManager.refreshKey(); + if (link != null) link.refreshKey(); inputStream.close(); } catch (IOException e) { Log.e(TAG, "Failed to import gs.key from " + uri); @@ -1517,27 +1561,17 @@ public void setDvrMP4(boolean enabled) { } @SuppressLint("UnspecifiedRegisterReceiverFlag") + /** Only the battery overlay: the USB receiver belongs to LinkService with the adapter. */ public void registerReceivers() { - IntentFilter usbFilter = new IntentFilter(); - usbFilter.addAction(UsbManager.ACTION_USB_DEVICE_ATTACHED); - usbFilter.addAction(UsbManager.ACTION_USB_DEVICE_DETACHED); - usbFilter.addAction(WfbLinkManager.ACTION_USB_PERMISSION); IntentFilter batFilter = new IntentFilter(Intent.ACTION_BATTERY_CHANGED); - if (Build.VERSION.SDK_INT >= 33) { - registerReceiver(wfbLinkManager, usbFilter, Context.RECEIVER_NOT_EXPORTED); registerReceiver(batteryReceiver, batFilter, Context.RECEIVER_NOT_EXPORTED); } else { - registerReceiver(wfbLinkManager, usbFilter); registerReceiver(batteryReceiver, batFilter); } } public void unregisterReceivers() { - try { - unregisterReceiver(wfbLinkManager); - } catch (IllegalArgumentException ignored) { - } try { unregisterReceiver(batteryReceiver); } catch (IllegalArgumentException ignored) { @@ -1549,18 +1583,11 @@ protected void onPause() { super.onPause(); stopObjectDetectionLoop(); - unregisterReceivers(); - videoPlayer.stop(); - videoPlayer.stopAudio(); - wfbLinkManager.stopAdapters(); - - // Stop VPN service - Log.w(TAG, "onPause: stopping service"); - Intent intent = new Intent(this, WfbNgVpnService.class); - intent.setAction("STOP_SERVICE"); - startService(intent); + // The link deliberately stays up: it belongs to LinkService now. Tearing it down + // here is what used to cost seconds of black screen every time a system menu or a + // dialog took focus. } @Override @@ -1574,9 +1601,6 @@ protected void onStop() { MavlinkNative.nativeStop(this); handler.removeCallbacks(runnable); unregisterReceivers(); - wfbLinkManager.stopAdapters(); - videoPlayer.stop(); - videoPlayer.stopAudio(); super.onStop(); } @@ -1584,16 +1608,11 @@ protected void onStop() { protected void onResume() { registerReceivers(); - wfbLinkManager.setChannel(getChannel(this)); - wfbLinkManager.setBandwidth(getBandwidth(this)); - - // On resume is called when the app is reopened, a device might have been plugged since the last time it started. - wfbLinkManager.refreshAdapters(); - - wfbLinkManager.startAdapters(); - videoPlayer.start(); - updateUdpForwardingState(); - videoPlayer.startAudio(); + // An adapter may have been plugged in while we were away. + if (link != null) { + link.ensureAdapters(); + updateUdpForwardingState(); + } SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); boolean odEnabled = prefs.getBoolean("od_enabled", false); @@ -1606,6 +1625,12 @@ protected void onResume() { super.onResume(); } + @Override + protected void onLinkServiceConnected(LinkService service) { + service.ensureAdapters(); + updateUdpForwardingState(); + } + @Override public void onChannelSettingChanged(int channel) { int currentChannel = getChannel(this); @@ -1616,9 +1641,9 @@ public void onChannelSettingChanged(int channel) { SharedPreferences.Editor editor = prefs.edit(); editor.putInt("wifi-channel", channel); editor.apply(); - wfbLinkManager.stopAdapters(); - wfbLinkManager.setChannel(channel); - wfbLinkManager.startAdapters(); + if (link != null) { + link.setChannel(channel); + } } @Override @@ -1631,13 +1656,13 @@ public void onBandwidthSettingChanged(int bandwidth) { SharedPreferences.Editor editor = prefs.edit(); editor.putInt("bandwidth", bandwidth); editor.apply(); - wfbLinkManager.stopAdapters(); - wfbLinkManager.setBandwidth(bandwidth); - wfbLinkManager.startAdapters(); + if (link != null) { + link.setBandwidth(bandwidth); + } } @Override - public void onVideoRatioChanged(final int videoW, final int videoH) { + public void onVideoResolution(final int videoW, final int videoH) { lastVideoW = videoW; lastVideoH = videoH; @@ -1666,7 +1691,7 @@ private void updateViewRatio(int viewId, int videoW, int videoH) { } @Override - public void onDecodingInfoChanged(final DecodingInfo decodingInfo) { + public void onDecodingInfo(final DecodingInfo decodingInfo) { mDecodingInfo = decodingInfo; runOnUiThread(() -> { if (lastCodec != decodingInfo.nCodec) { @@ -1687,7 +1712,7 @@ public void onDecodingInfoChanged(final DecodingInfo decodingInfo) { } @Override - public void onWfbNgStatsChanged(WfbNGStats data) { + public void onWfbStats(WfbNGStats data) { runOnUiThread(() -> { if (data.count_p_all > 0) { binding.tvMessage.setVisibility(View.INVISIBLE); @@ -1754,10 +1779,7 @@ public void onWfbNgStatsChanged(WfbNGStats data) { } @Override - public void showLinkMessage(String message) { - // Also to the log: this used to only reach the overlay, so a bug report had no - // record of what the app actually said about the link. - Log.i(TAG, "link: " + message); + public void onLinkStatus(String message) { runOnUiThread(() -> { binding.tvMessage.setVisibility(View.VISIBLE); binding.tvMessage.setText(message); @@ -1765,7 +1787,7 @@ public void showLinkMessage(String message) { } @Override - public void showLocalStreamHint(String url) { + public void onLocalStreamHint(String url) { runOnUiThread(() -> { binding.wifiMessage.setText(url); binding.wifiMessage.setVisibility(View.VISIBLE); diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index 2caefec6..800200cf 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -8,7 +8,6 @@ import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Paint; -import android.hardware.usb.UsbDevice; import android.hardware.usb.UsbManager; import android.net.Uri; import android.os.Build; @@ -25,18 +24,11 @@ import android.widget.Toast; import androidx.annotation.Nullable; -import androidx.appcompat.app.AppCompatActivity; -import com.openipc.videonative.DecodingInfo; -import com.openipc.videonative.IVideoParamsChanged; -import com.openipc.videonative.VideoPlayer; import com.openipc.wfbngrtl8812.WfbNGStats; -import com.openipc.wfbngrtl8812.WfbNGStatsChanged; -import com.openipc.wfbngrtl8812.WfbNgLink; import com.openipc.xr.XrGoggleSession; import java.io.IOException; -import java.util.Map; /** * Immersive ground station. @@ -47,12 +39,16 @@ * layer. That removes the view hierarchy, SurfaceFlinger and the panel's fixed resolution * from the video path. * + *

The adapter, the wfb-ng link and the decoder belong to {@link LinkService}, not to this + * activity. Entering and leaving immersive mode therefore only swaps the decoder's output + * surface - the adapter is never reopened, so there is no reconnect delay and no second USB + * permission prompt. + * *

There is no in-headset menu yet: channel, bandwidth and the DVR folder are still * configured in {@link VideoActivity} before putting the headset on. Controllers cover * what matters in flight. */ -public class XrVideoActivity extends AppCompatActivity - implements XrGoggleSession.Listener, IVideoParamsChanged, WfbNGStatsChanged, LinkStatusView { +public class XrVideoActivity extends LinkClientActivity implements XrGoggleSession.Listener { private static final String TAG = "pixelpilot-xr"; @@ -82,9 +78,6 @@ static boolean isActive() { private final Handler handler = new Handler(Looper.getMainLooper()); private XrGoggleSession xr; - private VideoPlayer videoPlayer; - private WfbNgLink wfbLink; - private WfbLinkManager wfbLinkManager; private TextView statusView; // The flat activity may still be releasing the USB interface when we get here, so the @@ -143,22 +136,6 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { return; } - videoPlayer = new VideoPlayer(this); - videoPlayer.setIVideoParamsChanged(this); - videoPlayer.setLowLatency(VideoActivity.getLowLatencySetting(this)); - - // The native aggregators open files/gs.key in their constructor and throw if it - // is not there, so this has to happen before WfbNgLink is created. - if (!GsKey.ensure(this)) { - failToFlat("no usable gs.key"); - return; - } - wfbLink = new WfbNgLink(this); - wfbLink.SetWfbNGStatsChanged(this); - // Not read by the native side on its own: adaptive link, TX power, FEC/LDPC/STBC. - WfbOptions.applyDefaults(this, wfbLink); - wfbLinkManager = new WfbLinkManager(this, this, wfbLink); - xr = new XrGoggleSession(this, this); SharedPreferences prefs = prefs(); // Match the swapchain to the stream so the compositor never resamples twice. The @@ -173,8 +150,7 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { } @Override - protected void onResume() { - super.onResume(); + protected void onLinkServiceConnected(LinkService service) { if (xr == null) { return; } @@ -182,6 +158,14 @@ protected void onResume() { tryBringUpLink(); } + @Override + protected void onResume() { + super.onResume(); + if (xr != null && link != null) { + tryBringUpLink(); + } + } + // Deliberately no teardown in onPause/onStop. In an immersive session the runtime // pauses and resumes us for its own reasons (system menu, proximity sensor), and // dropping the USB link there costs seconds of black screen on the way back. @@ -197,17 +181,16 @@ protected void onDestroy() { // never registered } stopDvr(); + // Hand the decoder's surface back but leave the link running: the flat activity is + // about to take over, and the whole point of the service is that the adapter stays + // open across the switch. + if (link != null) { + link.detachSurface(0); + } if (xr != null) { xr.release(); xr = null; } - if (wfbLinkManager != null) { - wfbLinkManager.stopAdapters(); - } - if (videoPlayer != null) { - videoPlayer.stopAudio(); - videoPlayer.stop(); - } super.onDestroy(); } @@ -237,17 +220,10 @@ private void registerUsbReceiver() { * immersive session. */ private void tryBringUpLink() { - if (xr == null) { - return; - } - wfbLinkManager.setChannel(VideoActivity.getChannel(this)); - wfbLinkManager.setBandwidth(VideoActivity.getBandwidth(this)); - wfbLinkManager.refreshAdapters(); - - if (!adaptersReady()) { - // refreshAdapters() has asked for permission; the broadcast brings us back. + if (xr == null || link == null) { return; } + link.ensureAdapters(); // Going immersive does not depend on the link, and the pilot is better off seeing the // status screen with a reason than a flat activity that says nothing. @@ -257,31 +233,17 @@ private void tryBringUpLink() { xr.start(); } - if (!wfbLinkManager.hasActiveAdapter() && linkRetries < LINK_RETRY_LIMIT) { + // The adapter may still be waiting for a USB permission the service asked for, so + // keep checking back rather than giving up on the first look. + if (!link.hasActiveAdapter() && linkRetries < LINK_RETRY_LIMIT) { linkRetries++; Log.i(TAG, "adapter not up yet, retry " + linkRetries + "/" + LINK_RETRY_LIMIT); handler.postDelayed(this::tryBringUpLink, LINK_RETRY_INTERVAL_MS); - } else if (wfbLinkManager.hasActiveAdapter()) { + } else if (link.hasActiveAdapter()) { linkRetries = 0; } } - /** True when every attached compatible adapter is usable (or there is none at all). */ - private boolean adaptersReady() { - Map adapters = wfbLinkManager.getAttachedAdapters(); - if (adapters == null || adapters.isEmpty()) { - // Nothing attached: still go immersive, the stream may arrive over UDP. - return true; - } - UsbManager usbManager = (UsbManager) getSystemService(Context.USB_SERVICE); - for (UsbDevice device : adapters.values()) { - if (!usbManager.hasPermission(device)) { - return false; - } - } - return true; - } - @Override public void onXrReady(Surface videoSurface) { Log.i(TAG, "immersive session up, handing the swapchain surface to the decoder"); @@ -290,15 +252,17 @@ public void onXrReady(Surface videoSurface) { // an opaque black rectangle until the first frame arrives, which is // indistinguishable from "nothing is being composited at all". drawStatusScreen(); - // The decoder writes into the compositor swapchain from here on. - videoPlayer.addAndStartDecoderReceiver(videoSurface, 0); - videoPlayer.start(); - videoPlayer.startAudio(); + // The decoder writes into the compositor swapchain from here on. Only the surface + // changes - the receivers and the adapter keep running in the service. + if (link != null) { + link.attachSurface(videoSurface, 0); + if (link.videoWidth() > 0) { + xr.setVideoResolution(link.videoWidth(), link.videoHeight()); + } + } applyRefreshRatePreference(); statusView.setVisibility(View.GONE); linkStarted = true; - // Not startAdapters(): tryBringUpLink() -> refreshAdapters() already owns bringing the - // adapter up, and it retries. Two paths racing for one dongle is what broke this. } private void applyRefreshRatePreference() { @@ -494,9 +458,12 @@ private void toggleDvr() { Toast.makeText(this, "Could not create the recording file", Toast.LENGTH_LONG).show(); return; } + if (link == null) { + return; + } try { dvrFd = getContentResolver().openFileDescriptor(target, "rw"); - videoPlayer.startDvr(dvrFd.getFd(), DvrFiles.fragmentedMp4(this)); + link.startDvr(dvrFd.getFd(), DvrFiles.fragmentedMp4(this)); xr.haptic(0.7f, 120); Toast.makeText(this, "Recording", Toast.LENGTH_SHORT).show(); } catch (IOException e) { @@ -510,7 +477,9 @@ private void stopDvr() { if (dvrFd == null) { return; } - videoPlayer.stopDvr(); + if (link != null) { + link.stopDvr(); + } try { dvrFd.close(); } catch (IOException e) { @@ -524,7 +493,7 @@ private void stopDvr() { // ------------------------------------------------------------------------------ @Override - public void onVideoRatioChanged(int videoW, int videoH) { + public void onVideoResolution(int videoW, int videoH) { if (videoW <= 0 || videoH <= 0) { return; } @@ -538,12 +507,7 @@ public void onVideoRatioChanged(int videoW, int videoH) { } @Override - public void onDecodingInfoChanged(DecodingInfo decodingInfo) { - // Nothing to draw in immersive mode yet. - } - - @Override - public void onWfbNgStatsChanged(WfbNGStats data) { + public void onWfbStats(WfbNGStats data) { lastStats = data; // Without this there is no way to tell "the adapter is dead" from "the link is fine // but nothing decodes" while wearing the headset. @@ -577,12 +541,11 @@ public void onWfbNgStatsChanged(WfbNGStats data) { } // ------------------------------------------------------------------------------ - // LinkStatusView + // link service callbacks // ------------------------------------------------------------------------------ @Override - public void showLinkMessage(String message) { - Log.i(TAG, message); + public void onLinkStatus(String message) { lastStatus = message; runOnUiThread(() -> { if (statusView != null && !linkStarted) { @@ -592,7 +555,7 @@ public void showLinkMessage(String message) { } @Override - public void showLocalStreamHint(String url) { + public void onLocalStreamHint(String url) { Log.i(TAG, "listening on " + url); } } diff --git a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java index e1c49d0f..25a9c284 100644 --- a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java +++ b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java @@ -9,7 +9,6 @@ import android.view.SurfaceHolder; import androidx.annotation.Nullable; -import androidx.appcompat.app.AppCompatActivity; import java.util.Timer; import java.util.TimerTask; @@ -35,7 +34,7 @@ public class VideoPlayer implements IVideoParamsChanged { private Timer timer; // Setup as much as possible without creating the decoder - public VideoPlayer(final AppCompatActivity parent) { + public VideoPlayer(final Context parent) { this.context = parent; nativeVideoPlayer = nativeInitialize(context); } @@ -161,6 +160,21 @@ public void addAndStartDecoderReceiver(Surface surface, int index) { setVideoSurface(surface, index); } + /** + * Point the decoder at a new output surface without touching the receivers. + * + *

This is what a surface change should do: the decoder is rebuilt for the new surface + * while UDP reception, and with it the wifi link, keeps running. + */ + public void attachSurface(Surface surface, int index) { + setVideoSurface(surface, index); + } + + /** Releases the decoder's output surface, leaving the receivers alone. */ + public void detachSurface(int index) { + setVideoSurface(null, index); + } + /** * Stop the Receiver * Stop the Decoder diff --git a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java index ce8e3e6a..897f0b08 100644 --- a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java +++ b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java @@ -7,7 +7,6 @@ import android.util.Log; import androidx.annotation.Keep; -import androidx.appcompat.app.AppCompatActivity; import java.util.HashMap; import java.util.Map; @@ -75,7 +74,7 @@ private static synchronized void release(String deviceName, Thread t) { public static native void nativeSetUseLdpc(long nativeInstance, int use); public static native void nativeSetUseStbc(long nativeInstance, int use); - public WfbNgLink(final AppCompatActivity parent) { + public WfbNgLink(final Context parent) { this.context = parent; nativeWfbngLink = nativeInitialize(context); timer = new Timer(); From 4493c300ad43f056a71520be2f41aac15027cb69 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:15:32 +0200 Subject: [PATCH 19/23] Fix: the surface was produced before the service had bound Smoke test of the service refactor: the link came up and packets arrived, but the decoder logged "inputPipeClosed." on every NALU - it had no output surface. bindService() is asynchronous and the view hierarchy does not wait for it, so SurfaceView.surfaceCreated() fires before onServiceConnected(). The callback found link == null and silently dropped the surface, and nothing handed it over afterwards. The surface is now parked and attached either immediately or as soon as the service connects, whichever comes second. The immersive activity is unaffected - its surface comes from the XR session, which is only started from onLinkServiceConnected(). --- .../com/openipc/pixelpilot/VideoActivity.java | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index 72744021..ee01cde5 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -133,6 +133,9 @@ public void run() { // Which view the main video is rendered into. SurfaceView is the low latency // default; the TextureView is only used when object detection needs getBitmap(). private boolean videoUsesTextureView = false; + // A Surface can become available before the service has bound - binding is async and the + // view hierarchy does not wait for it - so it is parked here until the link exists. + private final Surface[] pendingSurfaces = new Surface[2]; private ConstraintLayout constraintLayout; private ConstraintSet constraintSet; @@ -431,9 +434,7 @@ private SurfaceHolder.Callback surfaceCallback(final int index) { return new SurfaceHolder.Callback() { @Override public void surfaceCreated(SurfaceHolder holder) { - if (link != null) { - link.attachSurface(holder.getSurface(), index); - } + provideSurface(holder.getSurface(), index); } @Override @@ -442,20 +443,30 @@ public void surfaceChanged(SurfaceHolder holder, int format, int width, int heig @Override public void surfaceDestroyed(SurfaceHolder holder) { - if (link != null) { - link.detachSurface(index); - } + withdrawSurface(index); } }; } + private void provideSurface(Surface surface, int index) { + pendingSurfaces[index] = surface; + if (link != null) { + link.attachSurface(surface, index); + } + } + + private void withdrawSurface(int index) { + pendingSurfaces[index] = null; + if (link != null) { + link.detachSurface(index); + } + } + private TextureView.SurfaceTextureListener surfaceTextureListener(final int index) { return new TextureView.SurfaceTextureListener() { @Override public void onSurfaceTextureAvailable(SurfaceTexture texture, int width, int height) { - if (link != null) { - link.attachSurface(new Surface(texture), index); - } + provideSurface(new Surface(texture), index); } @Override @@ -464,9 +475,7 @@ public void onSurfaceTextureSizeChanged(SurfaceTexture texture, int width, int h @Override public boolean onSurfaceTextureDestroyed(SurfaceTexture texture) { - if (link != null) { - link.detachSurface(index); - } + withdrawSurface(index); return true; } @@ -1627,6 +1636,12 @@ protected void onResume() { @Override protected void onLinkServiceConnected(LinkService service) { + // Whatever the view hierarchy produced while we were still binding. + for (int i = 0; i < pendingSurfaces.length; i++) { + if (pendingSurfaces[i] != null) { + service.attachSurface(pendingSurfaces[i], i); + } + } service.ensureAdapters(); updateUdpForwardingState(); } From 5879741ca25188b649e4ea4ab2c524461eec928b Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:19:37 +0200 Subject: [PATCH 20/23] Add a way out of immersive mode There was none: every controller button and both pinch gestures were already bound, so the only way back to the flat view was launching it from the launcher and letting ModeOwner evict the session. - left menu button, the conventional app button and the last one still free - right thumb swipe left, so it also works without controllers, next to the swipe forward/backward that moves the panel A single gesture rather than a held one is fine here because leaving is now cheap: the link lives in LinkService, so switching back to the flat view keeps the adapter open and the video running - a stray trigger costs a mode switch, not a reconnect. The status screen also lists the controls now. It is the only surface the immersive mode has, and with everything on unlabelled buttons there was no way to discover any of them. --- .../openipc/pixelpilot/XrVideoActivity.java | 21 +++++++++++++++++++ app/xr/src/main/cpp/XrGoggleSession.cpp | 8 +++++++ app/xr/src/main/cpp/XrGoggleSession.h | 2 ++ .../java/com/openipc/xr/XrGoggleSession.java | 1 + 4 files changed, 32 insertions(+) diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index 800200cf..8b40185a 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -294,6 +294,9 @@ public void onXrButton(int button) { case XrGoggleSession.BUTTON_RECORD: toggleDvr(); break; + case XrGoggleSession.BUTTON_EXIT: + leaveVr(); + break; case XrGoggleSession.BUTTON_RAISE: case XrGoggleSession.BUTTON_LOWER: prefs().edit().putFloat(PREF_QUAD_HEIGHT, xr.quadHeightOffset()).apply(); @@ -313,6 +316,16 @@ public void onXrStopped(@Nullable String error) { } } + /** + * Back to the flat activity. Cheap now that the link lives in the service: the adapter + * stays open and video keeps running, only the presentation changes. + */ + private void leaveVr() { + Log.i(TAG, "leaving immersive mode on request"); + startActivity(new Intent(this, VideoActivity.class)); + finish(); + } + /** Immersive mode is not usable here - say why and go back to the flat activity. */ private void failToFlat(String reason) { Toast.makeText(this, "VR mode unavailable: " + reason, Toast.LENGTH_LONG).show(); @@ -433,6 +446,14 @@ private void paintStatus(Canvas canvas) { y += h / 20f; } canvas.drawText(xr != null && xr.isHeadLocked() ? "head locked" : "world locked", cx, y, body); + y += h / 20f; + + Paint hint = new Paint(body); + hint.setTextSize(h / 36f); + hint.setColor(Color.rgb(130, 140, 155)); + canvas.drawText("A recenter B passthrough stick click record", cx, y, hint); + y += h / 26f; + canvas.drawText("menu button or thumb swipe left: back to flat mode", cx, y, hint); } // ------------------------------------------------------------------------------ diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp index c0b892ce..d484c928 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.cpp +++ b/app/xr/src/main/cpp/XrGoggleSession.cpp @@ -683,6 +683,7 @@ bool XrGoggleSession::createActions() // button should still act the moment it is pressed. mActionHandRecenter = boolAction("hand_recenter", "Recenter view (hold)"); mActionHandPassthrough = boolAction("hand_passthrough", "Toggle passthrough (hold)"); + mActionExit = boolAction("exit", "Leave VR"); XrActionCreateInfo stick{XR_TYPE_ACTION_CREATE_INFO}; stick.actionType = XR_ACTION_TYPE_VECTOR2F_INPUT; @@ -745,6 +746,7 @@ bool XrGoggleSession::createActions() {mActionNearer, path("/user/hand/left/input/trigger/value")}, {mActionFarther, path("/user/hand/right/input/squeeze/value")}, {mActionFarther, path("/user/hand/left/input/squeeze/value")}, + {mActionExit, path("/user/hand/left/input/menu/click")}, {mActionHaptic, path("/user/hand/left/output/haptic")}, {mActionHaptic, path("/user/hand/right/output/haptic")}, }; @@ -795,6 +797,7 @@ bool XrGoggleSession::createActions() {mActionRaise, path("/user/hand/right/input/swipe_forward_meta")}); handBindings.push_back( {mActionLower, path("/user/hand/right/input/swipe_backward_meta")}); + handBindings.push_back({mActionExit, path("/user/hand/right/input/swipe_left_meta")}); } XrInteractionProfileSuggestedBinding hands{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING}; @@ -1228,6 +1231,7 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) {mActionLockMode, BUTTON_LOCK_MODE}, {mActionRaise, BUTTON_RAISE}, {mActionLower, BUTTON_LOWER}, + {mActionExit, BUTTON_EXIT}, }; for (const auto& b : buttons) @@ -1264,6 +1268,10 @@ void XrGoggleSession::syncActions(JNIEnv* env, jobject listener) setQuadHeightOffset(mQuadHeightOffset.load() - kHeightStep); requestHaptic(0.25f, 20); break; + case BUTTON_EXIT: + // Handled on the Java side; just confirm it was registered. + requestHaptic(0.6f, 80); + break; default: break; } diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h index c58e5683..8c18a24a 100644 --- a/app/xr/src/main/cpp/XrGoggleSession.h +++ b/app/xr/src/main/cpp/XrGoggleSession.h @@ -36,6 +36,7 @@ class XrGoggleSession BUTTON_LOCK_MODE = 3, BUTTON_RAISE = 4, BUTTON_LOWER = 5, + BUTTON_EXIT = 6, }; XrGoggleSession() = default; @@ -144,6 +145,7 @@ class XrGoggleSession XrAction mActionHandPassthrough = XR_NULL_HANDLE; XrAction mActionRaise = XR_NULL_HANDLE; XrAction mActionLower = XR_NULL_HANDLE; + XrAction mActionExit = XR_NULL_HANDLE; XrAction mActionHaptic = XR_NULL_HANDLE; // --- EGL ---------------------------------------------------------------------- diff --git a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java index 6d8ddf85..d78267ea 100644 --- a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java +++ b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java @@ -31,6 +31,7 @@ public final class XrGoggleSession { public static final int BUTTON_LOCK_MODE = 3; public static final int BUTTON_RAISE = 4; public static final int BUTTON_LOWER = 5; + public static final int BUTTON_EXIT = 6; private static final String TAG = "pixelpilot-xr"; From faed15ffc05518b6442ebd0a7c2b9c0d2bde27ab Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:01:50 +0200 Subject: [PATCH 21/23] Bound how long the packet queue holds a frame back BufferedPacketQueue reorders the RTP stream before it reaches the parser. When a packet is missing it holds everything behind the gap, which is right for a reorder and wrong for a loss - and past wfb-ng, on a loopback socket, a gap is almost always a loss that FEC could not recover. Three things made that wait longer than it needs to be. The monotonic-increase escape hatch, which exists precisely to notice "the sequence numbers keep climbing but the gap is not filling", never fires. calculateDistance(a, b) is how far b is ahead of a - seqLessThan reads it that way - but the call site asks for the distance from the incoming packet to the last delivered one. For a gap ahead of us that is negative, so the else branch clears the counter on every single packet. Fixing the argument order alone is not enough: the counter is only incremented while abs(dist) < MONOTONIC_THRESHOLD, and dist grows by one with every packet held back. Compared against the same constant that gates it, the counter tops out at three and can never reach five. The gate is now its own constant, so a run past a gap releases the buffer after five packets instead of never. That leaves MAX_BUFFER_SIZE as the only bound, and how much latency fifteen packets is depends entirely on the packet rate: about 20ms on a 1080p video stream, but roughly 300ms on the audio stream, which runs at a fraction of it. The buffer is now also bounded in time, so the worst case is the same on both. Twenty milliseconds is a little over one frame at 60fps and several orders of magnitude more than a loopback socket needs to reorder anything. Nothing is dropped by any of this - the buffered packets are still delivered, just without waiting on one that is not coming. The test target could not be built on a case-sensitive filesystem, since CMakeLists.txt spelled the source BufferedPacketqueue_test.cpp. The existing two cases both turn out to feed a strictly in-order stream, so neither reached the buffer at all; the new ones cover a permanent gap, the age bound, a reorder that resolves inside it, and a jump too large to be one. They drive the clock themselves rather than reading it, so the age bound cannot make them flaky on a loaded machine. --- .../src/main/cpp/BufferedPacketQueue.h | 94 +++++++++++++++++-- .../cpp/tests/BufferedPacketQueue_test.cpp | 75 ++++++++++++++- .../src/main/cpp/tests/CMakeLists.txt | 2 +- 3 files changed, 159 insertions(+), 12 deletions(-) diff --git a/app/videonative/src/main/cpp/BufferedPacketQueue.h b/app/videonative/src/main/cpp/BufferedPacketQueue.h index 88d6eab0..5efcc9d1 100644 --- a/app/videonative/src/main/cpp/BufferedPacketQueue.h +++ b/app/videonative/src/main/cpp/BufferedPacketQueue.h @@ -5,9 +5,11 @@ #include #endif #include +#include #include #include #include +#include #include #include @@ -15,10 +17,27 @@ // Define logging tag and maximum buffer size #define BUFFERED_QUEUE_LOG_TAG "BufferedPacketQueue" -// Considering the packet rate about 100 packets per second, 10 packets should be enough +// Last-resort cap on the buffer. How long this is in wall clock time depends entirely on the +// packet rate, which is why it cannot be the only bound - see MAX_BUFFER_AGE. constexpr size_t MAX_BUFFER_SIZE = 15; // Number of monotonically increasing packets constexpr size_t MONOTONIC_THRESHOLD = 5; +// A monotonic run is only tracked while the gap is plausibly a reorder. Kept separate from +// MONOTONIC_THRESHOLD: the gap grows by one with every packet held back, so gating the counter +// on the same value it is compared against means it can never reach it. +constexpr size_t MONOTONIC_MAX_DISTANCE = 64; + +// Clock used to bound how long a packet may be held back. +using QueueClock = std::chrono::steady_clock; +using QueueTimePoint = QueueClock::time_point; + +// A gap in the sequence numbers on this path is almost always a packet that FEC could not +// recover, not a reorder: by the time packets get here they have come through wfb-ng and a +// loopback socket, where reordering takes microseconds. Waiting for a packet that will never +// arrive is pure added latency, so the wait is bounded in time rather than in packets - the +// packet bound alone is worth ~20ms on a 1080p video stream but ~300ms on the audio stream, +// which runs at a fraction of the packet rate. +constexpr auto MAX_BUFFER_AGE = std::chrono::milliseconds(20); // Type definition for sequence numbers using SeqType = uint16_t; @@ -45,7 +64,12 @@ class BufferedPacketQueue * @param callback Callable to handle processed packets. */ template - void processPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length, Callback& callback) + void processPacket( + SeqType currPacketIdx, + const uint8_t* data, + std::size_t data_length, + Callback& callback, + QueueTimePoint now = QueueClock::now()) { logDebug( "Processing packet with Sequence=%u, lastPacketIdx=%u, firstPacket=%s", @@ -53,6 +77,19 @@ class BufferedPacketQueue mLastPacketIdx, mFirstPacket ? "true" : "false"); + // Before anything else: give up on a gap we have been waiting on for too long. Done + // here rather than in handleOutOfOrderPacket so that an in-order packet arriving after + // a stall does not get delivered ahead of what is already buffered. + if (!mPackets.empty() && (now - mOldestBufferedAt) >= MAX_BUFFER_AGE) + { + logWarning( + "Held %zu packet(s) for more than %lldms waiting on Sequence=%u. Flushing.", + mPackets.size(), + (long long) MAX_BUFFER_AGE.count(), + static_cast(static_cast(mLastPacketIdx + 1))); + mLastPacketIdx = drainBufferInOrder(callback); + } + if (isFirstPacket(currPacketIdx)) { handleFirstPacket(currPacketIdx); @@ -73,7 +110,7 @@ class BufferedPacketQueue else { // Out-of-order packet - handleOutOfOrderPacket(currPacketIdx, data, data_length, callback); + handleOutOfOrderPacket(currPacketIdx, data, data_length, callback, now); } } @@ -83,6 +120,9 @@ class BufferedPacketQueue std::unordered_map> mPackets; + // When the current stall started, i.e. when mPackets last went from empty to non-empty. + QueueTimePoint mOldestBufferedAt{}; + // This variable is used to track a situation where the sequence number is increasing monotonically while packets // are out of order. if this counter reaches MONOTONIC_THRESHOLD, we will restart buffering and update lastPacketIdx // to the highest sequence index received. @@ -174,7 +214,12 @@ class BufferedPacketQueue * @param callback Callable to handle processed packets. */ template - void handleOutOfOrderPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length, Callback& callback) + void handleOutOfOrderPacket( + SeqType currPacketIdx, + const uint8_t* data, + std::size_t data_length, + Callback& callback, + QueueTimePoint now) { logDebug("Out-of-order packet detected. Sequence=%u", currPacketIdx); @@ -184,10 +229,15 @@ class BufferedPacketQueue // return; } - bufferPacket(currPacketIdx, data, data_length); + bufferPacket(currPacketIdx, data, data_length, now); - auto dist = calculateDistance(currPacketIdx, mLastPacketIdx); - if (std::abs(dist) < MONOTONIC_THRESHOLD) + // calculateDistance(a, b) is how far b is ahead of a - see seqLessThan below - so the + // question "is this packet ahead of the last one we delivered" has to be asked in that + // order. Reversed, dist is negative for exactly the case this heuristic exists for (a + // gap ahead of us), the else branch below clears the counter every time, and the + // buffer only ever drains on the MAX_BUFFER_SIZE cap. + auto dist = calculateDistance(mLastPacketIdx, currPacketIdx); + if (static_cast(std::abs(dist)) < MONOTONIC_MAX_DISTANCE) { // Check for monotonic increases if (dist > 0) @@ -231,8 +281,15 @@ class BufferedPacketQueue * @param data Pointer to the packet data. * @param data_length Size of the packet data. */ - void bufferPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length) + void bufferPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length, QueueTimePoint now) { + // Only the start of a stall is recorded, because the buffer is always drained as a + // whole. A partial drain leaves the mark where it was, which errs towards flushing + // early - the safe direction on a live link. + if (mPackets.empty()) + { + mOldestBufferedAt = now; + } mPackets[currPacketIdx] = std::vector(data, data + data_length); logDebug("Buffered out-of-order packet. Buffer size: %zu", mPackets.size()); } @@ -244,6 +301,19 @@ class BufferedPacketQueue */ template void restartBuffering(Callback& callback, SeqType currPacketIdx) + { + drainBufferInOrder(callback); + mLastPacketIdx = currPacketIdx; + } + + /** + * @brief Delivers everything currently held back, in sequence order, and empties the buffer. + * @tparam Callback A callable type that processes the packet data. + * @param callback Callable to handle processed packets. + * @return The highest sequence number delivered, or mLastPacketIdx if nothing was held. + */ + template + SeqType drainBufferInOrder(Callback& callback) { // Process as many in-order buffered packets as possible processBufferedPackets(callback); @@ -269,18 +339,24 @@ class BufferedPacketQueue [](const auto& a, const auto& b) { return a->first < b->first; }); // Iterate over the sorted packets and invoke the callback + SeqType highest = mLastPacketIdx; for (const auto& it : sortedPackets) { const auto& packet = it->second; logDebug("Processing possibly out-of-order buffered packet with Sequence=%u.", it->first); callback(packet.data(), packet.size()); + if (calculateDistance(highest, it->first) > 0) + { + highest = it->first; + } } mPackets.clear(); // Reset the monotonic increase counter mMonotonicOutOfOrderIncreaseCount = 0; + return highest; } - mLastPacketIdx = currPacketIdx; + return mLastPacketIdx; } /** diff --git a/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp b/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp index c6f79a0b..03ad2d33 100644 --- a/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp +++ b/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp @@ -1,6 +1,7 @@ #include "BufferedPacketQueue.h" // the class under test #include #include +#include #include // ---------- Test fixture ---------------------------------------------------- @@ -10,7 +11,17 @@ class BufferedPacketQueueTest : public ::testing::Test BufferedPacketQueue q; std::vector delivered; - void SetUp() override { delivered.clear(); } + // The queue bounds how long it will hold a packet back, so the tests drive the clock + // themselves instead of letting a loaded machine decide whether the bound was hit. + QueueTimePoint now{}; + + void SetUp() override + { + delivered.clear(); + now = QueueTimePoint{}; + } + + void advance(int ms) { now += std::chrono::milliseconds(ms); } /* Helper: feed one packet and record what the queue actually delivers. */ void feed(uint16_t seq) @@ -23,7 +34,7 @@ class BufferedPacketQueueTest : public ::testing::Test delivered.push_back(*(uint16_t*) seq); }; - q.processPacket(seq, (uint8_t*) &dummy, 2, cb); + q.processPacket(seq, (uint8_t*) &dummy, 2, cb, now); } }; @@ -57,6 +68,66 @@ TEST_F(BufferedPacketQueueTest, ReorderedDeliversInOrder) ASSERT_EQ(delivered, expected) << "Overflow flush should deliver the entire block in one shot"; } +// ---------- A gap that will never be filled -------------------------------- +// The common case on a lossy link: FEC could not recover one packet, and every packet after it +// is held back waiting for it. Nothing is lost by waiting, but everything behind the gap gets +// later and later, so the queue has to give up at some point. +TEST_F(BufferedPacketQueueTest, PermanentGapDoesNotHoldTheStreamForFifteenPackets) +{ + for (uint16_t s = 1; s <= 5; ++s) feed(s); + ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5})); + + // 6 never arrives. + for (uint16_t s = 7; s <= 11; ++s) feed(s); + feed(12); + + ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12})) + << "A monotonic run past the gap should release the buffer, not wait for MAX_BUFFER_SIZE"; +} + +// The monotonic run is counted in packets, so how much latency it costs depends on the packet +// rate - which on the audio stream is a fraction of the video one. The age bound is what makes +// the wait the same on both. +TEST_F(BufferedPacketQueueTest, StaleBufferIsFlushedOnTimeout) +{ + feed(1); + // 2 never arrives. + feed(3); + ASSERT_EQ(delivered, (std::vector{1})) << "3 is held back waiting for 2"; + + advance(25); + feed(4); + + ASSERT_EQ(delivered, (std::vector{1, 3, 4})) + << "Once the buffer is older than MAX_BUFFER_AGE it has to be released"; +} + +// The other side of that bound: a reorder that resolves quickly must still be reordered, not +// flushed out of sequence. +TEST_F(BufferedPacketQueueTest, ReorderWithinTimeoutIsStillPutBackInOrder) +{ + feed(1); + feed(3); + advance(5); + feed(2); + + ASSERT_EQ(delivered, (std::vector{1, 2, 3})) + << "A packet that arrives late but within MAX_BUFFER_AGE must not be flushed early"; +} + +// A large jump is not a reorder - it is a stream that restarted somewhere else. It must not be +// mistaken for a monotonic run, and the buffer cap has to catch it. +TEST_F(BufferedPacketQueueTest, LargeJumpFallsBackToTheBufferCap) +{ + feed(1); + for (uint16_t s = 30000; s < 30000 + MAX_BUFFER_SIZE; ++s) feed(s); + + ASSERT_EQ(delivered.size(), 1u + MAX_BUFFER_SIZE) + << "The buffer cap should have released the jumped-to block"; + ASSERT_EQ(delivered.front(), 1); + ASSERT_EQ(delivered.back(), static_cast(30000 + MAX_BUFFER_SIZE - 1)); +} + // ---------- gtest boilerplate main ----------------------------------------- int main(int argc, char** argv) { diff --git a/app/videonative/src/main/cpp/tests/CMakeLists.txt b/app/videonative/src/main/cpp/tests/CMakeLists.txt index 1e70da39..6ce59ad5 100644 --- a/app/videonative/src/main/cpp/tests/CMakeLists.txt +++ b/app/videonative/src/main/cpp/tests/CMakeLists.txt @@ -25,7 +25,7 @@ enable_testing() # ---------- Test executable -------------------------------------------------- add_executable(queue_test - BufferedPacketqueue_test.cpp + BufferedPacketQueue_test.cpp ) target_include_directories(queue_test PUBLIC From 7d693e408567e6b326591c9a5a296f8c58e929a2 Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:08:22 +0200 Subject: [PATCH 22/23] Keep the immersive launcher entry off devices that cannot run it The immersive activity carries its own MAIN/LAUNCHER entry so Horizon OS can start the app straight into VR. On a phone that is a second icon which leads to an activity that cannot create a session and immediately hands back, and there is no manifest condition for "this device has an XR runtime". So it ships disabled and VideoActivity.onCreate switches it on where isSupportedDevice() agrees. A phone install is then indistinguishable from before this feature existed. On a headset the first launch is the flat activity, which enables the entry, and the VR entry is in the library from then on. Verified on a Quest 3: after install only .VideoActivity resolves for MAIN/LAUNCHER and nothing resolves for the VR category; after one flat launch .XrVideoActivity resolves for both, and a second launch is a no-op rather than another package-manager write. The menu item and the entry are gated on the same predicate, so an XR device that isSupportedDevice() fails to recognise gets neither - widening that check is then the fix. --- app/src/main/AndroidManifest.xml | 9 +++- .../com/openipc/pixelpilot/VideoActivity.java | 3 ++ .../openipc/pixelpilot/XrVideoActivity.java | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 1f0fd554..bbd64ffd 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -108,6 +108,7 @@ + working unchanged, including its USB_DEVICE_ATTACHED filter. + + Shipped disabled: on a phone this entry would be a second icon leading to + an activity that cannot create a session, and there is no manifest + condition for "device has an XR runtime". VideoActivity.onCreate turns it + on where it can actually run - see XrVideoActivity.syncLauncherEntry. --> diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index ee01cde5..7bd5a00f 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -307,6 +307,9 @@ protected void onCreate(Bundle savedInstanceState) { } ModeOwner.claim(this); + // Keep the immersive launcher entry in step with what this device can run. + XrVideoActivity.syncLauncherEntry(this); + // UI Setup initializeUI(); diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java index 8b40185a..fa927a4e 100644 --- a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java @@ -1,10 +1,12 @@ package com.openipc.pixelpilot; import android.content.BroadcastReceiver; +import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; import android.content.SharedPreferences; +import android.content.pm.PackageManager; import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Paint; @@ -52,6 +54,47 @@ public class XrVideoActivity extends LinkClientActivity implements XrGoggleSessi private static final String TAG = "pixelpilot-xr"; + /** + * Enables or disables this activity's launcher entry to match the device. + * + *

The entry is what lets Horizon OS start the app straight into VR, but the manifest + * cannot express "only on a device with an XR runtime", so it ships disabled and is + * switched on here. On a phone it stays off and nothing about the flat mode changes - + * no second icon, no way to land on an activity that cannot create a session. + * + *

Enabling the component is also what makes {@code startActivity} on it work at all, + * so this has to run before VideoActivity's menu can offer it - hence {@code onCreate} + * rather than when the menu is opened. Both are gated on the same predicate, so a device + * {@link XrGoggleSession#isSupportedDevice} does not recognise sees neither the entry nor + * the menu; if that ever turns out to be a false negative, widening that check is the fix. + */ + static void syncLauncherEntry(Context context) { + boolean wantEnabled = XrGoggleSession.isSupportedDevice(context); + ComponentName component = new ComponentName(context, XrVideoActivity.class); + PackageManager pm = context.getPackageManager(); + try { + // COMPONENT_ENABLED_STATE_DEFAULT means "whatever the manifest said", which here + // is disabled - so anything other than an explicit ENABLED counts as off. + boolean isEnabled = pm.getComponentEnabledSetting(component) + == PackageManager.COMPONENT_ENABLED_STATE_ENABLED; + if (isEnabled == wantEnabled) { + return; + } + pm.setComponentEnabledSetting( + component, + wantEnabled + ? PackageManager.COMPONENT_ENABLED_STATE_ENABLED + // Back to the manifest value rather than an explicit disable, so + // an unrecognised device is left exactly as it shipped. + : PackageManager.COMPONENT_ENABLED_STATE_DEFAULT, + PackageManager.DONT_KILL_APP); + Log.i(TAG, "immersive launcher entry " + (wantEnabled ? "enabled" : "disabled")); + } catch (Exception e) { + // A launcher entry is not worth taking the flat mode down over. + Log.w(TAG, "could not update the immersive launcher entry", e); + } + } + // Preference keys, all in the shared "general" store so the flat activity's settings // menu can write them. static final String PREF_HEAD_LOCKED = "xr_head_locked"; From db3717b5bcf52247b351ecfa395b661193af608d Mon Sep 17 00:00:00 2001 From: iflyhere <57563846+iflyhere@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:40:01 +0200 Subject: [PATCH 23/23] Advance the queue past everything a flush delivered Two problems with the flush, both found in review. restartBuffering() set mLastPacketIdx to the packet that triggered the flush rather than the newest one delivered. With 1..5 delivered, 6 lost, then 7 8 9 12 10, the flush hands over 7 8 9 10 12 but leaves the pointer on 10 - so 11 would be delivered after 12, and 13..16 are held until 17 arrives. The queue re-created the stall it exists to prevent. bufferPacket() always runs before the flush, so the triggering packet is in the buffer either way and mLastPacketIdx = drainBufferInOrder() is all that was needed; restartBuffering() is gone. That alone would have broken the resync on a large jump, because drainBufferInOrder() seeded `highest` from mLastPacketIdx and compared with calculateDistance(), which reads as negative past half the sequence space. RTP starts at a random sequence number, so a VTX that reboots mid-session lands exactly there: fed 1, 2, then 40000, 40001, ..., neither the seed nor the comparison ever moves and every packet comes out one flush late, forever. On video the fifteen-packet cap used to paper over it by rewinding to currPacketIdx; on audio nothing did. `highest` is now seeded from a packet that is actually in the buffer. While sorting the flush: by raw value a block straddling the wrap point (65534, 65535, 0, 1) sorts to 0, 1, 65534, 65535 and was handed to the parser in that order. Sorting by distance from the last delivered packet fixes the order and makes the last element the newest. Three tests added, one per problem. The large-jump case uses 40000 - the existing LargeJumpFallsBackToTheBufferCap uses 30000, which is still positive as an int16 and does not reach any of this. --- .../src/main/cpp/BufferedPacketQueue.h | 39 ++++++------- .../cpp/tests/BufferedPacketQueue_test.cpp | 58 +++++++++++++++++++ 2 files changed, 75 insertions(+), 22 deletions(-) diff --git a/app/videonative/src/main/cpp/BufferedPacketQueue.h b/app/videonative/src/main/cpp/BufferedPacketQueue.h index 5efcc9d1..891540a8 100644 --- a/app/videonative/src/main/cpp/BufferedPacketQueue.h +++ b/app/videonative/src/main/cpp/BufferedPacketQueue.h @@ -246,10 +246,8 @@ class BufferedPacketQueue logDebug("Monotonic increase count: %zu", mMonotonicOutOfOrderIncreaseCount); if (mMonotonicOutOfOrderIncreaseCount >= MONOTONIC_THRESHOLD) { - restartBuffering(callback, currPacketIdx); - // Update lastPacketIdx to the highest sequence index received - SeqType newLastIdx = currPacketIdx; - logWarning("Monotonic threshold reached. Updating lastPacketIdx to %u", newLastIdx); + mLastPacketIdx = drainBufferInOrder(callback); + logWarning("Monotonic threshold reached. Updating lastPacketIdx to %u", mLastPacketIdx); } } else @@ -264,7 +262,7 @@ class BufferedPacketQueue { logWarning( "Buffer size exceeded MAX_BUFFER_SIZE (%zu). Processing in-order buffered packets.", MAX_BUFFER_SIZE); - restartBuffering(callback, currPacketIdx); + mLastPacketIdx = drainBufferInOrder(callback); } } @@ -294,18 +292,6 @@ class BufferedPacketQueue logDebug("Buffered out-of-order packet. Buffer size: %zu", mPackets.size()); } - /** - * @brief Handles buffer overflow by processing in-order packets and discarding others. - * @tparam Callback A callable type that processes the packet data. - * @param callback Callable to handle processed packets. - */ - template - void restartBuffering(Callback& callback, SeqType currPacketIdx) - { - drainBufferInOrder(callback); - mLastPacketIdx = currPacketIdx; - } - /** * @brief Delivers everything currently held back, in sequence order, and empties the buffer. * @tparam Callback A callable type that processes the packet data. @@ -332,14 +318,23 @@ class BufferedPacketQueue sortedPackets.push_back(it); } - // Sort the vector based on the keys + // Sorted by distance from the last delivered packet, not by raw value: a block + // that straddles the wrap point (65534, 65535, 0, 1) sorts to 0, 1, 65534, 65535 + // by value, and would be handed to the parser in that order. + const SeqType from = mLastPacketIdx; std::sort( sortedPackets.begin(), sortedPackets.end(), - [](const auto& a, const auto& b) { return a->first < b->first; }); - - // Iterate over the sorted packets and invoke the callback - SeqType highest = mLastPacketIdx; + [from](const auto& a, const auto& b) { + return static_cast(a->first - from) < static_cast(b->first - from); + }); + + // Seeded from a packet that is actually in the buffer rather than from + // mLastPacketIdx. RTP starts at a random sequence number, so a VTX that reboots + // mid-session can land more than half the sequence space away, where + // calculateDistance() reads as negative - seeded from mLastPacketIdx nothing + // would ever move and the queue would never resync. + SeqType highest = sortedPackets.front()->first; for (const auto& it : sortedPackets) { const auto& packet = it->second; diff --git a/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp b/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp index 03ad2d33..6e803b62 100644 --- a/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp +++ b/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp @@ -128,6 +128,64 @@ TEST_F(BufferedPacketQueueTest, LargeJumpFallsBackToTheBufferCap) ASSERT_EQ(delivered.back(), static_cast(30000 + MAX_BUFFER_SIZE - 1)); } +// A flush delivers everything it is holding, so the pointer has to end up past all of it. +// Rewinding it to whichever packet happened to trigger the flush re-creates the stall that +// was just cleared. +TEST_F(BufferedPacketQueueTest, FlushAdvancesPastEverythingItDelivered) +{ + for (uint16_t s = 1; s <= 5; ++s) feed(s); + + // 6 never arrives, and 11 is missing from the run so the flush is triggered by 10 while + // 12 is already buffered behind it. + feed(7); + feed(8); + feed(9); + feed(12); + feed(10); + ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5, 7, 8, 9, 10, 12})); + + feed(13); + ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13})) + << "13 follows the highest packet already delivered and must not be held back"; +} + +// RTP starts at a random sequence number, so a VTX rebooting mid-session shows up as a jump +// that can be more than half the sequence space - where a signed 16 bit distance reads as +// "behind us" and the queue would never move again. +TEST_F(BufferedPacketQueueTest, StreamRestartFarAheadResyncs) +{ + feed(1); + feed(2); + + feed(40000); + ASSERT_EQ(delivered, (std::vector{1, 2})) << "40000 is held back at first"; + + advance(25); + feed(40001); + feed(40002); + + ASSERT_EQ(delivered, (std::vector{1, 2, 40000, 40001, 40002})) + << "after the age bound releases 40000 the queue has to follow the new base"; +} + +// The buffer is a hash map, so the flush has to sort it - and sorting by raw value puts a +// block that straddles the wrap point in the wrong order. +TEST_F(BufferedPacketQueueTest, FlushAcrossTheWrapDeliversInOrder) +{ + feed(65532); + + // 65533 never arrives; the rest of the run crosses zero. + feed(65534); + feed(65535); + feed(0); + feed(1); + feed(2); + feed(3); + + ASSERT_EQ(delivered, (std::vector{65532, 65534, 65535, 0, 1, 2, 3})) + << "sorted by value this comes out as 0, 1, 2, 65534, 65535"; +} + // ---------- gtest boilerplate main ----------------------------------------- int main(int argc, char** argv) {