diff --git a/app/build.gradle b/app/build.gradle
index 66bbd35b..656a8089 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -59,6 +59,7 @@ dependencies {
implementation(project(":app:mavlink"))
implementation(project(":app:videonative"))
implementation(project(":app:wfbngrtl8812"))
+ implementation(project(":app:xr"))
implementation(libs.appcompat)
implementation(libs.material)
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 1ad6bc61..bbd64ffd 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -7,6 +7,24 @@
+
+
+
+
+
+
@@ -19,6 +37,9 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/app/src/main/java/com/openipc/pixelpilot/DvrFiles.java b/app/src/main/java/com/openipc/pixelpilot/DvrFiles.java
new file mode 100644
index 00000000..3db7e67b
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/DvrFiles.java
@@ -0,0 +1,98 @@
+package com.openipc.pixelpilot;
+
+import android.content.Context;
+import android.net.Uri;
+import android.util.Log;
+
+import androidx.documentfile.provider.DocumentFile;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * DVR file naming and creation, shared by the flat and the immersive activity.
+ *
+ * Everything here works off the already granted {@code dvr_folder_} tree URI. Picking a
+ * folder needs an Activity result and stays with the flat activity.
+ */
+public final class DvrFiles {
+ private static final String TAG = "pixelpilot";
+ private static final String PREF_DVR_FILENAME = "dvr_filename";
+ private static final String DEFAULT_TEMPLATE = "pixelpilot_[yyyyMMdd-HHmmss]";
+ private static final Pattern TEMPLATE_PATTERN = Pattern.compile("\\[([^\\]]*)\\]");
+
+ private DvrFiles() {
+ }
+
+ public static String template(Context context) {
+ return context.getSharedPreferences("general", Context.MODE_PRIVATE)
+ .getString(PREF_DVR_FILENAME, DEFAULT_TEMPLATE);
+ }
+
+ public static void setTemplate(Context context, String template) {
+ context.getSharedPreferences("general", Context.MODE_PRIVATE)
+ .edit().putString(PREF_DVR_FILENAME, template).apply();
+ }
+
+ public static boolean fragmentedMp4(Context context) {
+ return context.getSharedPreferences("general", Context.MODE_PRIVATE)
+ .getBoolean("dvr_fmp4", true);
+ }
+
+ /** Expands the {@code [pattern]} placeholder in a file name template. */
+ public static String fileName(String template, LocalDateTime time) {
+ Matcher matcher = TEMPLATE_PATTERN.matcher(template);
+ String fallbackTime = time.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss"));
+
+ if (matcher.find()) {
+ String prefix = template.substring(0, matcher.start());
+ String pattern = matcher.group(1);
+ String suffix = template.substring(matcher.end());
+
+ try {
+ String timePart = time.format(DateTimeFormatter.ofPattern(pattern));
+ return prefix + timePart + suffix;
+ } catch (IllegalArgumentException e) {
+ return prefix + fallbackTime + suffix;
+ }
+ }
+ return "pixelpilot_" + fallbackTime;
+ }
+
+ public static boolean hasRecordingFolder(Context context) {
+ return !context.getSharedPreferences("general", Context.MODE_PRIVATE)
+ .getString("dvr_folder_", "").isEmpty();
+ }
+
+ /**
+ * Creates the next recording file in the configured folder.
+ *
+ * @return its URI, or null if no folder is configured or it is not writable
+ */
+ public static Uri createRecording(Context context) {
+ String dvrFolder = context.getSharedPreferences("general", Context.MODE_PRIVATE)
+ .getString("dvr_folder_", "");
+ if (dvrFolder.isEmpty()) {
+ Log.e(TAG, "dvrFolder is empty");
+ return null;
+ }
+ DocumentFile pickedDir = DocumentFile.fromTreeUri(context, Uri.parse(dvrFolder));
+ if (pickedDir == null || !pickedDir.canWrite()) {
+ return null;
+ }
+ String name = fileName(template(context), LocalDateTime.now()) + ".mp4";
+ DocumentFile newFile = pickedDir.createFile("video/mp4", name);
+ if (newFile == null) {
+ Log.e(TAG, "dvr newFile null");
+ return null;
+ }
+ return newFile.getUri();
+ }
+
+ /** File name the next {@link #createRecording} would use, for UI previews. */
+ public static String nextFileName(Context context) {
+ return fileName(template(context), LocalDateTime.now()) + ".mp4";
+ }
+}
diff --git a/app/src/main/java/com/openipc/pixelpilot/GsKey.java b/app/src/main/java/com/openipc/pixelpilot/GsKey.java
new file mode 100644
index 00000000..3f5d8e3c
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/GsKey.java
@@ -0,0 +1,87 @@
+package com.openipc.pixelpilot;
+
+import android.content.Context;
+import android.content.SharedPreferences;
+import android.util.Base64;
+import android.util.Log;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * The wfb-ng ground station key, shared by the flat and the immersive activity.
+ *
+ *
The key lives in preferences (so a user-picked one survives) and has to be
+ * materialised as {@code files/gs.key} before {@code WfbNgLink} is constructed: the native
+ * aggregators open that path in their constructor and throw if it is missing.
+ */
+public final class GsKey {
+ private static final String TAG = "pixelpilot";
+ private static final String PREF_KEY = "gs.key";
+ private static final String FILE_NAME = "gs.key";
+
+ private GsKey() {
+ }
+
+ public static byte[] get(Context context) {
+ String pref = context.getSharedPreferences("general", Context.MODE_PRIVATE)
+ .getString(PREF_KEY, "");
+ return Base64.decode(pref, Base64.DEFAULT);
+ }
+
+ public static void set(Context context, InputStream inputStream) throws IOException {
+ ByteArrayOutputStream result = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int length;
+ while ((length = inputStream.read(buffer)) != -1) {
+ result.write(buffer, 0, length);
+ }
+ SharedPreferences prefs = context.getSharedPreferences("general", Context.MODE_PRIVATE);
+ prefs.edit().putString(PREF_KEY, Base64.encodeToString(result.toByteArray(), Base64.DEFAULT)).apply();
+ }
+
+ /** Seeds preferences from the bundled asset the first time round. */
+ public static void importDefaultIfMissing(Context context) {
+ if (get(context).length > 0) {
+ Log.d(TAG, "gs.key already saved in preferences.");
+ return;
+ }
+ try (InputStream inputStream = context.getAssets().open(FILE_NAME)) {
+ Log.d(TAG, "Importing default gs.key...");
+ set(context, inputStream);
+ } catch (IOException e) {
+ Log.e(TAG, "Failed to import default gs.key", e);
+ }
+ }
+
+ /** Writes the key from preferences to {@code files/gs.key}. */
+ public static boolean writeToFilesDir(Context context) {
+ byte[] keyBytes = get(context);
+ if (keyBytes.length == 0) {
+ Log.e(TAG, "No gs.key available to write");
+ return false;
+ }
+ File file = new File(context.getApplicationContext().getFilesDir(), FILE_NAME);
+ try (OutputStream out = new FileOutputStream(file)) {
+ out.write(keyBytes, 0, keyBytes.length);
+ Log.d(TAG, "Wrote gs.key (" + keyBytes.length + " bytes) to " + file.getAbsolutePath());
+ return true;
+ } catch (IOException e) {
+ Log.e(TAG, "Failed to write gs.key", e);
+ return false;
+ }
+ }
+
+ /**
+ * Everything that has to happen before {@code new WfbNgLink(...)}. Call from any
+ * activity that brings up the link.
+ */
+ public static boolean ensure(Context context) {
+ importDefaultIfMissing(context);
+ return writeToFilesDir(context);
+ }
+}
diff --git a/app/src/main/java/com/openipc/pixelpilot/LinkClientActivity.java b/app/src/main/java/com/openipc/pixelpilot/LinkClientActivity.java
new file mode 100644
index 00000000..65f9a6c4
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/LinkClientActivity.java
@@ -0,0 +1,98 @@
+package com.openipc.pixelpilot;
+
+import android.content.ComponentName;
+import android.content.Context;
+import android.content.Intent;
+import android.content.ServiceConnection;
+import android.os.Bundle;
+import android.os.IBinder;
+import android.util.Log;
+
+import androidx.annotation.Nullable;
+import androidx.appcompat.app.AppCompatActivity;
+
+import com.openipc.videonative.DecodingInfo;
+import com.openipc.wfbngrtl8812.WfbNGStats;
+
+/**
+ * Binds to {@link LinkService} for the lifetime of the activity.
+ *
+ *
Bound in {@code onCreate} and unbound in {@code onDestroy} rather than around
+ * start/stop: the point of the service is that the link survives the activity being paused,
+ * so a system menu or taking the headset off no longer costs a reconnect.
+ */
+public abstract class LinkClientActivity extends AppCompatActivity implements LinkService.Client {
+
+ private static final String TAG = "pixelpilot";
+
+ @Nullable
+ protected LinkService link;
+ private boolean bound;
+
+ private final ServiceConnection connection = new ServiceConnection() {
+ @Override
+ public void onServiceConnected(ComponentName name, IBinder service) {
+ link = ((LinkService.LocalBinder) service).service();
+ link.addClient(LinkClientActivity.this);
+ Log.i(TAG, "attached to the link service");
+ onLinkServiceConnected(link);
+ }
+
+ @Override
+ public void onServiceDisconnected(ComponentName name) {
+ // The process is going away; nothing useful to do but stop talking to it.
+ link = null;
+ }
+ };
+
+ /** Called on the main thread once the link is available. */
+ protected abstract void onLinkServiceConnected(LinkService service);
+
+ @Override
+ protected void onCreate(@Nullable Bundle savedInstanceState) {
+ super.onCreate(savedInstanceState);
+ Intent intent = new Intent(this, LinkService.class);
+ // Started as well as bound, so it keeps running while no activity is attached - that
+ // is what carries the link across a mode switch.
+ startForegroundService(intent);
+ bound = bindService(intent, connection, Context.BIND_AUTO_CREATE);
+ if (!bound) {
+ Log.e(TAG, "could not bind the link service");
+ }
+ }
+
+ @Override
+ protected void onDestroy() {
+ if (link != null) {
+ link.removeClient(this);
+ }
+ if (bound) {
+ unbindService(connection);
+ bound = false;
+ }
+ link = null;
+ super.onDestroy();
+ }
+
+ // Defaults so a subclass only overrides what it draws.
+
+ @Override
+ public void onLinkStatus(String message) {
+ }
+
+ @Override
+ public void onLocalStreamHint(String url) {
+ }
+
+ @Override
+ public void onVideoResolution(int width, int height) {
+ }
+
+ @Override
+ public void onDecodingInfo(DecodingInfo info) {
+ }
+
+ @Override
+ public void onWfbStats(WfbNGStats stats) {
+ }
+}
diff --git a/app/src/main/java/com/openipc/pixelpilot/LinkService.java b/app/src/main/java/com/openipc/pixelpilot/LinkService.java
new file mode 100644
index 00000000..72d81d0a
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/LinkService.java
@@ -0,0 +1,405 @@
+package com.openipc.pixelpilot;
+
+import android.app.Notification;
+import android.app.NotificationChannel;
+import android.app.NotificationManager;
+import android.app.PendingIntent;
+import android.app.Service;
+import android.content.Context;
+import android.content.Intent;
+import android.content.IntentFilter;
+import android.hardware.usb.UsbManager;
+import android.os.Binder;
+import android.os.Build;
+import android.os.Handler;
+import android.os.IBinder;
+import android.os.Looper;
+import android.util.Log;
+import android.view.Surface;
+
+import androidx.annotation.Nullable;
+
+import com.openipc.videonative.DecodingInfo;
+import com.openipc.videonative.IVideoParamsChanged;
+import com.openipc.videonative.VideoPlayer;
+import com.openipc.wfbngrtl8812.WfbNGStats;
+import com.openipc.wfbngrtl8812.WfbNGStatsChanged;
+import com.openipc.wfbngrtl8812.WfbNgLink;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+
+/**
+ * Sole owner of the wifi adapter, the wfb-ng link and the decoder.
+ *
+ *
These used to belong to whichever Activity was on screen, which does not survive contact
+ * with a ground station: the flat activity tore the link down in {@code onPause()} (so
+ * glancing at a system menu cost seconds of black screen), and with a second presentation
+ * mode there were two owners for one adapter - both activities ended up alive in separate
+ * tasks with the adapter owned by neither, and no video in either mode.
+ *
+ *
Now the link outlives the activities. They bind, register as a {@link Client}, and hand
+ * over a Surface; switching modes only swaps the Surface, so the adapter is never reopened
+ * and the USB permission dialog is not asked again. The service is a foreground service
+ * because it must keep running while the pilot is looking at something else.
+ */
+public class LinkService extends Service implements LinkStatusView, IVideoParamsChanged, WfbNGStatsChanged {
+
+ public static final String ACTION_STOP = "com.openipc.pixelpilot.LINK_STOP";
+
+ private static final String TAG = "pixelpilot";
+ private static final String CHANNEL_ID = "link";
+ private static final int NOTIFICATION_ID = 1;
+
+ /**
+ * Grace period after the last client unbinds. A mode switch unbinds one activity and
+ * binds the next, and tearing the link down in between is exactly what this service
+ * exists to avoid.
+ */
+ private static final long IDLE_SHUTDOWN_MS = 20_000;
+
+ /** What an attached activity gets told. All callbacks arrive on the main thread. */
+ public interface Client {
+ void onLinkStatus(String message);
+
+ void onLocalStreamHint(String url);
+
+ void onVideoResolution(int width, int height);
+
+ void onDecodingInfo(DecodingInfo info);
+
+ void onWfbStats(WfbNGStats stats);
+ }
+
+ public class LocalBinder extends Binder {
+ public LinkService service() {
+ return LinkService.this;
+ }
+ }
+
+ private final LocalBinder binder = new LocalBinder();
+ private final Handler main = new Handler(Looper.getMainLooper());
+ private final CopyOnWriteArrayList clients = new CopyOnWriteArrayList<>();
+
+ private WfbNgLink wfbLink;
+ private WfbLinkManager wfbLinkManager;
+ private VideoPlayer videoPlayer;
+
+ private boolean ready;
+ private boolean usbReceiverRegistered;
+ private String lastStatus = "";
+ private WfbNGStats lastStats;
+ private int videoWidth;
+ private int videoHeight;
+
+ private final Runnable idleShutdown = () -> {
+ Log.i(TAG, "no client came back, stopping the link service");
+ stopSelf();
+ };
+
+ // ------------------------------------------------------------------------------
+ // lifecycle
+ // ------------------------------------------------------------------------------
+
+ @Override
+ public void onCreate() {
+ super.onCreate();
+ startForegroundNotification();
+
+ // The native aggregators open files/gs.key in their constructor and throw if it is
+ // missing, so this has to happen before WfbNgLink exists.
+ if (!GsKey.ensure(this)) {
+ Log.e(TAG, "no usable gs.key, the link cannot come up");
+ }
+
+ wfbLink = new WfbNgLink(this);
+ wfbLink.SetWfbNGStatsChanged(this);
+ // Not read by the native side on its own: adaptive link, TX power, FEC/LDPC/STBC.
+ WfbOptions.applyDefaults(this, wfbLink);
+
+ wfbLinkManager = new WfbLinkManager(this, this, wfbLink);
+
+ videoPlayer = new VideoPlayer(this);
+ videoPlayer.setIVideoParamsChanged(this);
+ videoPlayer.setLowLatency(VideoActivity.getLowLatencySetting(this));
+ // Receivers come up once and stay up; only the output surface changes with the mode.
+ videoPlayer.start();
+ videoPlayer.startAudio();
+
+ registerUsbReceiver();
+
+ ready = true;
+ Log.i(TAG, "link service up");
+ }
+
+ @Override
+ public int onStartCommand(Intent intent, int flags, int startId) {
+ if (intent != null && ACTION_STOP.equals(intent.getAction())) {
+ Log.i(TAG, "stop requested from the notification");
+ stopSelf();
+ return START_NOT_STICKY;
+ }
+ return START_STICKY;
+ }
+
+ @Nullable
+ @Override
+ public IBinder onBind(Intent intent) {
+ main.removeCallbacks(idleShutdown);
+ return binder;
+ }
+
+ @Override
+ public void onRebind(Intent intent) {
+ main.removeCallbacks(idleShutdown);
+ }
+
+ @Override
+ public boolean onUnbind(Intent intent) {
+ if (clients.isEmpty()) {
+ main.postDelayed(idleShutdown, IDLE_SHUTDOWN_MS);
+ }
+ // Allow onRebind, which is how a mode switch keeps the link alive.
+ return true;
+ }
+
+ @Override
+ public void onDestroy() {
+ Log.i(TAG, "link service going down");
+ main.removeCallbacks(idleShutdown);
+ ready = false;
+ if (usbReceiverRegistered) {
+ try {
+ unregisterReceiver(wfbLinkManager);
+ } catch (IllegalArgumentException ignored) {
+ }
+ usbReceiverRegistered = false;
+ }
+ if (wfbLinkManager != null) {
+ wfbLinkManager.stopAdapters();
+ }
+ if (videoPlayer != null) {
+ videoPlayer.stopAudio();
+ videoPlayer.stop();
+ }
+ super.onDestroy();
+ }
+
+ /**
+ * WfbLinkManager is itself the USB attach/detach receiver, and it has to be registered by
+ * whoever owns the adapter - previously an Activity, which meant plug events were missed
+ * whenever no activity was around.
+ */
+ private void registerUsbReceiver() {
+ IntentFilter filter = new IntentFilter();
+ filter.addAction(UsbManager.ACTION_USB_DEVICE_ATTACHED);
+ filter.addAction(UsbManager.ACTION_USB_DEVICE_DETACHED);
+ filter.addAction(WfbLinkManager.ACTION_USB_PERMISSION);
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+ registerReceiver(wfbLinkManager, filter, Context.RECEIVER_NOT_EXPORTED);
+ } else {
+ registerReceiver(wfbLinkManager, filter);
+ }
+ usbReceiverRegistered = true;
+ }
+
+ private void startForegroundNotification() {
+ NotificationManager manager = getSystemService(NotificationManager.class);
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && manager != null
+ && manager.getNotificationChannel(CHANNEL_ID) == null) {
+ NotificationChannel channel = new NotificationChannel(
+ CHANNEL_ID, "Video link", NotificationManager.IMPORTANCE_LOW);
+ channel.setShowBadge(false);
+ manager.createNotificationChannel(channel);
+ }
+
+ Intent stop = new Intent(this, LinkService.class).setAction(ACTION_STOP);
+ PendingIntent stopIntent = PendingIntent.getService(
+ this, 0, stop, PendingIntent.FLAG_IMMUTABLE | PendingIntent.FLAG_UPDATE_CURRENT);
+
+ Notification notification = new Notification.Builder(this, CHANNEL_ID)
+ .setContentTitle("PixelPilot link")
+ .setContentText("Holding the wifi adapter and decoder")
+ .setSmallIcon(R.drawable.baseline_settings_24)
+ .setOngoing(true)
+ .addAction(new Notification.Action.Builder(null, "Stop", stopIntent).build())
+ .build();
+
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+ startForeground(NOTIFICATION_ID, notification,
+ android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC);
+ } else {
+ startForeground(NOTIFICATION_ID, notification);
+ }
+ }
+
+ // ------------------------------------------------------------------------------
+ // client API
+ // ------------------------------------------------------------------------------
+
+ public boolean isReady() {
+ return ready;
+ }
+
+ /** Registers a client and replays the state it missed, so it never starts out blank. */
+ public void addClient(Client client) {
+ clients.addIfAbsent(client);
+ main.removeCallbacks(idleShutdown);
+ if (!lastStatus.isEmpty()) {
+ client.onLinkStatus(lastStatus);
+ }
+ if (videoWidth > 0 && videoHeight > 0) {
+ client.onVideoResolution(videoWidth, videoHeight);
+ }
+ WfbNGStats stats = lastStats;
+ if (stats != null) {
+ client.onWfbStats(stats);
+ }
+ }
+
+ public void removeClient(Client client) {
+ clients.remove(client);
+ }
+
+ /** Points the decoder at a new output surface. The link keeps running throughout. */
+ public void attachSurface(Surface surface, int index) {
+ if (videoPlayer != null) {
+ videoPlayer.attachSurface(surface, index);
+ }
+ }
+
+ public void detachSurface(int index) {
+ if (videoPlayer != null) {
+ videoPlayer.detachSurface(index);
+ }
+ }
+
+ /** Brings the adapter up if it is not already, using the persisted channel/bandwidth. */
+ public void ensureAdapters() {
+ if (wfbLinkManager == null) {
+ return;
+ }
+ wfbLinkManager.setChannel(VideoActivity.getChannel(this));
+ wfbLinkManager.setBandwidth(VideoActivity.getBandwidth(this));
+ wfbLinkManager.refreshAdapters();
+ }
+
+ public boolean hasActiveAdapter() {
+ return wfbLinkManager != null && wfbLinkManager.hasActiveAdapter();
+ }
+
+ public void setChannel(int channel) {
+ if (wfbLinkManager == null) {
+ return;
+ }
+ wfbLinkManager.stopAdapters();
+ wfbLinkManager.setChannel(channel);
+ wfbLinkManager.startAdapters();
+ }
+
+ public void setBandwidth(int bandwidth) {
+ if (wfbLinkManager == null) {
+ return;
+ }
+ wfbLinkManager.stopAdapters();
+ wfbLinkManager.setBandwidth(bandwidth);
+ wfbLinkManager.startAdapters();
+ }
+
+ public void refreshKey() {
+ if (wfbLinkManager != null) {
+ wfbLinkManager.refreshKey();
+ }
+ }
+
+ /** For the settings menus that drive the native link options directly. */
+ @Nullable
+ public WfbNgLink link() {
+ return wfbLink;
+ }
+
+ public void startDvr(int fd, boolean fragmentedMp4) {
+ if (videoPlayer != null) {
+ videoPlayer.startDvr(fd, fragmentedMp4);
+ }
+ }
+
+ public void stopDvr() {
+ if (videoPlayer != null) {
+ videoPlayer.stopDvr();
+ }
+ }
+
+ public void setUdpForwarding(String ip, int port, boolean enabled) {
+ if (videoPlayer != null) {
+ videoPlayer.setUdpForwarding(ip, port, enabled);
+ }
+ }
+
+ public void setLowLatency(boolean enabled) {
+ if (videoPlayer != null) {
+ videoPlayer.setLowLatency(enabled);
+ }
+ }
+
+ public int videoWidth() {
+ return videoWidth;
+ }
+
+ public int videoHeight() {
+ return videoHeight;
+ }
+
+ // ------------------------------------------------------------------------------
+ // fan-out to the attached activity
+ // ------------------------------------------------------------------------------
+
+ @Override
+ public void showLinkMessage(String message) {
+ Log.i(TAG, "link: " + message);
+ lastStatus = message;
+ main.post(() -> {
+ for (Client c : clients) {
+ c.onLinkStatus(message);
+ }
+ });
+ }
+
+ @Override
+ public void showLocalStreamHint(String url) {
+ main.post(() -> {
+ for (Client c : clients) {
+ c.onLocalStreamHint(url);
+ }
+ });
+ }
+
+ @Override
+ public void onVideoRatioChanged(int width, int height) {
+ videoWidth = width;
+ videoHeight = height;
+ main.post(() -> {
+ for (Client c : clients) {
+ c.onVideoResolution(width, height);
+ }
+ });
+ }
+
+ @Override
+ public void onDecodingInfoChanged(DecodingInfo decodingInfo) {
+ main.post(() -> {
+ for (Client c : clients) {
+ c.onDecodingInfo(decodingInfo);
+ }
+ });
+ }
+
+ @Override
+ public void onWfbNgStatsChanged(WfbNGStats stats) {
+ lastStats = stats;
+ main.post(() -> {
+ for (Client c : clients) {
+ c.onWfbStats(stats);
+ }
+ });
+ }
+}
diff --git a/app/src/main/java/com/openipc/pixelpilot/LinkStatusView.java b/app/src/main/java/com/openipc/pixelpilot/LinkStatusView.java
new file mode 100644
index 00000000..3d74a53b
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/LinkStatusView.java
@@ -0,0 +1,13 @@
+package com.openipc.pixelpilot;
+
+/**
+ * Where {@link WfbLinkManager} reports adapter/link status to. The flat activity draws it
+ * into its overlay; the immersive activity has no view hierarchy and logs it instead.
+ */
+public interface LinkStatusView {
+ /** Adapter or link state the pilot needs to see (starting, no permission, ...). */
+ void showLinkMessage(String message);
+
+ /** Shown when no compatible adapter is attached: where to push a stream instead. */
+ void showLocalStreamHint(String url);
+}
diff --git a/app/src/main/java/com/openipc/pixelpilot/ModeOwner.java b/app/src/main/java/com/openipc/pixelpilot/ModeOwner.java
new file mode 100644
index 00000000..80443d10
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/ModeOwner.java
@@ -0,0 +1,45 @@
+package com.openipc.pixelpilot;
+
+import android.app.Activity;
+import android.util.Log;
+
+import java.lang.ref.WeakReference;
+
+/**
+ * Guarantees that only one of the two presentation modes is alive at a time.
+ *
+ * The flat and the immersive activity each construct their own {@code WfbNgLink}, and the
+ * wifi adapter can only belong to one of them. Horizon OS keeps the immersive activity in its
+ * own task, so switching modes left both of them resumed in separate tasks with the adapter
+ * owned by neither, and no video in either mode.
+ *
+ *
Rather than trying to make two owners cooperate, whoever starts last evicts the other
+ * and its {@code onDestroy()} releases the adapter. The proper answer is a single owner that
+ * outlives both - a foreground service - but this removes the failure without that rework.
+ */
+final class ModeOwner {
+ private static final String TAG = "pixelpilot";
+
+ private static WeakReference current = new WeakReference<>(null);
+
+ private ModeOwner() {
+ }
+
+ /** Registers {@code activity} as the mode owner and finishes whoever held it before. */
+ static synchronized void claim(Activity activity) {
+ Activity previous = current.get();
+ if (previous != null && previous != activity && !previous.isFinishing()) {
+ Log.i(TAG, "mode switch: finishing " + previous.getClass().getSimpleName()
+ + " so " + activity.getClass().getSimpleName() + " can own the adapter");
+ previous.finish();
+ }
+ current = new WeakReference<>(activity);
+ }
+
+ /** Clears the registration if {@code activity} still holds it. */
+ static synchronized void release(Activity activity) {
+ if (current.get() == activity) {
+ current = new WeakReference<>(null);
+ }
+ }
+}
diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java
index e126fa2e..7bd5a00f 100644
--- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java
+++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java
@@ -16,6 +16,7 @@
import android.hardware.usb.UsbManager;
import android.net.Uri;
import android.net.VpnService;
+import android.net.wifi.WifiInfo;
import android.net.wifi.WifiManager;
import android.os.BatteryManager;
import android.os.Build;
@@ -60,11 +61,8 @@
import com.openipc.pixelpilot.osd.OSDElement;
import com.openipc.pixelpilot.osd.OSDManager;
import com.openipc.videonative.DecodingInfo;
-import com.openipc.videonative.IVideoParamsChanged;
-import com.openipc.videonative.VideoPlayer;
import com.openipc.wfbngrtl8812.WfbNGStats;
-import com.openipc.wfbngrtl8812.WfbNGStatsChanged;
-import com.openipc.wfbngrtl8812.WfbNgLink;
+import com.openipc.xr.XrGoggleSession;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
@@ -92,6 +90,7 @@
import android.os.SystemClock;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import android.view.SurfaceHolder;
import android.view.TextureView;
import android.graphics.SurfaceTexture;
import android.view.Surface;
@@ -100,8 +99,8 @@
// Most basic implementation of an activity that uses VideoNative to stream a video
// Into an Android Surface View
-public class VideoActivity extends AppCompatActivity implements IVideoParamsChanged,
- WfbNGStatsChanged, MavlinkUpdate, SettingsChanged {
+public class VideoActivity extends LinkClientActivity
+ implements MavlinkUpdate, SettingsChanged {
private static final String TAG = "pixelpilot";
private static final int PICK_KEY_REQUEST_CODE = 1;
private static final int PICK_DVR_REQUEST_CODE = 2;
@@ -114,7 +113,6 @@ public class VideoActivity extends AppCompatActivity implements IVideoParamsChan
private static final long MODEL_LITE2_BYTES = 23096891L;
private static final String PREF_OD_CUSTOM_MODEL_URI = "od_custom_model_uri";
private static final String PREF_OD_CUSTOM_MODEL_NAME = "od_custom_model_name";
- private static WifiManager wifiManager;
final Handler handler = new Handler(Looper.getMainLooper());
final Runnable runnable = new Runnable() {
public void run() {
@@ -124,9 +122,7 @@ public void run() {
};
protected DecodingInfo mDecodingInfo;
int lastVideoW = 0, lastVideoH = 0, lastCodec = 1;
- WfbLinkManager wfbLinkManager;
BroadcastReceiver batteryReceiver;
- VideoPlayer videoPlayer;
private ActivityVideoBinding binding;
private OSDManager osdManager;
private ParcelFileDescriptor dvrFd = null;
@@ -137,9 +133,11 @@ public void run() {
// Which view the main video is rendered into. SurfaceView is the low latency
// default; the TextureView is only used when object detection needs getBitmap().
private boolean videoUsesTextureView = false;
+ // A Surface can become available before the service has bound - binding is async and the
+ // view hierarchy does not wait for it - so it is parked here until the link exists.
+ private final Surface[] pendingSurfaces = new Surface[2];
private ConstraintLayout constraintLayout;
private ConstraintSet constraintSet;
- private WfbNgLink wfbLink;
private ObjectDetectorHelper objectDetectorHelper;
private ExecutorService objectDetectionExecutor;
@@ -178,8 +176,22 @@ public static int getBandwidth(Context context) {
Context.MODE_PRIVATE).getInt("bandwidth", 20);
}
- public static String wirelessInfo() {
- int address = wifiManager.getConnectionInfo().getIpAddress();
+ /**
+ * IPv4 address of the phone/headset on its own wifi, used to tell the pilot where to
+ * push a stream when no adapter is attached. Returns null when there is nothing to
+ * report - a headset in particular may have no WifiManager state to read at all.
+ */
+ public static String wirelessInfo(Context context) {
+ WifiManager manager =
+ (WifiManager) context.getApplicationContext().getSystemService(Context.WIFI_SERVICE);
+ if (manager == null) {
+ return null;
+ }
+ WifiInfo info = manager.getConnectionInfo();
+ if (info == null) {
+ return null;
+ }
+ int address = info.getIpAddress();
return (address == 0) ? null : Formatter.formatIpAddress(address);
}
@@ -284,6 +296,20 @@ protected void onCreate(Bundle savedInstanceState) {
Log.d(TAG, "lifecycle onCreate");
super.onCreate(savedInstanceState);
+ // The USB_DEVICE_ATTACHED filter is on this activity, so replugging the adapter
+ // starts it even while the immersive activity owns the link. Two activities fighting
+ // over one adapter and one decoder means neither gets video, so step aside.
+ if (XrVideoActivity.isActive()
+ && UsbManager.ACTION_USB_DEVICE_ATTACHED.equals(getIntent().getAction())) {
+ Log.i(TAG, "adapter attached while the immersive session is running, staying out of it");
+ finish();
+ return;
+ }
+ ModeOwner.claim(this);
+
+ // Keep the immersive launcher entry in step with what this device can run.
+ XrVideoActivity.syncLauncherEntry(this);
+
// UI Setup
initializeUI();
@@ -332,7 +358,6 @@ private void initializeUI() {
getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
getWindow().getDecorView().setSystemUiVisibility(View.SYSTEM_UI_FLAG_HIDE_NAVIGATION);
- wifiManager = (WifiManager) getSystemService(WIFI_SERVICE);
}
// ----------------------------------------------------------------------------
@@ -343,12 +368,13 @@ private void initializeUI() {
* Initializes WFB-NG related logic such as setting default gs.key and linking
* to WFB-NG stats changes.
*/
+ /**
+ * The adapter, the link and the decoder live in {@link LinkService}; this only makes sure
+ * the key it needs exists before the service constructs the native side.
+ */
private void initializeWfbNg() {
setDefaultGsKey();
copyGSKey();
- wfbLink = new WfbNgLink(this);
- wfbLink.SetWfbNGStatsChanged(this);
- wfbLinkManager = new WfbLinkManager(this, binding, wfbLink);
}
// ----------------------------------------------------------------------------
@@ -359,10 +385,6 @@ private void initializeWfbNg() {
* Initializes VideoPlayer and configures surfaces for VR or standard mode.
*/
private void initializeVideoPlayers() {
- videoPlayer = new VideoPlayer(this);
- videoPlayer.setIVideoParamsChanged(this);
- videoPlayer.setLowLatency(getLowLatencySetting(this));
-
isVRMode = getVRSetting();
if (isVRMode) {
@@ -378,8 +400,8 @@ private void initializeVideoPlayers() {
private void setupVRVideoPlayers() {
binding.mainVideo.setVisibility(View.GONE);
binding.mainVideoSurface.setVisibility(View.GONE);
- binding.surfaceViewLeft.getHolder().addCallback(videoPlayer.configure1(0));
- binding.surfaceViewRight.getHolder().addCallback(videoPlayer.configure1(1));
+ binding.surfaceViewLeft.getHolder().addCallback(surfaceCallback(0));
+ binding.surfaceViewRight.getHolder().addCallback(surfaceCallback(1));
}
/**
@@ -398,14 +420,74 @@ private void setupStandardVideoPlayer() {
if (videoUsesTextureView) {
binding.mainVideoSurface.setVisibility(View.GONE);
binding.mainVideo.setVisibility(View.VISIBLE);
- binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0));
+ binding.mainVideo.setSurfaceTextureListener(surfaceTextureListener(0));
} else {
binding.mainVideo.setVisibility(View.GONE);
binding.mainVideoSurface.setVisibility(View.VISIBLE);
- binding.mainVideoSurface.getHolder().addCallback(videoPlayer.configure1(0));
+ binding.mainVideoSurface.getHolder().addCallback(surfaceCallback(0));
+ }
+ }
+
+ /**
+ * Hands a Surface to the service and takes it back again. Only the decoder's output
+ * changes; reception and the adapter are untouched, which is what makes switching modes
+ * and surviving a pause free of a reconnect.
+ */
+ private SurfaceHolder.Callback surfaceCallback(final int index) {
+ return new SurfaceHolder.Callback() {
+ @Override
+ public void surfaceCreated(SurfaceHolder holder) {
+ provideSurface(holder.getSurface(), index);
+ }
+
+ @Override
+ public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+ }
+
+ @Override
+ public void surfaceDestroyed(SurfaceHolder holder) {
+ withdrawSurface(index);
+ }
+ };
+ }
+
+ private void provideSurface(Surface surface, int index) {
+ pendingSurfaces[index] = surface;
+ if (link != null) {
+ link.attachSurface(surface, index);
}
}
+ private void withdrawSurface(int index) {
+ pendingSurfaces[index] = null;
+ if (link != null) {
+ link.detachSurface(index);
+ }
+ }
+
+ private TextureView.SurfaceTextureListener surfaceTextureListener(final int index) {
+ return new TextureView.SurfaceTextureListener() {
+ @Override
+ public void onSurfaceTextureAvailable(SurfaceTexture texture, int width, int height) {
+ provideSurface(new Surface(texture), index);
+ }
+
+ @Override
+ public void onSurfaceTextureSizeChanged(SurfaceTexture texture, int width, int height) {
+ }
+
+ @Override
+ public boolean onSurfaceTextureDestroyed(SurfaceTexture texture) {
+ withdrawSurface(index);
+ return true;
+ }
+
+ @Override
+ public void onSurfaceTextureUpdated(SurfaceTexture texture) {
+ }
+ };
+ }
+
// ----------------------------------------------------------------------------
// VR SEEK BARS (only in VR mode)
// ----------------------------------------------------------------------------
@@ -606,6 +688,9 @@ private void showSettingsMenu(View anchor) {
// VR submenu
setupVRSubMenu(popup);
+ // Immersive OpenXR submenu, headsets only
+ setupXrSubMenu(popup);
+
// Channel submenu
setupChannelSubMenu(popup);
@@ -659,6 +744,77 @@ private void setupVRSubMenu(PopupMenu popup) {
});
}
+ /**
+ * Submenu for the immersive OpenXR mode. Only offered on a device that looks like a
+ * headset - on a phone there is no runtime to talk to, so the entry stays hidden.
+ */
+ private void setupXrSubMenu(PopupMenu popup) {
+ if (!XrGoggleSession.isSupportedDevice(this)) {
+ return;
+ }
+ SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE);
+ SubMenu xrMenu = popup.getMenu().addSubMenu("VR goggles (OpenXR)");
+
+ xrMenu.add("Enter VR").setOnMenuItemClickListener(item -> {
+ startActivity(new Intent(this, XrVideoActivity.class));
+ return true;
+ });
+
+ MenuItem headLocked = xrMenu.add("Head locked");
+ headLocked.setCheckable(true);
+ headLocked.setChecked(prefs.getBoolean(XrVideoActivity.PREF_HEAD_LOCKED, true));
+ headLocked.setOnMenuItemClickListener(item -> {
+ boolean enabled = !item.isChecked();
+ item.setChecked(enabled);
+ prefs.edit().putBoolean(XrVideoActivity.PREF_HEAD_LOCKED, enabled).apply();
+ item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW);
+ item.setActionView(new View(this));
+ return false;
+ });
+
+ MenuItem passthrough = xrMenu.add("Passthrough");
+ passthrough.setCheckable(true);
+ passthrough.setChecked(prefs.getBoolean(XrVideoActivity.PREF_PASSTHROUGH, false));
+ passthrough.setOnMenuItemClickListener(item -> {
+ boolean enabled = !item.isChecked();
+ item.setChecked(enabled);
+ prefs.edit().putBoolean(XrVideoActivity.PREF_PASSTHROUGH, enabled).apply();
+ item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW);
+ item.setActionView(new View(this));
+ return false;
+ });
+
+ MenuItem sharpening = xrMenu.add("Layer sharpening");
+ sharpening.setCheckable(true);
+ sharpening.setChecked(prefs.getBoolean(XrVideoActivity.PREF_SHARPENING, true));
+ sharpening.setOnMenuItemClickListener(item -> {
+ boolean enabled = !item.isChecked();
+ item.setChecked(enabled);
+ prefs.edit().putBoolean(XrVideoActivity.PREF_SHARPENING, enabled).apply();
+ item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW);
+ item.setActionView(new View(this));
+ return false;
+ });
+
+ SubMenu refresh = xrMenu.addSubMenu("Display refresh rate");
+ float currentHz = prefs.getFloat(XrVideoActivity.PREF_REFRESH_HZ, 0f);
+ final float[] options = {0f, 72f, 90f, 120f};
+ for (float hz : options) {
+ String label = hz <= 0f ? "Auto (highest)" : String.format(Locale.US, "%.0f Hz", hz);
+ MenuItem entry = refresh.add(label);
+ entry.setCheckable(true);
+ entry.setChecked(Math.abs(currentHz - hz) < 0.5f);
+ final float selected = hz;
+ entry.setOnMenuItemClickListener(item -> {
+ prefs.edit().putFloat(XrVideoActivity.PREF_REFRESH_HZ, selected).apply();
+ return true;
+ });
+ }
+
+ MenuItem help = xrMenu.add("Controls: A recenter, B passthrough, X record, Y lock");
+ help.setEnabled(false);
+ }
+
/**
* Submenu that lists available channels and allows the user to select one.
*/
@@ -716,7 +872,7 @@ private void setupVideoSubMenu(PopupMenu popup) {
item.setChecked(enabled);
getSharedPreferences("general", MODE_PRIVATE).edit()
.putBoolean("low_latency_decoder", enabled).apply();
- videoPlayer.setLowLatency(enabled);
+ if (link != null) link.setLowLatency(enabled);
Toast.makeText(this, "Low latency " + (enabled ? "enabled" : "disabled")
+ ", applies on next video start.", Toast.LENGTH_SHORT).show();
item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW);
@@ -791,8 +947,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) {
SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit();
editor.putBoolean("adaptive_link_enabled", newState);
editor.apply();
- // Call instance method on the WfbNgLink instance via the wfbLinkManager.
- wfbLink.nativeSetAdaptiveLinkEnabled(newState);
+ if (link != null && link.link() != null) link.link().nativeSetAdaptiveLinkEnabled(newState);
return true;
});
@@ -814,8 +969,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) {
SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit();
editor.putInt("adaptive_tx_power", power);
editor.apply();
- // Call instance method on the WfbNgLink instance via the wfbLinkManager.
- wfbLink.nativeSetTxPower(power);
+ if (link != null && link.link() != null) link.link().nativeSetTxPower(power);
return true;
});
}
@@ -832,8 +986,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) {
SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit();
editor.putBoolean("custom_fec_enabled", newState);
editor.apply();
- // Call instance method on the WfbNgLink instance via the wfbLinkManager.
- wfbLink.nativeSetUseFec(newState ? 1 : 0);
+ if (link != null && link.link() != null) link.link().nativeSetUseFec(newState ? 1 : 0);
return true;
});
@@ -848,7 +1001,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) {
SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit();
editor.putBoolean("custom_ldpc_enabled", newState);
editor.apply();
- wfbLink.nativeSetUseLdpc(newState ? 1 : 0);
+ if (link != null && link.link() != null) link.link().nativeSetUseLdpc(newState ? 1 : 0);
return true;
});
@@ -863,7 +1016,7 @@ private void setupAdaptiveLinkSubMenu(PopupMenu popup) {
SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit();
editor.putBoolean("custom_stbc_enabled", newState);
editor.apply();
- wfbLink.nativeSetUseStbc(newState ? 1 : 0);
+ if (link != null && link.link() != null) link.link().nativeSetUseStbc(newState ? 1 : 0);
return true;
});
@@ -932,35 +1085,17 @@ private void showFecThresholdsDialog() {
.show();
}
+ /** The service applies these when it constructs the link; this re-applies after edits. */
void initDefaultOptions() {
- SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE);
- boolean adaptiveEnabled = prefs.getBoolean("adaptive_link_enabled", true);
- int adaptiveTxPower = prefs.getInt("adaptive_tx_power", 20);
- wfbLink.nativeSetAdaptiveLinkEnabled(adaptiveEnabled);
- wfbLink.nativeSetTxPower(adaptiveTxPower);
- boolean fecEnabled = prefs.getBoolean("custom_fec_enabled", true);
- wfbLink.nativeSetUseFec(fecEnabled ? 1 : 0);
-
- // LDPC and STBC default options
- boolean ldpcEnabled = prefs.getBoolean("custom_ldpc_enabled", true);
- wfbLink.nativeSetUseLdpc(ldpcEnabled ? 1 : 0);
-
- boolean stbcEnabled = prefs.getBoolean("custom_stbc_enabled", true);
- wfbLink.nativeSetUseStbc(stbcEnabled ? 1 : 0);
-
- setFecThresholdsFromPrefs();
+ if (link != null) {
+ WfbOptions.applyDefaults(this, link.link());
+ }
}
// Read FEC thresholds from prefs and call native method to apply
private void setFecThresholdsFromPrefs() {
- SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE);
- int lostTo5 = prefs.getInt("fec_lost_to_5", 2);
- int recTo4 = prefs.getInt("fec_recovered_to_4", 30);
- int recTo3 = prefs.getInt("fec_recovered_to_3", 24);
- int recTo2 = prefs.getInt("fec_recovered_to_2", 14);
- int recTo1 = prefs.getInt("fec_recovered_to_1", 8);
- if (wfbLink != null) {
- wfbLink.setFecThresholds(lostTo5, recTo4, recTo3, recTo2, recTo1);
+ if (link != null) {
+ WfbOptions.applyFecThresholds(this, link.link());
}
}
@@ -1097,8 +1232,8 @@ private void updateUdpForwardingState() {
String ip = prefs.getString("forward_udp_ip", "192.168.1.100");
int port = prefs.getInt("forward_udp_port", 5600);
- if (videoPlayer != null) {
- videoPlayer.setUdpForwarding(ip, port, enabled);
+ if (link != null) {
+ link.setUdpForwarding(ip, port, enabled);
}
}
@@ -1267,24 +1402,12 @@ private void startVpnService() {
}
private Uri openDvrFile() {
- String dvrFolder = getSharedPreferences("general",
- Context.MODE_PRIVATE).getString("dvr_folder_", "");
- if (dvrFolder.isEmpty()) {
- Log.e(TAG, "dvrFolder is empty");
- return null;
- }
- Uri uri = Uri.parse(dvrFolder);
- DocumentFile pickedDir = DocumentFile.fromTreeUri(this, uri);
- if (pickedDir != null && pickedDir.canWrite()) {
- LocalDateTime now = LocalDateTime.now();
- String filename = getDvrFileName(getDvrFileNameTemplate(), now) + ".mp4";
- DocumentFile newFile = pickedDir.createFile("video/mp4", filename);
+ String filename = DvrFiles.nextFileName(this);
+ Uri target = DvrFiles.createRecording(this);
+ if (target != null) {
Toast.makeText(this, "Recording to " + filename, Toast.LENGTH_SHORT).show();
- if (newFile == null)
- Log.e(TAG, "dvr newFile null");
- return newFile != null ? newFile.getUri() : null;
}
- return null;
+ return target;
}
private void startStopDvr() {
@@ -1293,10 +1416,7 @@ private void startStopDvr() {
if (dvrUri != null) {
startDvr(dvrUri);
} else {
- wfbLinkManager.stopAdapters();
- videoPlayer.stop();
- videoPlayer.stopAudio();
-
+ // No need to tear the link down for a file picker any more.
Intent intent = new Intent(Intent.ACTION_OPEN_DOCUMENT_TREE);
intent.addCategory(Intent.CATEGORY_DEFAULT);
intent.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION |
@@ -1315,7 +1435,7 @@ private void startDvr(Uri dvrUri) {
}
try {
dvrFd = getContentResolver().openFileDescriptor(dvrUri, "rw");
- videoPlayer.startDvr(dvrFd.getFd(), getDvrMP4());
+ if (link != null) link.startDvr(dvrFd.getFd(), getDvrMP4());
binding.imgBtnRecord.setImageResource(R.drawable.recording);
} catch (IOException e) {
Log.e(TAG, "Failed to open dvr file ", e);
@@ -1352,7 +1472,7 @@ private void stopDvr() {
}
binding.imgRecIndicator.setVisibility(View.INVISIBLE);
binding.imgBtnRecord.setImageResource(R.drawable.record);
- videoPlayer.stopDvr();
+ if (link != null) link.stopDvr();
if (recordTimer != null) {
recordTimer.cancel();
recordTimer.purge();
@@ -1384,7 +1504,7 @@ protected void onActivityResult(int requestCode, int resultCode, @Nullable Inten
InputStream inputStream = getContentResolver().openInputStream(uri);
setGsKey(inputStream);
copyGSKey();
- wfbLinkManager.refreshKey();
+ if (link != null) link.refreshKey();
inputStream.close();
} catch (IOException e) {
Log.e(TAG, "Failed to import gs.key from " + uri);
@@ -1430,40 +1550,19 @@ protected void onActivityResult(int requestCode, int resultCode, @Nullable Inten
}
public void setDefaultGsKey() {
- if (getGsKey().length > 0) {
- Log.d(TAG, "gs.key already saved in preferences.");
- return;
- }
- try {
- Log.d(TAG, "Importing default gs.key...");
- InputStream inputStream = getAssets().open("gs.key");
- setGsKey(inputStream);
- inputStream.close();
- } catch (IOException e) {
- Log.e(TAG, "Failed to import default gs.key");
- }
+ GsKey.importDefaultIfMissing(this);
}
public byte[] getGsKey() {
- String pref = getSharedPreferences("general", Context.MODE_PRIVATE).getString("gs.key", "");
- return Base64.decode(pref, Base64.DEFAULT);
+ return GsKey.get(this);
}
public void setGsKey(InputStream inputStream) throws IOException {
- ByteArrayOutputStream result = new ByteArrayOutputStream();
- byte[] buffer = new byte[1024];
- int length;
- while ((length = inputStream.read(buffer)) != -1) {
- result.write(buffer, 0, length);
- }
- SharedPreferences prefs = getSharedPreferences("general", Context.MODE_PRIVATE);
- SharedPreferences.Editor editor = prefs.edit();
- editor.putString("gs.key", Base64.encodeToString(result.toByteArray(), Base64.DEFAULT));
- editor.apply();
+ GsKey.set(this, inputStream);
}
public boolean getDvrMP4() {
- return getSharedPreferences("general", Context.MODE_PRIVATE).getBoolean("dvr_fmp4", true);
+ return DvrFiles.fragmentedMp4(this);
}
public void setDvrMP4(boolean enabled) {
@@ -1474,27 +1573,17 @@ public void setDvrMP4(boolean enabled) {
}
@SuppressLint("UnspecifiedRegisterReceiverFlag")
+ /** Only the battery overlay: the USB receiver belongs to LinkService with the adapter. */
public void registerReceivers() {
- IntentFilter usbFilter = new IntentFilter();
- usbFilter.addAction(UsbManager.ACTION_USB_DEVICE_ATTACHED);
- usbFilter.addAction(UsbManager.ACTION_USB_DEVICE_DETACHED);
- usbFilter.addAction(WfbLinkManager.ACTION_USB_PERMISSION);
IntentFilter batFilter = new IntentFilter(Intent.ACTION_BATTERY_CHANGED);
-
if (Build.VERSION.SDK_INT >= 33) {
- registerReceiver(wfbLinkManager, usbFilter, Context.RECEIVER_NOT_EXPORTED);
registerReceiver(batteryReceiver, batFilter, Context.RECEIVER_NOT_EXPORTED);
} else {
- registerReceiver(wfbLinkManager, usbFilter);
registerReceiver(batteryReceiver, batFilter);
}
}
public void unregisterReceivers() {
- try {
- unregisterReceiver(wfbLinkManager);
- } catch (IllegalArgumentException ignored) {
- }
try {
unregisterReceiver(batteryReceiver);
} catch (IllegalArgumentException ignored) {
@@ -1506,18 +1595,17 @@ protected void onPause() {
super.onPause();
stopObjectDetectionLoop();
-
unregisterReceivers();
- videoPlayer.stop();
- videoPlayer.stopAudio();
- wfbLinkManager.stopAdapters();
+ // The link deliberately stays up: it belongs to LinkService now. Tearing it down
+ // here is what used to cost seconds of black screen every time a system menu or a
+ // dialog took focus.
+ }
- // Stop VPN service
- Log.w(TAG, "onPause: stopping service");
- Intent intent = new Intent(this, WfbNgVpnService.class);
- intent.setAction("STOP_SERVICE");
- startService(intent);
+ @Override
+ protected void onDestroy() {
+ ModeOwner.release(this);
+ super.onDestroy();
}
@Override
@@ -1525,9 +1613,6 @@ protected void onStop() {
MavlinkNative.nativeStop(this);
handler.removeCallbacks(runnable);
unregisterReceivers();
- wfbLinkManager.stopAdapters();
- videoPlayer.stop();
- videoPlayer.stopAudio();
super.onStop();
}
@@ -1535,16 +1620,11 @@ protected void onStop() {
protected void onResume() {
registerReceivers();
- wfbLinkManager.setChannel(getChannel(this));
- wfbLinkManager.setBandwidth(getBandwidth(this));
-
- // On resume is called when the app is reopened, a device might have been plugged since the last time it started.
- wfbLinkManager.refreshAdapters();
-
- wfbLinkManager.startAdapters();
- videoPlayer.start();
- updateUdpForwardingState();
- videoPlayer.startAudio();
+ // An adapter may have been plugged in while we were away.
+ if (link != null) {
+ link.ensureAdapters();
+ updateUdpForwardingState();
+ }
SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE);
boolean odEnabled = prefs.getBoolean("od_enabled", false);
@@ -1557,6 +1637,18 @@ protected void onResume() {
super.onResume();
}
+ @Override
+ protected void onLinkServiceConnected(LinkService service) {
+ // Whatever the view hierarchy produced while we were still binding.
+ for (int i = 0; i < pendingSurfaces.length; i++) {
+ if (pendingSurfaces[i] != null) {
+ service.attachSurface(pendingSurfaces[i], i);
+ }
+ }
+ service.ensureAdapters();
+ updateUdpForwardingState();
+ }
+
@Override
public void onChannelSettingChanged(int channel) {
int currentChannel = getChannel(this);
@@ -1567,9 +1659,9 @@ public void onChannelSettingChanged(int channel) {
SharedPreferences.Editor editor = prefs.edit();
editor.putInt("wifi-channel", channel);
editor.apply();
- wfbLinkManager.stopAdapters();
- wfbLinkManager.setChannel(channel);
- wfbLinkManager.startAdapters();
+ if (link != null) {
+ link.setChannel(channel);
+ }
}
@Override
@@ -1582,13 +1674,13 @@ public void onBandwidthSettingChanged(int bandwidth) {
SharedPreferences.Editor editor = prefs.edit();
editor.putInt("bandwidth", bandwidth);
editor.apply();
- wfbLinkManager.stopAdapters();
- wfbLinkManager.setBandwidth(bandwidth);
- wfbLinkManager.startAdapters();
+ if (link != null) {
+ link.setBandwidth(bandwidth);
+ }
}
@Override
- public void onVideoRatioChanged(final int videoW, final int videoH) {
+ public void onVideoResolution(final int videoW, final int videoH) {
lastVideoW = videoW;
lastVideoH = videoH;
@@ -1617,7 +1709,7 @@ private void updateViewRatio(int viewId, int videoW, int videoH) {
}
@Override
- public void onDecodingInfoChanged(final DecodingInfo decodingInfo) {
+ public void onDecodingInfo(final DecodingInfo decodingInfo) {
mDecodingInfo = decodingInfo;
runOnUiThread(() -> {
if (lastCodec != decodingInfo.nCodec) {
@@ -1638,7 +1730,7 @@ public void onDecodingInfoChanged(final DecodingInfo decodingInfo) {
}
@Override
- public void onWfbNgStatsChanged(WfbNGStats data) {
+ public void onWfbStats(WfbNGStats data) {
runOnUiThread(() -> {
if (data.count_p_all > 0) {
binding.tvMessage.setVisibility(View.INVISIBLE);
@@ -1704,30 +1796,29 @@ public void onWfbNgStatsChanged(WfbNGStats data) {
});
}
+ @Override
+ public void onLinkStatus(String message) {
+ runOnUiThread(() -> {
+ binding.tvMessage.setVisibility(View.VISIBLE);
+ binding.tvMessage.setText(message);
+ });
+ }
+
+ @Override
+ public void onLocalStreamHint(String url) {
+ runOnUiThread(() -> {
+ binding.wifiMessage.setText(url);
+ binding.wifiMessage.setVisibility(View.VISIBLE);
+ });
+ }
+
@Override
public void onNewMavlinkData(MavlinkData data) {
runOnUiThread(() -> osdManager.render(data));
}
private void copyGSKey() {
- File file = new File(getApplicationContext().getFilesDir(), "gs.key");
- OutputStream out = null;
- try {
- byte[] keyBytes = getGsKey();
- Log.d(TAG, "Using gs.key:" + bytesToHex(keyBytes) + "; Copying to" + file.getAbsolutePath());
- out = new FileOutputStream(file);
- out.write(keyBytes, 0, keyBytes.length);
- } catch (IOException e) {
- Log.e(TAG, "Failed to copy asset", e);
- } finally {
- if (out != null) {
- try {
- out.close();
- } catch (IOException e) {
- // NOOP
- }
- }
- }
+ GsKey.writeToFilesDir(this);
}
private void showLoginCredentialsDialog() {
@@ -1832,22 +1923,7 @@ public void afterTextChanged(Editable editable) {}
// pixelpilot_[yyyyMMdd-HHmmss]
// ".mp4" will append later
private String getDvrFileName(String template, LocalDateTime time) {
- Matcher matcher = Pattern.compile("\\[([^\\]]*)\\]").matcher(template);
- String fallbackTime = time.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss"));
-
- if (matcher.find()) {
- String prefix = template.substring(0, matcher.start());
- String pattern = matcher.group(1);
- String suffix = template.substring(matcher.end());
-
- try {
- String timePart = time.format(DateTimeFormatter.ofPattern(pattern));
- return prefix + timePart + suffix;
- } catch (IllegalArgumentException e) {
- return prefix + fallbackTime + suffix;
- }
- }
- return "pixelpilot_" + fallbackTime;
+ return DvrFiles.fileName(template, time);
}
// Helper method to retrieve the drone username
@@ -1856,9 +1932,8 @@ private String getDroneUsername() {
return getSharedPreferences("general", Context.MODE_PRIVATE).getString(PREF_DRONE_USERNAME, "root");
}
- private String getDvrFileNameTemplate()
- {
- return getSharedPreferences("general", Context.MODE_PRIVATE).getString(PREF_DVR_FILENAME, "pixelpilot_[yyyyMMdd-HHmmss]");
+ private String getDvrFileNameTemplate() {
+ return DvrFiles.template(this);
}
// Helper method to save the drone username
@@ -1870,10 +1945,7 @@ private void setDroneUsername(String username) {
}
private void setDvrFileName(String fileName) {
- SharedPreferences prefs = getSharedPreferences("general", Context.MODE_PRIVATE);
- SharedPreferences.Editor editor = prefs.edit();
- editor.putString(PREF_DVR_FILENAME, fileName);
- editor.apply();
+ DvrFiles.setTemplate(this, fileName);
}
// Helper method to retrieve the drone password
diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java
index 0c0390e0..e1fa0dae 100644
--- a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java
+++ b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java
@@ -7,9 +7,7 @@
import android.hardware.usb.UsbDevice;
import android.hardware.usb.UsbManager;
import android.util.Log;
-import android.view.View;
-import com.openipc.pixelpilot.databinding.ActivityVideoBinding;
import com.openipc.wfbngrtl8812.WfbNgLink;
import java.util.HashMap;
@@ -20,9 +18,12 @@
public class WfbLinkManager extends BroadcastReceiver {
public static final String ACTION_USB_PERMISSION = "com.openipc.pixelpilot.USB_PERMISSION";
private static final String TAG = "pixelpilot";
- static Map activeWifiAdapters = new HashMap<>();
+ // Not static: it used to be, and with two activities each owning its own WfbNgLink
+ // that shared map let both of them decide the adapter still needed starting, so the
+ // same dongle ended up with several RX loops.
+ private final Map activeWifiAdapters = new HashMap<>();
private final WfbNgLink wfbLink;
- private final ActivityVideoBinding binding;
+ private final LinkStatusView status;
private final Context context;
private int wifiChannel;
private Bandwidth bandWidth;
@@ -42,8 +43,8 @@ public int getValue() {
}
}
- public WfbLinkManager(Context context, ActivityVideoBinding binding, WfbNgLink wfbNgLink) {
- this.binding = binding;
+ public WfbLinkManager(Context context, LinkStatusView status, WfbNgLink wfbNgLink) {
+ this.status = status;
this.context = context;
this.wfbLink = wfbNgLink;
}
@@ -102,7 +103,12 @@ public Map getAttachedAdapters() {
}
Map res = new HashMap<>();
- for (UsbDevice dev : manager.getDeviceList().values()) {
+ Map attached = manager.getDeviceList();
+ // Without this there is no way to tell "nothing is plugged in" from "the dongle is
+ // there but its id is not in usb_device_filter.xml", and the id is what a bug report
+ // needs in order to add it.
+ Log.i(TAG, "usb devices attached: " + attached.size());
+ for (UsbDevice dev : attached.values()) {
boolean allowed = false;
for (UsbDeviceFilter filter : filters) {
if (filter.productId == dev.getProductId() && filter.vendorId == dev.getVendorId()) {
@@ -110,6 +116,13 @@ public Map getAttachedAdapters() {
break;
}
}
+ Log.i(TAG, String.format(" %s %04X:%04X %s %s -> %s",
+ dev.getDeviceName(),
+ dev.getVendorId(),
+ dev.getProductId(),
+ String.valueOf(dev.getManufacturerName()),
+ String.valueOf(dev.getProductName()),
+ allowed ? "supported" : "NOT in usb_device_filter.xml"));
if (!allowed) {
continue;
}
@@ -130,8 +143,7 @@ public synchronized void refreshAdapters() {
(android.hardware.usb.UsbManager) context.getSystemService(Context.USB_SERVICE);
for (Map.Entry entry : attachedAdapters.entrySet()) {
if (!usbManager.hasPermission(entry.getValue())) {
- binding.tvMessage.setVisibility(View.VISIBLE);
- binding.tvMessage.setText("No permission for wifi adapter(s) " + entry.getValue().getDeviceName());
+ status.showLinkMessage("No permission for wifi adapter(s) " + entry.getValue().getDeviceName());
// Android 14 refuses to deliver a PendingIntent built from an implicit
// intent to a runtime registered receiver, so the permission result never
// arrives unless the package is set explicitly.
@@ -160,6 +172,7 @@ public synchronized void refreshAdapters() {
}
// Starts newly attached adapters.
+ boolean startFailed = false;
for (Map.Entry entry : attachedAdapters.entrySet()) {
if (activeWifiAdapters.containsKey(entry.getKey())) {
continue;
@@ -168,23 +181,34 @@ public synchronized void refreshAdapters() {
// is never retried on the next refresh.
if (startAdapter(entry.getValue())) {
activeWifiAdapters.put(entry.getKey(), entry.getValue());
+ } else {
+ startFailed = true;
}
}
if (activeWifiAdapters.isEmpty()) {
- String text = "No compatible wifi adapter found.";
- binding.tvMessage.setText(text);
- binding.tvMessage.setVisibility(View.VISIBLE);
+ // These are three different problems and telling the pilot the wrong one costs
+ // a lot of pointless searching.
+ if (startFailed) {
+ status.showLinkMessage("Wifi adapter found but could not be started - see the log.");
+ } else if (attachedAdapters.isEmpty()) {
+ status.showLinkMessage("No compatible wifi adapter found.");
+ } else {
+ status.showLinkMessage("Waiting for wifi adapter permission…");
+ }
- String wifi = VideoActivity.wirelessInfo();
+ String wifi = VideoActivity.wirelessInfo(context);
if (wifi != null) {
- String local = "udp://" + wifi + ":5600";
- binding.wifiMessage.setText(local);
- binding.wifiMessage.setVisibility(View.VISIBLE);
+ status.showLocalStreamHint("udp://" + wifi + ":5600");
}
}
}
+ /** True when at least one adapter is up, so a caller can retry instead of guessing. */
+ public synchronized boolean hasActiveAdapter() {
+ return !activeWifiAdapters.isEmpty();
+ }
+
public synchronized void stopAdapters() {
try {
wfbLink.stopAll();
@@ -212,12 +236,10 @@ public synchronized void startAdapters() {
}
public synchronized boolean startAdapter(UsbDevice dev) {
- binding.tvMessage.setVisibility(View.VISIBLE);
- String text = "Starting wfb-ng channel " + wifiChannel + " with " + String.format(
- "[%04X", dev.getVendorId()) + ":" + String.format("%04X]", dev.getProductId());
- binding.tvMessage.setText(text);
+ status.showLinkMessage("Starting wfb-ng channel " + wifiChannel + " with " + String.format(
+ "[%04X", dev.getVendorId()) + ":" + String.format("%04X]", dev.getProductId()));
if (!wfbLink.start(wifiChannel, bandWidth.getValue(), dev)) {
- binding.tvMessage.setText("Could not open wifi adapter " + dev.getDeviceName());
+ status.showLinkMessage("Could not open wifi adapter " + dev.getDeviceName());
return false;
}
return true;
diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbOptions.java b/app/src/main/java/com/openipc/pixelpilot/WfbOptions.java
new file mode 100644
index 00000000..a495587e
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/WfbOptions.java
@@ -0,0 +1,46 @@
+package com.openipc.pixelpilot;
+
+import android.content.Context;
+import android.content.SharedPreferences;
+
+import com.openipc.wfbngrtl8812.WfbNgLink;
+
+/**
+ * Pushes the persisted wfb-ng options into the native link.
+ *
+ * These are not read by the native side on its own, so any activity that constructs a
+ * {@link WfbNgLink} has to apply them explicitly or the user's adaptive link, TX power and
+ * FEC settings are silently ignored.
+ */
+public final class WfbOptions {
+ private WfbOptions() {
+ }
+
+ public static void applyDefaults(Context context, WfbNgLink wfbLink) {
+ if (wfbLink == null) {
+ return;
+ }
+ SharedPreferences prefs = context.getSharedPreferences("general", Context.MODE_PRIVATE);
+
+ wfbLink.nativeSetAdaptiveLinkEnabled(prefs.getBoolean("adaptive_link_enabled", true));
+ wfbLink.nativeSetTxPower(prefs.getInt("adaptive_tx_power", 20));
+ wfbLink.nativeSetUseFec(prefs.getBoolean("custom_fec_enabled", true) ? 1 : 0);
+ wfbLink.nativeSetUseLdpc(prefs.getBoolean("custom_ldpc_enabled", true) ? 1 : 0);
+ wfbLink.nativeSetUseStbc(prefs.getBoolean("custom_stbc_enabled", true) ? 1 : 0);
+
+ applyFecThresholds(context, wfbLink);
+ }
+
+ public static void applyFecThresholds(Context context, WfbNgLink wfbLink) {
+ if (wfbLink == null) {
+ return;
+ }
+ SharedPreferences prefs = context.getSharedPreferences("general", Context.MODE_PRIVATE);
+ wfbLink.setFecThresholds(
+ prefs.getInt("fec_lost_to_5", 2),
+ prefs.getInt("fec_recovered_to_4", 30),
+ prefs.getInt("fec_recovered_to_3", 24),
+ prefs.getInt("fec_recovered_to_2", 14),
+ prefs.getInt("fec_recovered_to_1", 8));
+ }
+}
diff --git a/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java
new file mode 100644
index 00000000..fa927a4e
--- /dev/null
+++ b/app/src/main/java/com/openipc/pixelpilot/XrVideoActivity.java
@@ -0,0 +1,625 @@
+package com.openipc.pixelpilot;
+
+import android.content.BroadcastReceiver;
+import android.content.ComponentName;
+import android.content.Context;
+import android.content.Intent;
+import android.content.IntentFilter;
+import android.content.SharedPreferences;
+import android.content.pm.PackageManager;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+import android.hardware.usb.UsbManager;
+import android.net.Uri;
+import android.os.Build;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.Looper;
+import android.os.ParcelFileDescriptor;
+import android.util.Log;
+import android.view.Gravity;
+import android.view.Surface;
+import android.view.View;
+import android.view.WindowManager;
+import android.widget.TextView;
+import android.widget.Toast;
+
+import androidx.annotation.Nullable;
+
+import com.openipc.wfbngrtl8812.WfbNGStats;
+import com.openipc.xr.XrGoggleSession;
+
+import java.io.IOException;
+
+/**
+ * Immersive ground station.
+ *
+ *
Instead of a 2D panel that the headset compositor resamples onto a quad, the decoder
+ * writes straight into a compositor swapchain
+ * ({@code xrCreateSwapchainAndroidSurfaceKHR}) which is presented as a head locked quad
+ * layer. That removes the view hierarchy, SurfaceFlinger and the panel's fixed resolution
+ * from the video path.
+ *
+ *
The adapter, the wfb-ng link and the decoder belong to {@link LinkService}, not to this
+ * activity. Entering and leaving immersive mode therefore only swaps the decoder's output
+ * surface - the adapter is never reopened, so there is no reconnect delay and no second USB
+ * permission prompt.
+ *
+ *
There is no in-headset menu yet: channel, bandwidth and the DVR folder are still
+ * configured in {@link VideoActivity} before putting the headset on. Controllers cover
+ * what matters in flight.
+ */
+public class XrVideoActivity extends LinkClientActivity implements XrGoggleSession.Listener {
+
+ private static final String TAG = "pixelpilot-xr";
+
+ /**
+ * Enables or disables this activity's launcher entry to match the device.
+ *
+ *
The entry is what lets Horizon OS start the app straight into VR, but the manifest
+ * cannot express "only on a device with an XR runtime", so it ships disabled and is
+ * switched on here. On a phone it stays off and nothing about the flat mode changes -
+ * no second icon, no way to land on an activity that cannot create a session.
+ *
+ *
Enabling the component is also what makes {@code startActivity} on it work at all,
+ * so this has to run before VideoActivity's menu can offer it - hence {@code onCreate}
+ * rather than when the menu is opened. Both are gated on the same predicate, so a device
+ * {@link XrGoggleSession#isSupportedDevice} does not recognise sees neither the entry nor
+ * the menu; if that ever turns out to be a false negative, widening that check is the fix.
+ */
+ static void syncLauncherEntry(Context context) {
+ boolean wantEnabled = XrGoggleSession.isSupportedDevice(context);
+ ComponentName component = new ComponentName(context, XrVideoActivity.class);
+ PackageManager pm = context.getPackageManager();
+ try {
+ // COMPONENT_ENABLED_STATE_DEFAULT means "whatever the manifest said", which here
+ // is disabled - so anything other than an explicit ENABLED counts as off.
+ boolean isEnabled = pm.getComponentEnabledSetting(component)
+ == PackageManager.COMPONENT_ENABLED_STATE_ENABLED;
+ if (isEnabled == wantEnabled) {
+ return;
+ }
+ pm.setComponentEnabledSetting(
+ component,
+ wantEnabled
+ ? PackageManager.COMPONENT_ENABLED_STATE_ENABLED
+ // Back to the manifest value rather than an explicit disable, so
+ // an unrecognised device is left exactly as it shipped.
+ : PackageManager.COMPONENT_ENABLED_STATE_DEFAULT,
+ PackageManager.DONT_KILL_APP);
+ Log.i(TAG, "immersive launcher entry " + (wantEnabled ? "enabled" : "disabled"));
+ } catch (Exception e) {
+ // A launcher entry is not worth taking the flat mode down over.
+ Log.w(TAG, "could not update the immersive launcher entry", e);
+ }
+ }
+
+ // Preference keys, all in the shared "general" store so the flat activity's settings
+ // menu can write them.
+ static final String PREF_HEAD_LOCKED = "xr_head_locked";
+ static final String PREF_PASSTHROUGH = "xr_passthrough";
+ static final String PREF_SHARPENING = "xr_sharpening";
+ static final String PREF_REFRESH_HZ = "xr_refresh_hz";
+ static final String PREF_QUAD_WIDTH = "xr_quad_width";
+ static final String PREF_QUAD_DISTANCE = "xr_quad_distance";
+ static final String PREF_QUAD_HEIGHT = "xr_quad_height";
+ static final String PREF_LAST_VIDEO_W = "xr_last_video_width";
+ static final String PREF_LAST_VIDEO_H = "xr_last_video_height";
+
+ /**
+ * Whether an immersive session owns the link right now. The USB_DEVICE_ATTACHED filter
+ * lives on VideoActivity, so replugging the adapter would otherwise start the flat
+ * activity alongside a running immersive session.
+ */
+ private static volatile boolean active;
+
+ static boolean isActive() {
+ return active;
+ }
+
+ private final Handler handler = new Handler(Looper.getMainLooper());
+
+ private XrGoggleSession xr;
+ private TextView statusView;
+
+ // The flat activity may still be releasing the USB interface when we get here, so the
+ // adapter is retried for a while rather than given up on after one attempt.
+ private static final int LINK_RETRY_INTERVAL_MS = 700;
+ private static final int LINK_RETRY_LIMIT = 20;
+
+ private int linkRetries;
+ private boolean xrStarting;
+ private boolean linkStarted;
+ private Surface xrSurface;
+ private volatile String lastStatus = "";
+ private boolean videoSeen;
+ private boolean statusScreenUsable = true;
+ private ParcelFileDescriptor dvrFd;
+
+ // Link quality haptics
+ private long lastHapticAt;
+ private boolean linkWasBad;
+ private volatile WfbNGStats lastStats;
+ private long lastStatsLogAt;
+
+ private final BroadcastReceiver usbReceiver = new BroadcastReceiver() {
+ @Override
+ public void onReceive(Context context, Intent intent) {
+ // Permission granted / adapter (re)appeared - re-evaluate whether we can go
+ // immersive now.
+ handler.post(XrVideoActivity.this::tryBringUpLink);
+ }
+ };
+
+ // ------------------------------------------------------------------------------
+ // lifecycle
+ // ------------------------------------------------------------------------------
+
+ @Override
+ protected void onCreate(@Nullable Bundle savedInstanceState) {
+ super.onCreate(savedInstanceState);
+ active = true;
+ // Evict the flat activity: it owns a WfbNgLink too, and the adapter can only belong
+ // to one of them.
+ ModeOwner.claim(this);
+ getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
+
+ // Only shown while still flat: USB permission, errors, "waiting for adapter".
+ statusView = new TextView(this);
+ statusView.setTextColor(Color.WHITE);
+ statusView.setBackgroundColor(Color.BLACK);
+ statusView.setTextSize(22f);
+ statusView.setGravity(Gravity.CENTER);
+ statusView.setText(R.string.xr_starting);
+ setContentView(statusView);
+
+ if (!XrGoggleSession.isLibraryLoaded()) {
+ failToFlat("This build has no OpenXR support");
+ return;
+ }
+
+ xr = new XrGoggleSession(this, this);
+ SharedPreferences prefs = prefs();
+ // Match the swapchain to the stream so the compositor never resamples twice. The
+ // resolution from the last session is a good guess; 1080p until we have seen one.
+ xr.setSwapchainSize(prefs.getInt(PREF_LAST_VIDEO_W, 1920), prefs.getInt(PREF_LAST_VIDEO_H, 1080));
+ xr.setHeadLocked(prefs.getBoolean(PREF_HEAD_LOCKED, true));
+ xr.setPassthrough(prefs.getBoolean(PREF_PASSTHROUGH, false));
+ xr.setSharpening(prefs.getBoolean(PREF_SHARPENING, true));
+ xr.setQuadWidth(prefs.getFloat(PREF_QUAD_WIDTH, 2.2f));
+ xr.setQuadDistance(prefs.getFloat(PREF_QUAD_DISTANCE, 1.6f));
+ xr.setQuadHeightOffset(prefs.getFloat(PREF_QUAD_HEIGHT, 0f));
+ }
+
+ @Override
+ protected void onLinkServiceConnected(LinkService service) {
+ if (xr == null) {
+ return;
+ }
+ registerUsbReceiver();
+ tryBringUpLink();
+ }
+
+ @Override
+ protected void onResume() {
+ super.onResume();
+ if (xr != null && link != null) {
+ tryBringUpLink();
+ }
+ }
+
+ // Deliberately no teardown in onPause/onStop. In an immersive session the runtime
+ // pauses and resumes us for its own reasons (system menu, proximity sensor), and
+ // dropping the USB link there costs seconds of black screen on the way back.
+
+ @Override
+ protected void onDestroy() {
+ active = false;
+ ModeOwner.release(this);
+ persistGeometry();
+ try {
+ unregisterReceiver(usbReceiver);
+ } catch (IllegalArgumentException ignored) {
+ // never registered
+ }
+ stopDvr();
+ // Hand the decoder's surface back but leave the link running: the flat activity is
+ // about to take over, and the whole point of the service is that the adapter stays
+ // open across the switch.
+ if (link != null) {
+ link.detachSurface(0);
+ }
+ if (xr != null) {
+ xr.release();
+ xr = null;
+ }
+ super.onDestroy();
+ }
+
+ private SharedPreferences prefs() {
+ return getSharedPreferences("general", Context.MODE_PRIVATE);
+ }
+
+ private void registerUsbReceiver() {
+ IntentFilter filter = new IntentFilter();
+ filter.addAction(WfbLinkManager.ACTION_USB_PERMISSION);
+ filter.addAction(UsbManager.ACTION_USB_DEVICE_ATTACHED);
+ filter.addAction(UsbManager.ACTION_USB_DEVICE_DETACHED);
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+ registerReceiver(usbReceiver, filter, Context.RECEIVER_NOT_EXPORTED);
+ } else {
+ registerReceiver(usbReceiver, filter);
+ }
+ }
+
+ // ------------------------------------------------------------------------------
+ // bring-up
+ // ------------------------------------------------------------------------------
+
+ /**
+ * Resolves USB permission while still on a flat panel - a permission dialog is far
+ * easier to deal with before the compositor takes over - and only then starts the
+ * immersive session.
+ */
+ private void tryBringUpLink() {
+ if (xr == null || link == null) {
+ return;
+ }
+ link.ensureAdapters();
+
+ // Going immersive does not depend on the link, and the pilot is better off seeing the
+ // status screen with a reason than a flat activity that says nothing.
+ if (!xrStarting) {
+ xrStarting = true;
+ statusView.setText(R.string.xr_entering);
+ xr.start();
+ }
+
+ // The adapter may still be waiting for a USB permission the service asked for, so
+ // keep checking back rather than giving up on the first look.
+ if (!link.hasActiveAdapter() && linkRetries < LINK_RETRY_LIMIT) {
+ linkRetries++;
+ Log.i(TAG, "adapter not up yet, retry " + linkRetries + "/" + LINK_RETRY_LIMIT);
+ handler.postDelayed(this::tryBringUpLink, LINK_RETRY_INTERVAL_MS);
+ } else if (link.hasActiveAdapter()) {
+ linkRetries = 0;
+ }
+ }
+
+ @Override
+ public void onXrReady(Surface videoSurface) {
+ Log.i(TAG, "immersive session up, handing the swapchain surface to the decoder");
+ xrSurface = videoSurface;
+ // Put something on the layer before the decoder owns it. Without this the panel is
+ // an opaque black rectangle until the first frame arrives, which is
+ // indistinguishable from "nothing is being composited at all".
+ drawStatusScreen();
+ // The decoder writes into the compositor swapchain from here on. Only the surface
+ // changes - the receivers and the adapter keep running in the service.
+ if (link != null) {
+ link.attachSurface(videoSurface, 0);
+ if (link.videoWidth() > 0) {
+ xr.setVideoResolution(link.videoWidth(), link.videoHeight());
+ }
+ }
+ applyRefreshRatePreference();
+ statusView.setVisibility(View.GONE);
+ linkStarted = true;
+ }
+
+ private void applyRefreshRatePreference() {
+ float wanted = prefs().getFloat(PREF_REFRESH_HZ, 0f);
+ if (wanted <= 0f) {
+ // "Auto" means the highest the headset offers: at 60 fps video a higher
+ // display rate only makes the cadence more even.
+ for (float hz : xr.refreshRates()) {
+ wanted = Math.max(wanted, hz);
+ }
+ }
+ if (wanted > 0f) {
+ xr.requestRefreshRate(wanted);
+ }
+ }
+
+ @Override
+ public void onXrButton(int button) {
+ switch (button) {
+ case XrGoggleSession.BUTTON_RECENTER:
+ // Applied natively already, nothing to persist.
+ break;
+ case XrGoggleSession.BUTTON_PASSTHROUGH:
+ prefs().edit().putBoolean(PREF_PASSTHROUGH, xr.isPassthroughEnabled()).apply();
+ break;
+ case XrGoggleSession.BUTTON_LOCK_MODE:
+ prefs().edit().putBoolean(PREF_HEAD_LOCKED, xr.isHeadLocked()).apply();
+ break;
+ case XrGoggleSession.BUTTON_RECORD:
+ toggleDvr();
+ break;
+ case XrGoggleSession.BUTTON_EXIT:
+ leaveVr();
+ break;
+ case XrGoggleSession.BUTTON_RAISE:
+ case XrGoggleSession.BUTTON_LOWER:
+ prefs().edit().putFloat(PREF_QUAD_HEIGHT, xr.quadHeightOffset()).apply();
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void onXrStopped(@Nullable String error) {
+ if (error != null) {
+ Log.e(TAG, "immersive session failed: " + error);
+ failToFlat(error);
+ } else {
+ finish();
+ }
+ }
+
+ /**
+ * Back to the flat activity. Cheap now that the link lives in the service: the adapter
+ * stays open and video keeps running, only the presentation changes.
+ */
+ private void leaveVr() {
+ Log.i(TAG, "leaving immersive mode on request");
+ startActivity(new Intent(this, VideoActivity.class));
+ finish();
+ }
+
+ /** Immersive mode is not usable here - say why and go back to the flat activity. */
+ private void failToFlat(String reason) {
+ Toast.makeText(this, "VR mode unavailable: " + reason, Toast.LENGTH_LONG).show();
+ if (statusView != null) {
+ statusView.setVisibility(View.VISIBLE);
+ statusView.setText(getString(R.string.xr_failed, reason));
+ }
+ handler.postDelayed(() -> {
+ startActivity(new Intent(this, VideoActivity.class));
+ finish();
+ }, 2500);
+ }
+
+ private void persistGeometry() {
+ if (xr == null) {
+ return;
+ }
+ prefs().edit()
+ .putFloat(PREF_QUAD_WIDTH, xr.quadWidth())
+ .putFloat(PREF_QUAD_DISTANCE, xr.quadDistance())
+ .putFloat(PREF_QUAD_HEIGHT, xr.quadHeightOffset())
+ .putBoolean(PREF_HEAD_LOCKED, xr.isHeadLocked())
+ .putBoolean(PREF_PASSTHROUGH, xr.isPassthroughEnabled())
+ .apply();
+ }
+
+ // ------------------------------------------------------------------------------
+ // "no signal" screen
+ // ------------------------------------------------------------------------------
+
+ /**
+ * Paints status straight into the compositor swapchain's Surface.
+ *
+ *
Only valid until MediaCodec becomes the producer for that Surface - a buffer queue
+ * has one producer - so this stops as soon as a frame has been decoded, and gives up
+ * quietly if the canvas is refused.
+ */
+ private void drawStatusScreen() {
+ if (videoSeen || !statusScreenUsable || xrSurface == null || !xrSurface.isValid()) {
+ return;
+ }
+ Canvas canvas = null;
+ try {
+ canvas = xrSurface.lockHardwareCanvas();
+ if (canvas == null) {
+ statusScreenUsable = false;
+ return;
+ }
+ paintStatus(canvas);
+ xrSurface.unlockCanvasAndPost(canvas);
+ } catch (Throwable t) {
+ // Most likely the decoder already connected as the producer.
+ Log.w(TAG, "status screen unavailable: " + t);
+ statusScreenUsable = false;
+ return;
+ }
+ handler.postDelayed(this::drawStatusScreen, 1000);
+ }
+
+ private void paintStatus(Canvas canvas) {
+ final int w = canvas.getWidth();
+ final int h = canvas.getHeight();
+
+ canvas.drawColor(Color.rgb(12, 12, 16));
+
+ Paint line = new Paint();
+ line.setColor(Color.rgb(70, 80, 100));
+ line.setStrokeWidth(Math.max(2f, h / 300f));
+ line.setStyle(Paint.Style.STROKE);
+ // A border and a centre cross: makes the panel's real extent and centre visible, so
+ // size and distance can be judged with nothing else on screen.
+ float inset = h / 40f;
+ canvas.drawRect(inset, inset, w - inset, h - inset, line);
+ float cx = w / 2f, cy = h / 2f, arm = h / 20f;
+ canvas.drawLine(cx - arm, cy, cx + arm, cy, line);
+ canvas.drawLine(cx, cy - arm, cx, cy + arm, line);
+
+ Paint title = new Paint();
+ title.setAntiAlias(true);
+ title.setColor(Color.rgb(235, 235, 240));
+ title.setTextSize(h / 14f);
+ title.setTextAlign(Paint.Align.CENTER);
+ canvas.drawText("No video", cx, cy - h / 8f, title);
+
+ Paint body = new Paint();
+ body.setAntiAlias(true);
+ body.setColor(Color.rgb(170, 180, 195));
+ body.setTextSize(h / 28f);
+ body.setTextAlign(Paint.Align.CENTER);
+
+ String status = lastStatus;
+ float y = cy + h / 7f;
+ if (!status.isEmpty()) {
+ canvas.drawText(status, cx, y, body);
+ y += h / 20f;
+ }
+ // The link counters answer the first question a pilot has when the screen stays
+ // empty: is anything arriving at all?
+ WfbNGStats stats = lastStats;
+ if (stats != null) {
+ canvas.drawText("wfb packets " + stats.count_p_all
+ + " ok " + stats.count_p_dec_ok
+ + " lost " + stats.count_p_lost
+ + " err " + stats.count_p_dec_err,
+ cx, y, body);
+ y += h / 20f;
+ } else {
+ canvas.drawText("no wfb statistics yet", cx, y, body);
+ y += h / 20f;
+ }
+ canvas.drawText("channel " + VideoActivity.getChannel(this)
+ + " @ " + VideoActivity.getBandwidth(this) + " MHz",
+ cx, y, body);
+ y += h / 20f;
+ String wifi = VideoActivity.wirelessInfo(this);
+ if (wifi != null) {
+ canvas.drawText("or push a stream to udp://" + wifi + ":5600", cx, y, body);
+ y += h / 20f;
+ }
+ canvas.drawText(xr != null && xr.isHeadLocked() ? "head locked" : "world locked", cx, y, body);
+ y += h / 20f;
+
+ Paint hint = new Paint(body);
+ hint.setTextSize(h / 36f);
+ hint.setColor(Color.rgb(130, 140, 155));
+ canvas.drawText("A recenter B passthrough stick click record", cx, y, hint);
+ y += h / 26f;
+ canvas.drawText("menu button or thumb swipe left: back to flat mode", cx, y, hint);
+ }
+
+ // ------------------------------------------------------------------------------
+ // DVR
+ // ------------------------------------------------------------------------------
+
+ private void toggleDvr() {
+ if (dvrFd != null) {
+ stopDvr();
+ xr.haptic(0.7f, 60);
+ Toast.makeText(this, "Recording stopped", Toast.LENGTH_SHORT).show();
+ return;
+ }
+ if (!DvrFiles.hasRecordingFolder(this)) {
+ // Picking a folder needs the flat activity; refuse rather than drop out of VR.
+ xr.haptic(1.0f, 250);
+ Toast.makeText(this, "Pick a DVR folder in flat mode first", Toast.LENGTH_LONG).show();
+ return;
+ }
+ Uri target = DvrFiles.createRecording(this);
+ if (target == null) {
+ xr.haptic(1.0f, 250);
+ Toast.makeText(this, "Could not create the recording file", Toast.LENGTH_LONG).show();
+ return;
+ }
+ if (link == null) {
+ return;
+ }
+ try {
+ dvrFd = getContentResolver().openFileDescriptor(target, "rw");
+ link.startDvr(dvrFd.getFd(), DvrFiles.fragmentedMp4(this));
+ xr.haptic(0.7f, 120);
+ Toast.makeText(this, "Recording", Toast.LENGTH_SHORT).show();
+ } catch (IOException e) {
+ Log.e(TAG, "Failed to open dvr file", e);
+ dvrFd = null;
+ xr.haptic(1.0f, 250);
+ }
+ }
+
+ private void stopDvr() {
+ if (dvrFd == null) {
+ return;
+ }
+ if (link != null) {
+ link.stopDvr();
+ }
+ try {
+ dvrFd.close();
+ } catch (IOException e) {
+ Log.e(TAG, "Failed to close dvr file", e);
+ }
+ dvrFd = null;
+ }
+
+ // ------------------------------------------------------------------------------
+ // stream + link callbacks
+ // ------------------------------------------------------------------------------
+
+ @Override
+ public void onVideoResolution(int videoW, int videoH) {
+ if (videoW <= 0 || videoH <= 0) {
+ return;
+ }
+ Log.i(TAG, "stream resolution " + videoW + "x" + videoH);
+ videoSeen = true;
+ if (xr != null) {
+ xr.setVideoResolution(videoW, videoH);
+ }
+ // So the next session can size its swapchain exactly.
+ prefs().edit().putInt(PREF_LAST_VIDEO_W, videoW).putInt(PREF_LAST_VIDEO_H, videoH).apply();
+ }
+
+ @Override
+ public void onWfbStats(WfbNGStats data) {
+ lastStats = data;
+ // Without this there is no way to tell "the adapter is dead" from "the link is fine
+ // but nothing decodes" while wearing the headset.
+ final long nowMs = System.currentTimeMillis();
+ if (nowMs - lastStatsLogAt > 2000) {
+ lastStatsLogAt = nowMs;
+ Log.i(TAG, "wfb: all=" + data.count_p_all
+ + " decOk=" + data.count_p_dec_ok
+ + " decErr=" + data.count_p_dec_err
+ + " fecRec=" + data.count_p_fec_recovered
+ + " lost=" + data.count_p_lost
+ + " bad=" + data.count_p_bad
+ + " rssi=" + data.avg_rssi
+ + " videoSeen=" + videoSeen);
+ }
+ if (xr == null || !linkStarted || data.count_p_all <= 0) {
+ return;
+ }
+ // A headset can do something goggles cannot: warn you through the controllers, so
+ // a degrading link does not depend on you reading an OSD number mid-manoeuvre.
+ final int lost = data.count_p_lost;
+ final int recovered = data.count_p_fec_recovered;
+ final boolean bad = lost > 0 || recovered > data.count_p_all / 8;
+
+ final long now = System.currentTimeMillis();
+ if (bad && (!linkWasBad || now - lastHapticAt > 2000)) {
+ lastHapticAt = now;
+ xr.haptic(lost > 0 ? 1.0f : 0.5f, lost > 0 ? 180 : 70);
+ }
+ linkWasBad = bad;
+ }
+
+ // ------------------------------------------------------------------------------
+ // link service callbacks
+ // ------------------------------------------------------------------------------
+
+ @Override
+ public void onLinkStatus(String message) {
+ lastStatus = message;
+ runOnUiThread(() -> {
+ if (statusView != null && !linkStarted) {
+ statusView.setText(message);
+ }
+ });
+ }
+
+ @Override
+ public void onLocalStreamHint(String url) {
+ Log.i(TAG, "listening on " + url);
+ }
+}
diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
index 9ebd0632..d3c6705e 100644
--- a/app/src/main/res/values/strings.xml
+++ b/app/src/main/res/values/strings.xml
@@ -1,5 +1,9 @@
PixelPilot
+ PixelPilot VR
+ Waiting for the wifi adapter…
+ Entering VR…
+ VR mode unavailable: %1$s — falling back to flat mode…
- 1
- 2
diff --git a/app/videonative/src/main/cpp/BufferedPacketQueue.h b/app/videonative/src/main/cpp/BufferedPacketQueue.h
index 88d6eab0..891540a8 100644
--- a/app/videonative/src/main/cpp/BufferedPacketQueue.h
+++ b/app/videonative/src/main/cpp/BufferedPacketQueue.h
@@ -5,9 +5,11 @@
#include
#endif
#include
+#include
#include
#include
#include
+#include
#include
#include
@@ -15,10 +17,27 @@
// Define logging tag and maximum buffer size
#define BUFFERED_QUEUE_LOG_TAG "BufferedPacketQueue"
-// Considering the packet rate about 100 packets per second, 10 packets should be enough
+// Last-resort cap on the buffer. How long this is in wall clock time depends entirely on the
+// packet rate, which is why it cannot be the only bound - see MAX_BUFFER_AGE.
constexpr size_t MAX_BUFFER_SIZE = 15;
// Number of monotonically increasing packets
constexpr size_t MONOTONIC_THRESHOLD = 5;
+// A monotonic run is only tracked while the gap is plausibly a reorder. Kept separate from
+// MONOTONIC_THRESHOLD: the gap grows by one with every packet held back, so gating the counter
+// on the same value it is compared against means it can never reach it.
+constexpr size_t MONOTONIC_MAX_DISTANCE = 64;
+
+// Clock used to bound how long a packet may be held back.
+using QueueClock = std::chrono::steady_clock;
+using QueueTimePoint = QueueClock::time_point;
+
+// A gap in the sequence numbers on this path is almost always a packet that FEC could not
+// recover, not a reorder: by the time packets get here they have come through wfb-ng and a
+// loopback socket, where reordering takes microseconds. Waiting for a packet that will never
+// arrive is pure added latency, so the wait is bounded in time rather than in packets - the
+// packet bound alone is worth ~20ms on a 1080p video stream but ~300ms on the audio stream,
+// which runs at a fraction of the packet rate.
+constexpr auto MAX_BUFFER_AGE = std::chrono::milliseconds(20);
// Type definition for sequence numbers
using SeqType = uint16_t;
@@ -45,7 +64,12 @@ class BufferedPacketQueue
* @param callback Callable to handle processed packets.
*/
template
- void processPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length, Callback& callback)
+ void processPacket(
+ SeqType currPacketIdx,
+ const uint8_t* data,
+ std::size_t data_length,
+ Callback& callback,
+ QueueTimePoint now = QueueClock::now())
{
logDebug(
"Processing packet with Sequence=%u, lastPacketIdx=%u, firstPacket=%s",
@@ -53,6 +77,19 @@ class BufferedPacketQueue
mLastPacketIdx,
mFirstPacket ? "true" : "false");
+ // Before anything else: give up on a gap we have been waiting on for too long. Done
+ // here rather than in handleOutOfOrderPacket so that an in-order packet arriving after
+ // a stall does not get delivered ahead of what is already buffered.
+ if (!mPackets.empty() && (now - mOldestBufferedAt) >= MAX_BUFFER_AGE)
+ {
+ logWarning(
+ "Held %zu packet(s) for more than %lldms waiting on Sequence=%u. Flushing.",
+ mPackets.size(),
+ (long long) MAX_BUFFER_AGE.count(),
+ static_cast(static_cast(mLastPacketIdx + 1)));
+ mLastPacketIdx = drainBufferInOrder(callback);
+ }
+
if (isFirstPacket(currPacketIdx))
{
handleFirstPacket(currPacketIdx);
@@ -73,7 +110,7 @@ class BufferedPacketQueue
else
{
// Out-of-order packet
- handleOutOfOrderPacket(currPacketIdx, data, data_length, callback);
+ handleOutOfOrderPacket(currPacketIdx, data, data_length, callback, now);
}
}
@@ -83,6 +120,9 @@ class BufferedPacketQueue
std::unordered_map> mPackets;
+ // When the current stall started, i.e. when mPackets last went from empty to non-empty.
+ QueueTimePoint mOldestBufferedAt{};
+
// This variable is used to track a situation where the sequence number is increasing monotonically while packets
// are out of order. if this counter reaches MONOTONIC_THRESHOLD, we will restart buffering and update lastPacketIdx
// to the highest sequence index received.
@@ -174,7 +214,12 @@ class BufferedPacketQueue
* @param callback Callable to handle processed packets.
*/
template
- void handleOutOfOrderPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length, Callback& callback)
+ void handleOutOfOrderPacket(
+ SeqType currPacketIdx,
+ const uint8_t* data,
+ std::size_t data_length,
+ Callback& callback,
+ QueueTimePoint now)
{
logDebug("Out-of-order packet detected. Sequence=%u", currPacketIdx);
@@ -184,10 +229,15 @@ class BufferedPacketQueue
// return;
}
- bufferPacket(currPacketIdx, data, data_length);
+ bufferPacket(currPacketIdx, data, data_length, now);
- auto dist = calculateDistance(currPacketIdx, mLastPacketIdx);
- if (std::abs(dist) < MONOTONIC_THRESHOLD)
+ // calculateDistance(a, b) is how far b is ahead of a - see seqLessThan below - so the
+ // question "is this packet ahead of the last one we delivered" has to be asked in that
+ // order. Reversed, dist is negative for exactly the case this heuristic exists for (a
+ // gap ahead of us), the else branch below clears the counter every time, and the
+ // buffer only ever drains on the MAX_BUFFER_SIZE cap.
+ auto dist = calculateDistance(mLastPacketIdx, currPacketIdx);
+ if (static_cast(std::abs(dist)) < MONOTONIC_MAX_DISTANCE)
{
// Check for monotonic increases
if (dist > 0)
@@ -196,10 +246,8 @@ class BufferedPacketQueue
logDebug("Monotonic increase count: %zu", mMonotonicOutOfOrderIncreaseCount);
if (mMonotonicOutOfOrderIncreaseCount >= MONOTONIC_THRESHOLD)
{
- restartBuffering(callback, currPacketIdx);
- // Update lastPacketIdx to the highest sequence index received
- SeqType newLastIdx = currPacketIdx;
- logWarning("Monotonic threshold reached. Updating lastPacketIdx to %u", newLastIdx);
+ mLastPacketIdx = drainBufferInOrder(callback);
+ logWarning("Monotonic threshold reached. Updating lastPacketIdx to %u", mLastPacketIdx);
}
}
else
@@ -214,7 +262,7 @@ class BufferedPacketQueue
{
logWarning(
"Buffer size exceeded MAX_BUFFER_SIZE (%zu). Processing in-order buffered packets.", MAX_BUFFER_SIZE);
- restartBuffering(callback, currPacketIdx);
+ mLastPacketIdx = drainBufferInOrder(callback);
}
}
@@ -231,19 +279,27 @@ class BufferedPacketQueue
* @param data Pointer to the packet data.
* @param data_length Size of the packet data.
*/
- void bufferPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length)
+ void bufferPacket(SeqType currPacketIdx, const uint8_t* data, std::size_t data_length, QueueTimePoint now)
{
+ // Only the start of a stall is recorded, because the buffer is always drained as a
+ // whole. A partial drain leaves the mark where it was, which errs towards flushing
+ // early - the safe direction on a live link.
+ if (mPackets.empty())
+ {
+ mOldestBufferedAt = now;
+ }
mPackets[currPacketIdx] = std::vector(data, data + data_length);
logDebug("Buffered out-of-order packet. Buffer size: %zu", mPackets.size());
}
/**
- * @brief Handles buffer overflow by processing in-order packets and discarding others.
+ * @brief Delivers everything currently held back, in sequence order, and empties the buffer.
* @tparam Callback A callable type that processes the packet data.
* @param callback Callable to handle processed packets.
+ * @return The highest sequence number delivered, or mLastPacketIdx if nothing was held.
*/
template
- void restartBuffering(Callback& callback, SeqType currPacketIdx)
+ SeqType drainBufferInOrder(Callback& callback)
{
// Process as many in-order buffered packets as possible
processBufferedPackets(callback);
@@ -262,25 +318,40 @@ class BufferedPacketQueue
sortedPackets.push_back(it);
}
- // Sort the vector based on the keys
+ // Sorted by distance from the last delivered packet, not by raw value: a block
+ // that straddles the wrap point (65534, 65535, 0, 1) sorts to 0, 1, 65534, 65535
+ // by value, and would be handed to the parser in that order.
+ const SeqType from = mLastPacketIdx;
std::sort(
sortedPackets.begin(),
sortedPackets.end(),
- [](const auto& a, const auto& b) { return a->first < b->first; });
-
- // Iterate over the sorted packets and invoke the callback
+ [from](const auto& a, const auto& b) {
+ return static_cast(a->first - from) < static_cast(b->first - from);
+ });
+
+ // Seeded from a packet that is actually in the buffer rather than from
+ // mLastPacketIdx. RTP starts at a random sequence number, so a VTX that reboots
+ // mid-session can land more than half the sequence space away, where
+ // calculateDistance() reads as negative - seeded from mLastPacketIdx nothing
+ // would ever move and the queue would never resync.
+ SeqType highest = sortedPackets.front()->first;
for (const auto& it : sortedPackets)
{
const auto& packet = it->second;
logDebug("Processing possibly out-of-order buffered packet with Sequence=%u.", it->first);
callback(packet.data(), packet.size());
+ if (calculateDistance(highest, it->first) > 0)
+ {
+ highest = it->first;
+ }
}
mPackets.clear();
// Reset the monotonic increase counter
mMonotonicOutOfOrderIncreaseCount = 0;
+ return highest;
}
- mLastPacketIdx = currPacketIdx;
+ return mLastPacketIdx;
}
/**
diff --git a/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp b/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp
index c6f79a0b..6e803b62 100644
--- a/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp
+++ b/app/videonative/src/main/cpp/tests/BufferedPacketQueue_test.cpp
@@ -1,6 +1,7 @@
#include "BufferedPacketQueue.h" // the class under test
#include
#include
+#include
#include
// ---------- Test fixture ----------------------------------------------------
@@ -10,7 +11,17 @@ class BufferedPacketQueueTest : public ::testing::Test
BufferedPacketQueue q;
std::vector delivered;
- void SetUp() override { delivered.clear(); }
+ // The queue bounds how long it will hold a packet back, so the tests drive the clock
+ // themselves instead of letting a loaded machine decide whether the bound was hit.
+ QueueTimePoint now{};
+
+ void SetUp() override
+ {
+ delivered.clear();
+ now = QueueTimePoint{};
+ }
+
+ void advance(int ms) { now += std::chrono::milliseconds(ms); }
/* Helper: feed one packet and record what the queue actually delivers. */
void feed(uint16_t seq)
@@ -23,7 +34,7 @@ class BufferedPacketQueueTest : public ::testing::Test
delivered.push_back(*(uint16_t*) seq);
};
- q.processPacket(seq, (uint8_t*) &dummy, 2, cb);
+ q.processPacket(seq, (uint8_t*) &dummy, 2, cb, now);
}
};
@@ -57,6 +68,124 @@ TEST_F(BufferedPacketQueueTest, ReorderedDeliversInOrder)
ASSERT_EQ(delivered, expected) << "Overflow flush should deliver the entire block in one shot";
}
+// ---------- A gap that will never be filled --------------------------------
+// The common case on a lossy link: FEC could not recover one packet, and every packet after it
+// is held back waiting for it. Nothing is lost by waiting, but everything behind the gap gets
+// later and later, so the queue has to give up at some point.
+TEST_F(BufferedPacketQueueTest, PermanentGapDoesNotHoldTheStreamForFifteenPackets)
+{
+ for (uint16_t s = 1; s <= 5; ++s) feed(s);
+ ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5}));
+
+ // 6 never arrives.
+ for (uint16_t s = 7; s <= 11; ++s) feed(s);
+ feed(12);
+
+ ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12}))
+ << "A monotonic run past the gap should release the buffer, not wait for MAX_BUFFER_SIZE";
+}
+
+// The monotonic run is counted in packets, so how much latency it costs depends on the packet
+// rate - which on the audio stream is a fraction of the video one. The age bound is what makes
+// the wait the same on both.
+TEST_F(BufferedPacketQueueTest, StaleBufferIsFlushedOnTimeout)
+{
+ feed(1);
+ // 2 never arrives.
+ feed(3);
+ ASSERT_EQ(delivered, (std::vector{1})) << "3 is held back waiting for 2";
+
+ advance(25);
+ feed(4);
+
+ ASSERT_EQ(delivered, (std::vector{1, 3, 4}))
+ << "Once the buffer is older than MAX_BUFFER_AGE it has to be released";
+}
+
+// The other side of that bound: a reorder that resolves quickly must still be reordered, not
+// flushed out of sequence.
+TEST_F(BufferedPacketQueueTest, ReorderWithinTimeoutIsStillPutBackInOrder)
+{
+ feed(1);
+ feed(3);
+ advance(5);
+ feed(2);
+
+ ASSERT_EQ(delivered, (std::vector{1, 2, 3}))
+ << "A packet that arrives late but within MAX_BUFFER_AGE must not be flushed early";
+}
+
+// A large jump is not a reorder - it is a stream that restarted somewhere else. It must not be
+// mistaken for a monotonic run, and the buffer cap has to catch it.
+TEST_F(BufferedPacketQueueTest, LargeJumpFallsBackToTheBufferCap)
+{
+ feed(1);
+ for (uint16_t s = 30000; s < 30000 + MAX_BUFFER_SIZE; ++s) feed(s);
+
+ ASSERT_EQ(delivered.size(), 1u + MAX_BUFFER_SIZE)
+ << "The buffer cap should have released the jumped-to block";
+ ASSERT_EQ(delivered.front(), 1);
+ ASSERT_EQ(delivered.back(), static_cast(30000 + MAX_BUFFER_SIZE - 1));
+}
+
+// A flush delivers everything it is holding, so the pointer has to end up past all of it.
+// Rewinding it to whichever packet happened to trigger the flush re-creates the stall that
+// was just cleared.
+TEST_F(BufferedPacketQueueTest, FlushAdvancesPastEverythingItDelivered)
+{
+ for (uint16_t s = 1; s <= 5; ++s) feed(s);
+
+ // 6 never arrives, and 11 is missing from the run so the flush is triggered by 10 while
+ // 12 is already buffered behind it.
+ feed(7);
+ feed(8);
+ feed(9);
+ feed(12);
+ feed(10);
+ ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5, 7, 8, 9, 10, 12}));
+
+ feed(13);
+ ASSERT_EQ(delivered, (std::vector{1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13}))
+ << "13 follows the highest packet already delivered and must not be held back";
+}
+
+// RTP starts at a random sequence number, so a VTX rebooting mid-session shows up as a jump
+// that can be more than half the sequence space - where a signed 16 bit distance reads as
+// "behind us" and the queue would never move again.
+TEST_F(BufferedPacketQueueTest, StreamRestartFarAheadResyncs)
+{
+ feed(1);
+ feed(2);
+
+ feed(40000);
+ ASSERT_EQ(delivered, (std::vector{1, 2})) << "40000 is held back at first";
+
+ advance(25);
+ feed(40001);
+ feed(40002);
+
+ ASSERT_EQ(delivered, (std::vector{1, 2, 40000, 40001, 40002}))
+ << "after the age bound releases 40000 the queue has to follow the new base";
+}
+
+// The buffer is a hash map, so the flush has to sort it - and sorting by raw value puts a
+// block that straddles the wrap point in the wrong order.
+TEST_F(BufferedPacketQueueTest, FlushAcrossTheWrapDeliversInOrder)
+{
+ feed(65532);
+
+ // 65533 never arrives; the rest of the run crosses zero.
+ feed(65534);
+ feed(65535);
+ feed(0);
+ feed(1);
+ feed(2);
+ feed(3);
+
+ ASSERT_EQ(delivered, (std::vector{65532, 65534, 65535, 0, 1, 2, 3}))
+ << "sorted by value this comes out as 0, 1, 2, 65534, 65535";
+}
+
// ---------- gtest boilerplate main -----------------------------------------
int main(int argc, char** argv)
{
diff --git a/app/videonative/src/main/cpp/tests/CMakeLists.txt b/app/videonative/src/main/cpp/tests/CMakeLists.txt
index 1e70da39..6ce59ad5 100644
--- a/app/videonative/src/main/cpp/tests/CMakeLists.txt
+++ b/app/videonative/src/main/cpp/tests/CMakeLists.txt
@@ -25,7 +25,7 @@ enable_testing()
# ---------- Test executable --------------------------------------------------
add_executable(queue_test
- BufferedPacketqueue_test.cpp
+ BufferedPacketQueue_test.cpp
)
target_include_directories(queue_test PUBLIC
diff --git a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java
index e1c49d0f..25a9c284 100644
--- a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java
+++ b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java
@@ -9,7 +9,6 @@
import android.view.SurfaceHolder;
import androidx.annotation.Nullable;
-import androidx.appcompat.app.AppCompatActivity;
import java.util.Timer;
import java.util.TimerTask;
@@ -35,7 +34,7 @@ public class VideoPlayer implements IVideoParamsChanged {
private Timer timer;
// Setup as much as possible without creating the decoder
- public VideoPlayer(final AppCompatActivity parent) {
+ public VideoPlayer(final Context parent) {
this.context = parent;
nativeVideoPlayer = nativeInitialize(context);
}
@@ -161,6 +160,21 @@ public void addAndStartDecoderReceiver(Surface surface, int index) {
setVideoSurface(surface, index);
}
+ /**
+ * Point the decoder at a new output surface without touching the receivers.
+ *
+ * This is what a surface change should do: the decoder is rebuilt for the new surface
+ * while UDP reception, and with it the wifi link, keeps running.
+ */
+ public void attachSurface(Surface surface, int index) {
+ setVideoSurface(surface, index);
+ }
+
+ /** Releases the decoder's output surface, leaving the receivers alone. */
+ public void detachSurface(int index) {
+ setVideoSurface(null, index);
+ }
+
/**
* Stop the Receiver
* Stop the Decoder
diff --git a/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp b/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp
index a3046b07..ae365310 100644
--- a/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp
+++ b/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp
@@ -56,6 +56,7 @@ WfbngLink::WfbngLink(JNIEnv *env, jobject context)
}
void WfbngLink::initAgg() {
+ try {
std::string client_addr = "127.0.0.1";
uint64_t epoch = 0;
@@ -81,6 +82,19 @@ void WfbngLink::initAgg() {
udp_aggregator =
std::make_unique(client_addr, udp_client_port, keyPath, epoch, udp_channel_id_f, 0);
+ } catch (const std::exception &e) {
+ // The aggregators open the key in their constructor. Letting that escape a JNI call
+ // aborts the whole process, which is what a missing gs.key used to do.
+ __android_log_print(ANDROID_LOG_ERROR,
+ TAG,
+ "initAgg failed (%s) - check gs.key at %s. Link stays down until "
+ "the key is refreshed.",
+ e.what(),
+ keyPath);
+ video_aggregator.reset();
+ mavlink_aggregator.reset();
+ udp_aggregator.reset();
+ }
}
int WfbngLink::run(JNIEnv *env, jobject context, jint wifiChannel, jint bw, jint fd) {
diff --git a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java
index 39dca17d..897f0b08 100644
--- a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java
+++ b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java
@@ -7,7 +7,6 @@
import android.util.Log;
import androidx.annotation.Keep;
-import androidx.appcompat.app.AppCompatActivity;
import java.util.HashMap;
import java.util.Map;
@@ -30,6 +29,31 @@ public void setFecThresholds(int lostTo5, int recTo4, int recTo3, int recTo2, in
System.loadLibrary("WfbngRtl8812");
}
+ /**
+ * Devices with a live RX loop anywhere in the process, keyed by device name.
+ *
+ * Per-instance bookkeeping is not enough: each Activity owns its own WfbNgLink, so
+ * two of them both see an empty map and both claim the same dongle. Keyed on the thread
+ * so a claim left behind by a thread that died is simply ignored rather than locking the
+ * adapter out for good.
+ */
+ private static final Map claimedDevices = new HashMap<>();
+
+ private static synchronized boolean claim(String deviceName, Thread t) {
+ Thread holder = claimedDevices.get(deviceName);
+ if (holder != null && holder.isAlive()) {
+ return false;
+ }
+ claimedDevices.put(deviceName, t);
+ return true;
+ }
+
+ private static synchronized void release(String deviceName, Thread t) {
+ if (claimedDevices.get(deviceName) == t) {
+ claimedDevices.remove(deviceName);
+ }
+ }
+
private final long nativeWfbngLink;
private final Timer timer;
private final Context context;
@@ -50,7 +74,7 @@ public void setFecThresholds(int lostTo5, int recTo4, int recTo3, int recTo2, in
public static native void nativeSetUseLdpc(long nativeInstance, int use);
public static native void nativeSetUseStbc(long nativeInstance, int use);
- public WfbNgLink(final AppCompatActivity parent) {
+ public WfbNgLink(final Context parent) {
this.context = parent;
nativeWfbngLink = nativeInitialize(context);
timer = new Timer();
@@ -93,6 +117,19 @@ public void nativeSetUseStbc(int use) {
}
public synchronized boolean start(int wifiChannel, int bandWidth, UsbDevice usbDevice) {
+ // A second RX loop on the same adapter means two libusb handles on one interface,
+ // and the old thread is orphaned because linkThreads.put() overwrites its entry, so
+ // stopAll() then joins only the last one.
+ //
+ // (Note: three threads named "wfb-001/002" is *normal* for one adapter - the TX and
+ // adaptive-link threads inherit the RX thread's name because nothing renames them.
+ // An earlier reading of that as three RX loops was wrong; the guard is still right,
+ // because two activities each with their own WfbNgLink really can double-start.)
+ Thread existing = linkThreads.get(usbDevice);
+ if (existing != null && existing.isAlive()) {
+ Log.w(TAG, "wfb-ng already running on " + usbDevice.getDeviceName() + ", not starting a second");
+ return true;
+ }
Log.d(TAG, "wfb-ng monitoring on " + usbDevice.getDeviceName() + " using wifi channel " + wifiChannel);
UsbManager usbManager = (UsbManager) context.getSystemService(Context.USB_SERVICE);
// Returns null when the permission was revoked or the device disappeared between
@@ -108,12 +145,26 @@ public synchronized boolean start(int wifiChannel, int bandWidth, UsbDevice usbD
usbDeviceConnection.close();
return false;
}
- Thread t = new Thread(() -> nativeRun(nativeWfbngLink, context, wifiChannel, bandWidth, fd));
+ final String deviceName = usbDevice.getDeviceName();
+ final Thread[] self = new Thread[1];
+ Thread t = new Thread(() -> {
+ try {
+ nativeRun(nativeWfbngLink, context, wifiChannel, bandWidth, fd);
+ } finally {
+ release(deviceName, self[0]);
+ }
+ });
+ self[0] = t;
t.setName(threadNameFor(usbDevice));
+ if (!claim(deviceName, t)) {
+ Log.w(TAG, "another wfb-ng RX loop already owns " + deviceName + ", not starting a second");
+ usbDeviceConnection.close();
+ return false;
+ }
linkThreads.put(usbDevice, t);
linkConns.put(usbDevice, usbDeviceConnection);
t.start();
- Log.d(TAG, "wfb-ng thread on " + usbDevice.getDeviceName() + " started.");
+ Log.d(TAG, "wfb-ng thread on " + deviceName + " started.");
return true;
}
@@ -123,19 +174,44 @@ private static String threadNameFor(UsbDevice usbDevice) {
return "wfb-" + (parts.length > 1 ? parts[1] : name);
}
+ /**
+ * The RX loop is joined so the USB interface is released before anything reopens it, but
+ * these calls come from Activity lifecycle callbacks on the main thread. An unbounded
+ * join there is a five second ANR waiting to happen, and it did: a driver thread that
+ * does not come back froze the UI. Wait, but give up and say so.
+ */
+ // StopRxLoop() only breaks the receive loop; the thread then still has to stop the TX
+ // frame, the adaptive link, power the chip down and release the USB interface. Too
+ // short a wait here and the next owner cannot claim the interface yet.
+ private static final long JOIN_TIMEOUT_MS = 3000;
+
+ private static void joinBounded(Thread t, String what) throws InterruptedException {
+ if (t == null) {
+ return;
+ }
+ t.join(JOIN_TIMEOUT_MS);
+ if (t.isAlive()) {
+ Log.e(TAG, "wfb-ng thread on " + what + " did not stop within " + JOIN_TIMEOUT_MS
+ + "ms, giving up the claim anyway");
+ } else {
+ Log.d(TAG, "wfb-ng thread on " + what + " done.");
+ }
+ // Drop the claim even if the thread outstayed its welcome. Holding it would leave the
+ // adapter unopenable until the process restarts, and the user would be told there is
+ // no compatible adapter - a worse failure than the duplicate this guards against,
+ // which the per-instance check and the single-owner handoff already prevent.
+ release(what, t);
+ }
+
public synchronized void stopAll() throws InterruptedException {
for (Map.Entry entry : linkConns.entrySet()) {
nativeStop(nativeWfbngLink, context, entry.getValue().getFileDescriptor());
}
for (Map.Entry entry : linkConns.entrySet()) {
- Thread t = linkThreads.get(entry.getKey());
- if (t != null) {
- t.join();
- }
+ joinBounded(linkThreads.get(entry.getKey()), entry.getKey().getDeviceName());
// The connection holds a dup of the usbfs fd. Without close() every
// attach/detach cycle leaks one, until the process runs out.
entry.getValue().close();
- Log.d(TAG, "wfb-ng thread on " + entry.getKey().getDeviceName() + " done.");
}
linkThreads.clear();
linkConns.clear();
@@ -148,10 +224,7 @@ public synchronized void stop(UsbDevice dev) throws InterruptedException {
}
int fd = conn.getFileDescriptor();
nativeStop(nativeWfbngLink, context, fd);
- Thread t = linkThreads.get(dev);
- if (t != null) {
- t.join();
- }
+ joinBounded(linkThreads.get(dev), dev.getDeviceName());
linkThreads.remove(dev);
linkConns.remove(dev);
conn.close();
diff --git a/app/xr/build.gradle b/app/xr/build.gradle
new file mode 100644
index 00000000..20e81cc1
--- /dev/null
+++ b/app/xr/build.gradle
@@ -0,0 +1,53 @@
+plugins {
+ alias(libs.plugins.androidLibrary)
+}
+
+android {
+ namespace = "com.openipc.xr"
+ compileSdk = 34
+
+ defaultConfig {
+ minSdk = 26
+ }
+
+ buildTypes {
+ release {
+ ndk {
+ abiFilters.add("arm64-v8a")
+ abiFilters.add("armeabi-v7a")
+ }
+ }
+ debug {
+ ndk {
+ abiFilters.add("arm64-v8a")
+ abiFilters.add("armeabi-v7a")
+ }
+ }
+ }
+
+ externalNativeBuild {
+ cmake {
+ path("src/main/cpp/CMakeLists.txt")
+ version = "3.22.1"
+ }
+ }
+
+ // The OpenXR loader is consumed as a prefab package (OpenXR::openxr_loader,
+ // OpenXR::headers) out of the Khronos AAR.
+ buildFeatures {
+ prefab = true
+ }
+
+ compileOptions {
+ sourceCompatibility = JavaVersion.VERSION_17
+ targetCompatibility = JavaVersion.VERSION_17
+ }
+}
+
+dependencies {
+ implementation(libs.annotation)
+
+ // api, not implementation: the loader's .so and its manifest additions
+ // (OpenXR broker queries/permissions) have to reach the application module.
+ api(libs.openxr.loader)
+}
diff --git a/app/xr/src/main/cpp/CMakeLists.txt b/app/xr/src/main/cpp/CMakeLists.txt
new file mode 100644
index 00000000..15cc16e2
--- /dev/null
+++ b/app/xr/src/main/cpp/CMakeLists.txt
@@ -0,0 +1,21 @@
+
+cmake_minimum_required(VERSION 3.22.1)
+
+project("XrGoggle")
+
+find_package(OpenXR REQUIRED CONFIG)
+
+add_library(${CMAKE_PROJECT_NAME} SHARED
+ XrGoggleSession.cpp
+ jni_bindings.cpp)
+
+target_link_libraries(${CMAKE_PROJECT_NAME}
+ OpenXR::openxr_loader
+ OpenXR::headers
+ android
+ EGL
+ GLESv3
+ log)
+
+set_property(TARGET ${CMAKE_PROJECT_NAME} PROPERTY CXX_STANDARD 20)
+target_compile_options(${CMAKE_PROJECT_NAME} PRIVATE -fno-omit-frame-pointer)
diff --git a/app/xr/src/main/cpp/XrGoggleSession.cpp b/app/xr/src/main/cpp/XrGoggleSession.cpp
new file mode 100644
index 00000000..d484c928
--- /dev/null
+++ b/app/xr/src/main/cpp/XrGoggleSession.cpp
@@ -0,0 +1,1543 @@
+#include "XrGoggleSession.h"
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#define TAG "pixelpilot-xr"
+#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
+#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
+#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
+
+namespace
+{
+constexpr float kStickDeadzone = 0.30f;
+constexpr float kTriggerDeadzone = 0.15f;
+// A hand gesture has to be held this long before it counts. Grabbing a drone or a radio
+// squeezes the hand for a moment; deliberately holding a pinch does not happen by accident.
+constexpr int64_t kHandHoldNs = 700LL * 1000 * 1000;
+constexpr float kDistancePerSec = 0.9f; // meters per second of full stick deflection
+constexpr float kWidthPerSec = 1.4f;
+constexpr float kMinDistance = 0.4f;
+constexpr float kMaxDistance = 8.0f;
+constexpr float kMinWidth = 0.4f;
+constexpr float kMaxWidth = 12.0f;
+constexpr float kMaxHeightOffset = 1.5f;
+// One nudge, about 1.8 degrees at the default distance - small enough to fine-tune.
+constexpr float kHeightStep = 0.05f;
+constexpr float kHeightPerSec = 0.7f;
+
+float clampf(float v, float lo, float hi)
+{
+ return v < lo ? lo : (v > hi ? hi : v);
+}
+
+// Yaw-only quaternion, so the panel never ends up tilted when it is anchored.
+XrQuaternionf quatFromYaw(float yaw)
+{
+ return XrQuaternionf{0.0f, std::sin(yaw * 0.5f), 0.0f, std::cos(yaw * 0.5f)};
+}
+
+float yawOf(const XrQuaternionf& q)
+{
+ return std::atan2(2.0f * (q.w * q.y + q.x * q.z), 1.0f - 2.0f * (q.y * q.y + q.z * q.z));
+}
+} // namespace
+
+// ---------------------------------------------------------------------------------
+// error handling
+// ---------------------------------------------------------------------------------
+
+void XrGoggleSession::setError(const std::string& what)
+{
+ std::lock_guard lock(mErrorMutex);
+ if (mLastError.empty())
+ {
+ mLastError = what;
+ }
+ LOGE("%s", what.c_str());
+}
+
+std::string XrGoggleSession::lastError()
+{
+ std::lock_guard lock(mErrorMutex);
+ return mLastError;
+}
+
+bool XrGoggleSession::check(XrResult result, const char* what)
+{
+ if (XR_SUCCEEDED(result))
+ {
+ return true;
+ }
+ char name[XR_MAX_RESULT_STRING_SIZE] = {0};
+ if (mInstance != XR_NULL_HANDLE)
+ {
+ xrResultToString(mInstance, result, name);
+ }
+ else
+ {
+ snprintf(name, sizeof(name), "%d", (int) result);
+ }
+ setError(std::string(what) + " failed: " + name);
+ return false;
+}
+
+XrPath XrGoggleSession::path(const char* str)
+{
+ XrPath p = XR_NULL_PATH;
+ XrResult r = xrStringToPath(mInstance, str, &p);
+ if (XR_FAILED(r))
+ {
+ LOGW("xrStringToPath(%s) failed", str);
+ return XR_NULL_PATH;
+ }
+ return p;
+}
+
+bool XrGoggleSession::extensionSupported(const char* name) const
+{
+ for (const auto& e : mAvailableExtensions)
+ {
+ if (e == name) return true;
+ }
+ return false;
+}
+
+// ---------------------------------------------------------------------------------
+// bring-up
+// ---------------------------------------------------------------------------------
+
+bool XrGoggleSession::create(JNIEnv* env, jobject activity)
+{
+ if (!initLoader(env, activity)) return false;
+ if (!resolveRuntime()) return false;
+ if (!createInstance(env, activity)) return false;
+ if (!createEgl()) return false;
+ if (!createSession()) return false;
+ if (!createSpaces()) return false;
+ if (!createSwapchain(env)) return false;
+ if (!createActions()) return false;
+ // Passthrough is optional: a runtime without XR_FB_passthrough still flies.
+ if (mHasPassthrough && !createPassthrough())
+ {
+ LOGW("passthrough unavailable, continuing without it");
+ mHasPassthrough = false;
+ }
+ if (mHasRefreshRate)
+ {
+ uint32_t count = 0;
+ if (check(mXrEnumerateDisplayRefreshRatesFB(mSession, 0, &count, nullptr),
+ "xrEnumerateDisplayRefreshRatesFB") &&
+ count > 0)
+ {
+ mRefreshRates.resize(count);
+ mXrEnumerateDisplayRefreshRatesFB(mSession, count, &count, mRefreshRates.data());
+ for (float hz : mRefreshRates)
+ {
+ LOGI("display refresh rate available: %.1f Hz", hz);
+ }
+ }
+ }
+ return true;
+}
+
+// ---------------------------------------------------------------------------------
+// runtime discovery
+// ---------------------------------------------------------------------------------
+
+namespace
+{
+#if defined(__aarch64__)
+constexpr const char* kAbi = "arm64-v8a";
+#elif defined(__arm__)
+constexpr const char* kAbi = "armeabi-v7a";
+#elif defined(__x86_64__)
+constexpr const char* kAbi = "x86_64";
+#else
+constexpr const char* kAbi = "x86";
+#endif
+} // namespace
+
+// True when a real runtime answered - a runtime-provided extension has to be in the list,
+// not just "the call did not fail".
+bool XrGoggleSession::runtimeAnswers()
+{
+ uint32_t count = 0;
+ if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, 0, &count, nullptr)) || count == 0)
+ {
+ return false;
+ }
+ std::vector props(count, {XR_TYPE_EXTENSION_PROPERTIES});
+ if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, count, &count, props.data())))
+ {
+ return false;
+ }
+ for (const auto& p : props)
+ {
+ if (std::strcmp(p.extensionName, XR_KHR_ANDROID_SURFACE_SWAPCHAIN_EXTENSION_NAME) == 0)
+ {
+ return true;
+ }
+ }
+ return false;
+}
+
+// Drops a runtime manifest next to our files and points the loader at it.
+bool XrGoggleSession::pointLoaderAt(const char* libraryPath)
+{
+ if (mManifestDir.empty()) return false;
+ const std::string manifest = mManifestDir + "/openxr_runtime.json";
+ {
+ std::ofstream out(manifest, std::ios::trunc);
+ if (!out) return false;
+ out << "{\n"
+ << " \"file_format_version\": \"1.0.0\",\n"
+ << " \"runtime\": {\n"
+ << " \"name\": \"Android system OpenXR runtime\",\n"
+ << " \"library_path\": \"" << libraryPath << "\"\n"
+ << " }\n"
+ << "}\n";
+ }
+ setenv("XR_RUNTIME_JSON", manifest.c_str(), 1);
+ return true;
+}
+
+/*
+ * Fallback for a headset where the loader cannot find a runtime by itself.
+ *
+ * On a Quest 3 this is not needed: the Khronos loader resolves the Horizon runtime through
+ * the org.khronos.openxr.runtime_broker content provider and the first candidate ("change
+ * nothing") wins. Querying that provider from `adb shell` returns no rows, but only because
+ * the shell does not hold org.khronos.openxr.permission.OPENXR - the app does.
+ *
+ * It is kept because the loader has exactly three discovery paths - the broker,
+ * active_runtime.json on disk, and XR_RUNTIME_JSON - and a device that offers neither of the
+ * first two would otherwise be dead in the water. Where that happens the runtime is still
+ * reachable as libopenxr_forwardloader.so from the VrDriver APEX: that library sits on every
+ * app's linker search path and exports xrNegotiateLoaderRuntimeInterface, so as far as a
+ * loader is concerned it *is* the runtime.
+ */
+bool XrGoggleSession::resolveRuntime()
+{
+ if (runtimeAnswers())
+ {
+ LOGI("runtime found without an override");
+ mRuntimeLibrary = "";
+ return true;
+ }
+
+ const std::string apkPath = std::string("/apex/com.meta.xr/priv-app/VrDriver/VrDriver.apk!/lib/") +
+ kAbi + "/libopenxr_forwardloader.so";
+ const char* candidates[] = {
+ // A bare file name means "use the system library search path", which is where the
+ // APEX library actually lives for us.
+ "libopenxr_forwardloader.so",
+ // Some loaders insist on an absolute path; Android's linker understands apk!/entry.
+ apkPath.c_str(),
+ };
+
+ for (const char* candidate : candidates)
+ {
+ if (!pointLoaderAt(candidate))
+ {
+ LOGW("could not write a runtime manifest into %s", mManifestDir.c_str());
+ break;
+ }
+ LOGI("trying runtime library_path=%s", candidate);
+ if (runtimeAnswers())
+ {
+ LOGI("runtime resolved via %s", candidate);
+ mRuntimeLibrary = candidate;
+ return true;
+ }
+ }
+
+ unsetenv("XR_RUNTIME_JSON");
+ setError(
+ "No OpenXR runtime answered. Tried the system default, then "
+ "libopenxr_forwardloader.so by name and by absolute APEX path.");
+ return false;
+}
+
+bool XrGoggleSession::initLoader(JNIEnv* env, jobject activity)
+{
+ PFN_xrInitializeLoaderKHR initializeLoader = nullptr;
+ XrResult r = xrGetInstanceProcAddr(
+ XR_NULL_HANDLE, "xrInitializeLoaderKHR", (PFN_xrVoidFunction*) &initializeLoader);
+ if (XR_FAILED(r) || initializeLoader == nullptr)
+ {
+ // Not fatal on every runtime, but on Android it usually is.
+ LOGW("xrInitializeLoaderKHR not exposed by the loader");
+ return true;
+ }
+
+ JavaVM* vm = nullptr;
+ env->GetJavaVM(&vm);
+
+ XrLoaderInitInfoAndroidKHR init{XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR};
+ init.applicationVM = vm;
+ init.applicationContext = env->NewGlobalRef(activity);
+ r = initializeLoader((const XrLoaderInitInfoBaseHeaderKHR*) &init);
+ if (XR_FAILED(r))
+ {
+ setError("xrInitializeLoaderKHR failed");
+ return false;
+ }
+ return true;
+}
+
+bool XrGoggleSession::createInstance(JNIEnv* env, jobject activity)
+{
+ uint32_t extCount = 0;
+ if (!check(xrEnumerateInstanceExtensionProperties(nullptr, 0, &extCount, nullptr),
+ "xrEnumerateInstanceExtensionProperties"))
+ {
+ return false;
+ }
+ std::vector props(extCount, {XR_TYPE_EXTENSION_PROPERTIES});
+ if (extCount > 0)
+ {
+ if (!check(
+ xrEnumerateInstanceExtensionProperties(nullptr, extCount, &extCount, props.data()),
+ "xrEnumerateInstanceExtensionProperties"))
+ {
+ return false;
+ }
+ }
+ mAvailableExtensions.clear();
+ for (const auto& p : props)
+ {
+ mAvailableExtensions.emplace_back(p.extensionName);
+ }
+ // Worth the log lines: it is the only way to see from a bug report what the runtime on
+ // a given headset actually offers.
+ LOGI("runtime exposes %zu instance extensions:", mAvailableExtensions.size());
+ for (const auto& name : mAvailableExtensions)
+ {
+ LOGI(" %s", name.c_str());
+ }
+
+ // Without these three there is no immersive video path at all.
+ const char* required[] = {
+ XR_KHR_ANDROID_CREATE_INSTANCE_EXTENSION_NAME,
+ XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME,
+ XR_KHR_ANDROID_SURFACE_SWAPCHAIN_EXTENSION_NAME,
+ };
+ std::vector enabled;
+ for (const char* name : required)
+ {
+ if (!extensionSupported(name))
+ {
+ setError(std::string("OpenXR runtime does not support ") + name);
+ return false;
+ }
+ enabled.push_back(name);
+ }
+
+ mHasPassthrough = extensionSupported(XR_FB_PASSTHROUGH_EXTENSION_NAME);
+ mHasLayerSettings = extensionSupported(XR_FB_COMPOSITION_LAYER_SETTINGS_EXTENSION_NAME);
+ mHasRefreshRate = extensionSupported(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME);
+ // Lets the same actions be driven by a pinch when flying without controllers.
+ mHasHandInteraction = extensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME);
+ // Needed to correct the origin of an Android-produced image, see renderFrame().
+ mHasImageLayout = extensionSupported(XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME);
+ // Thumb swipes along the index finger: a discrete control that does not occupy the
+ // hand, so pinch and grasp stay free for the flight actions.
+ mHasMicrogestures = mHasHandInteraction &&
+ extensionSupported(XR_META_HAND_TRACKING_MICROGESTURES_EXTENSION_NAME);
+ // Without this only one interaction profile is live at a time: with hands tracked the
+ // controller bindings never fire, which looks exactly like "the stick is broken".
+ mHasSimultaneous =
+ extensionSupported(XR_META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME);
+ if (mHasPassthrough) enabled.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME);
+ if (mHasLayerSettings) enabled.push_back(XR_FB_COMPOSITION_LAYER_SETTINGS_EXTENSION_NAME);
+ if (mHasRefreshRate) enabled.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME);
+ if (mHasHandInteraction) enabled.push_back(XR_EXT_HAND_INTERACTION_EXTENSION_NAME);
+ if (mHasImageLayout) enabled.push_back(XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME);
+ if (mHasMicrogestures) enabled.push_back(XR_META_HAND_TRACKING_MICROGESTURES_EXTENSION_NAME);
+ if (mHasSimultaneous)
+ enabled.push_back(XR_META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME);
+ LOGI(
+ "optional extensions: passthrough=%d layerSettings=%d refreshRate=%d handInteraction=%d "
+ "imageLayout=%d microgestures=%d simultaneousHandsControllers=%d",
+ (int) mHasPassthrough,
+ (int) mHasLayerSettings,
+ (int) mHasRefreshRate,
+ (int) mHasHandInteraction,
+ (int) mHasImageLayout,
+ (int) mHasMicrogestures,
+ (int) mHasSimultaneous);
+
+ JavaVM* vm = nullptr;
+ env->GetJavaVM(&vm);
+
+ XrInstanceCreateInfoAndroidKHR androidInfo{XR_TYPE_INSTANCE_CREATE_INFO_ANDROID_KHR};
+ androidInfo.applicationVM = vm;
+ androidInfo.applicationActivity = env->NewGlobalRef(activity);
+
+ XrInstanceCreateInfo info{XR_TYPE_INSTANCE_CREATE_INFO};
+ info.next = &androidInfo;
+ std::strncpy(info.applicationInfo.applicationName, "PixelPilot", XR_MAX_APPLICATION_NAME_SIZE - 1);
+ std::strncpy(info.applicationInfo.engineName, "PixelPilot", XR_MAX_ENGINE_NAME_SIZE - 1);
+ info.applicationInfo.applicationVersion = 1;
+ info.applicationInfo.engineVersion = 1;
+ // Ask for 1.0: every shipping Android runtime implements it, and nothing here needs 1.1.
+ info.applicationInfo.apiVersion = XR_API_VERSION_1_0;
+ info.enabledExtensionCount = (uint32_t) enabled.size();
+ info.enabledExtensionNames = enabled.data();
+
+ if (!check(xrCreateInstance(&info, &mInstance), "xrCreateInstance")) return false;
+
+ XrSystemGetInfo systemInfo{XR_TYPE_SYSTEM_GET_INFO};
+ systemInfo.formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY;
+ if (!check(xrGetSystem(mInstance, &systemInfo, &mSystemId), "xrGetSystem")) return false;
+
+ xrGetInstanceProcAddr(mInstance,
+ "xrCreateSwapchainAndroidSurfaceKHR",
+ (PFN_xrVoidFunction*) &mXrCreateSwapchainAndroidSurfaceKHR);
+ if (mXrCreateSwapchainAndroidSurfaceKHR == nullptr)
+ {
+ setError("xrCreateSwapchainAndroidSurfaceKHR not available");
+ return false;
+ }
+ if (mHasPassthrough)
+ {
+ xrGetInstanceProcAddr(
+ mInstance, "xrCreatePassthroughFB", (PFN_xrVoidFunction*) &mXrCreatePassthroughFB);
+ xrGetInstanceProcAddr(
+ mInstance, "xrDestroyPassthroughFB", (PFN_xrVoidFunction*) &mXrDestroyPassthroughFB);
+ xrGetInstanceProcAddr(
+ mInstance, "xrPassthroughStartFB", (PFN_xrVoidFunction*) &mXrPassthroughStartFB);
+ xrGetInstanceProcAddr(
+ mInstance, "xrPassthroughPauseFB", (PFN_xrVoidFunction*) &mXrPassthroughPauseFB);
+ xrGetInstanceProcAddr(mInstance,
+ "xrCreatePassthroughLayerFB",
+ (PFN_xrVoidFunction*) &mXrCreatePassthroughLayerFB);
+ xrGetInstanceProcAddr(mInstance,
+ "xrDestroyPassthroughLayerFB",
+ (PFN_xrVoidFunction*) &mXrDestroyPassthroughLayerFB);
+ xrGetInstanceProcAddr(mInstance,
+ "xrPassthroughLayerResumeFB",
+ (PFN_xrVoidFunction*) &mXrPassthroughLayerResumeFB);
+ xrGetInstanceProcAddr(mInstance,
+ "xrPassthroughLayerPauseFB",
+ (PFN_xrVoidFunction*) &mXrPassthroughLayerPauseFB);
+ xrGetInstanceProcAddr(mInstance,
+ "xrPassthroughLayerSetStyleFB",
+ (PFN_xrVoidFunction*) &mXrPassthroughLayerSetStyleFB);
+ mHasPassthrough = mXrCreatePassthroughFB != nullptr && mXrCreatePassthroughLayerFB != nullptr &&
+ mXrPassthroughStartFB != nullptr;
+ }
+ if (mHasSimultaneous)
+ {
+ xrGetInstanceProcAddr(mInstance,
+ "xrResumeSimultaneousHandsAndControllersTrackingMETA",
+ (PFN_xrVoidFunction*) &mXrResumeSimultaneous);
+ mHasSimultaneous = mXrResumeSimultaneous != nullptr;
+ }
+ if (mHasRefreshRate)
+ {
+ xrGetInstanceProcAddr(mInstance,
+ "xrEnumerateDisplayRefreshRatesFB",
+ (PFN_xrVoidFunction*) &mXrEnumerateDisplayRefreshRatesFB);
+ xrGetInstanceProcAddr(mInstance,
+ "xrRequestDisplayRefreshRateFB",
+ (PFN_xrVoidFunction*) &mXrRequestDisplayRefreshRateFB);
+ mHasRefreshRate =
+ mXrEnumerateDisplayRefreshRatesFB != nullptr && mXrRequestDisplayRefreshRateFB != nullptr;
+ }
+ return true;
+}
+
+bool XrGoggleSession::createEgl()
+{
+ // Nothing is rendered through this context - OpenXR just requires a graphics binding,
+ // and the runtime needs a current context on the session thread.
+ mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+ if (mEglDisplay == EGL_NO_DISPLAY)
+ {
+ setError("eglGetDisplay failed");
+ return false;
+ }
+ if (eglInitialize(mEglDisplay, nullptr, nullptr) == EGL_FALSE)
+ {
+ setError("eglInitialize failed");
+ return false;
+ }
+
+ const EGLint configAttribs[] = {EGL_RENDERABLE_TYPE,
+ EGL_OPENGL_ES3_BIT_KHR,
+ EGL_SURFACE_TYPE,
+ EGL_PBUFFER_BIT,
+ EGL_RED_SIZE,
+ 8,
+ EGL_GREEN_SIZE,
+ 8,
+ EGL_BLUE_SIZE,
+ 8,
+ EGL_ALPHA_SIZE,
+ 8,
+ EGL_DEPTH_SIZE,
+ 0,
+ EGL_STENCIL_SIZE,
+ 0,
+ EGL_SAMPLES,
+ 0,
+ EGL_NONE};
+ EGLint numConfigs = 0;
+ if (eglChooseConfig(mEglDisplay, configAttribs, &mEglConfig, 1, &numConfigs) == EGL_FALSE ||
+ numConfigs < 1)
+ {
+ setError("eglChooseConfig found no usable config");
+ return false;
+ }
+
+ const EGLint contextAttribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE};
+ mEglContext = eglCreateContext(mEglDisplay, mEglConfig, EGL_NO_CONTEXT, contextAttribs);
+ if (mEglContext == EGL_NO_CONTEXT)
+ {
+ setError("eglCreateContext failed");
+ return false;
+ }
+
+ const EGLint surfaceAttribs[] = {EGL_WIDTH, 16, EGL_HEIGHT, 16, EGL_NONE};
+ mEglSurface = eglCreatePbufferSurface(mEglDisplay, mEglConfig, surfaceAttribs);
+ if (mEglSurface == EGL_NO_SURFACE)
+ {
+ setError("eglCreatePbufferSurface failed");
+ return false;
+ }
+ if (eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, mEglContext) == EGL_FALSE)
+ {
+ setError("eglMakeCurrent failed");
+ return false;
+ }
+ return true;
+}
+
+bool XrGoggleSession::createSession()
+{
+ // Required by the spec before xrCreateSession, even though we never render.
+ PFN_xrGetOpenGLESGraphicsRequirementsKHR getRequirements = nullptr;
+ xrGetInstanceProcAddr(mInstance,
+ "xrGetOpenGLESGraphicsRequirementsKHR",
+ (PFN_xrVoidFunction*) &getRequirements);
+ if (getRequirements != nullptr)
+ {
+ XrGraphicsRequirementsOpenGLESKHR requirements{XR_TYPE_GRAPHICS_REQUIREMENTS_OPENGL_ES_KHR};
+ getRequirements(mInstance, mSystemId, &requirements);
+ }
+
+ XrGraphicsBindingOpenGLESAndroidKHR binding{XR_TYPE_GRAPHICS_BINDING_OPENGL_ES_ANDROID_KHR};
+ binding.display = mEglDisplay;
+ binding.config = mEglConfig;
+ binding.context = mEglContext;
+
+ XrSessionCreateInfo info{XR_TYPE_SESSION_CREATE_INFO};
+ info.next = &binding;
+ info.systemId = mSystemId;
+ return check(xrCreateSession(mInstance, &info, &mSession), "xrCreateSession");
+}
+
+bool XrGoggleSession::createSpaces()
+{
+ XrReferenceSpaceCreateInfo viewInfo{XR_TYPE_REFERENCE_SPACE_CREATE_INFO};
+ viewInfo.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW;
+ viewInfo.poseInReferenceSpace = XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}};
+ if (!check(xrCreateReferenceSpace(mSession, &viewInfo, &mViewSpace),
+ "xrCreateReferenceSpace(VIEW)"))
+ {
+ return false;
+ }
+
+ XrReferenceSpaceCreateInfo localInfo{XR_TYPE_REFERENCE_SPACE_CREATE_INFO};
+ localInfo.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_LOCAL;
+ localInfo.poseInReferenceSpace = XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}};
+ return check(xrCreateReferenceSpace(mSession, &localInfo, &mLocalSpace),
+ "xrCreateReferenceSpace(LOCAL)");
+}
+
+/*
+ * MediaCodec is the producer for an Android surface swapchain, so the image description
+ * fields are not really ours to choose. The extension documents them as ignored, but Meta's
+ * runtime validates them and wants them zeroed: passing the "obvious" 1s gets
+ * XR_ERROR_VALIDATION_FAILURE on a Quest 3 (Horizon OS, Android 14), while all-zero is
+ * accepted. The zeroed shape is therefore first; the rest stay as a fallback for a runtime
+ * that wants something else, and the accepted one is logged.
+ */
+bool XrGoggleSession::createSwapchain(JNIEnv* env)
+{
+ // GL_RGBA8, for the variant where the runtime wants a real format.
+ constexpr int64_t kGlRgba8 = 0x8058;
+
+ struct Variant
+ {
+ const char* what;
+ XrSwapchainUsageFlags usage;
+ int64_t format;
+ uint32_t counts; // sampleCount / faceCount / arraySize / mipCount
+ };
+ const Variant variants[] = {
+ {"ignored fields zeroed", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, 0, 0},
+ {"ignored fields set to 1", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, 0, 1},
+ {"zeroed + color attachment",
+ XR_SWAPCHAIN_USAGE_SAMPLED_BIT | XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT,
+ 0,
+ 0},
+ {"zeroed + explicit RGBA8", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, kGlRgba8, 0},
+ {"1s + explicit RGBA8", XR_SWAPCHAIN_USAGE_SAMPLED_BIT, kGlRgba8, 1},
+ };
+
+ jobject surface = nullptr;
+ XrResult last = XR_ERROR_VALIDATION_FAILURE;
+ for (const auto& variant : variants)
+ {
+ XrSwapchainCreateInfo info{XR_TYPE_SWAPCHAIN_CREATE_INFO};
+ info.usageFlags = variant.usage;
+ info.format = variant.format;
+ info.sampleCount = variant.counts;
+ info.width = (uint32_t) mSwapchainWidth;
+ info.height = (uint32_t) mSwapchainHeight;
+ info.faceCount = variant.counts;
+ info.arraySize = variant.counts;
+ info.mipCount = variant.counts;
+
+ surface = nullptr;
+ last = mXrCreateSwapchainAndroidSurfaceKHR(mSession, &info, &mSwapchain, &surface);
+ if (XR_SUCCEEDED(last) && surface != nullptr)
+ {
+ LOGI("android surface swapchain %dx%d created (%s)",
+ mSwapchainWidth,
+ mSwapchainHeight,
+ variant.what);
+ mVideoSurface = env->NewGlobalRef(surface);
+ return true;
+ }
+ char name[XR_MAX_RESULT_STRING_SIZE] = {0};
+ xrResultToString(mInstance, last, name);
+ LOGW("swapchain variant \'%s\' rejected: %s", variant.what, name);
+ if (XR_SUCCEEDED(last) && mSwapchain != XR_NULL_HANDLE)
+ {
+ // Succeeded but handed back no Surface - not usable, do not leak it.
+ xrDestroySwapchain(mSwapchain);
+ mSwapchain = XR_NULL_HANDLE;
+ }
+ }
+
+ // Nothing worked. Dump what the runtime does offer so the next attempt is informed.
+ uint32_t formatCount = 0;
+ if (XR_SUCCEEDED(xrEnumerateSwapchainFormats(mSession, 0, &formatCount, nullptr)) &&
+ formatCount > 0)
+ {
+ std::vector formats(formatCount);
+ if (XR_SUCCEEDED(
+ xrEnumerateSwapchainFormats(mSession, formatCount, &formatCount, formats.data())))
+ {
+ for (int64_t f : formats)
+ {
+ LOGI("runtime swapchain format 0x%llx", (unsigned long long) f);
+ }
+ }
+ }
+ return check(last, "xrCreateSwapchainAndroidSurfaceKHR");
+}
+
+XrAction XrGoggleSession::boolAction(const char* name, const char* localized)
+{
+ XrActionCreateInfo info{XR_TYPE_ACTION_CREATE_INFO};
+ info.actionType = XR_ACTION_TYPE_BOOLEAN_INPUT;
+ std::strncpy(info.actionName, name, XR_MAX_ACTION_NAME_SIZE - 1);
+ std::strncpy(info.localizedActionName, localized, XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1);
+ XrAction action = XR_NULL_HANDLE;
+ if (!check(xrCreateAction(mActionSet, &info, &action), "xrCreateAction"))
+ {
+ return XR_NULL_HANDLE;
+ }
+ return action;
+}
+
+bool XrGoggleSession::createActions()
+{
+ XrActionSetCreateInfo setInfo{XR_TYPE_ACTION_SET_CREATE_INFO};
+ std::strncpy(setInfo.actionSetName, "goggle", XR_MAX_ACTION_SET_NAME_SIZE - 1);
+ std::strncpy(setInfo.localizedActionSetName, "Goggle", XR_MAX_LOCALIZED_ACTION_SET_NAME_SIZE - 1);
+ setInfo.priority = 0;
+ if (!check(xrCreateActionSet(mInstance, &setInfo, &mActionSet), "xrCreateActionSet"))
+ {
+ return false;
+ }
+
+ mActionRecenter = boolAction("recenter", "Recenter view");
+ mActionPassthrough = boolAction("passthrough", "Toggle passthrough");
+ mActionRecord = boolAction("record", "Toggle recording");
+ mActionLockMode = boolAction("lock_mode", "Toggle head lock");
+ mActionRaise = boolAction("raise", "Move the screen up");
+ mActionLower = boolAction("lower", "Move the screen down");
+ // Separate actions so the hold requirement applies to the hand only - a controller
+ // button should still act the moment it is pressed.
+ mActionHandRecenter = boolAction("hand_recenter", "Recenter view (hold)");
+ mActionHandPassthrough = boolAction("hand_passthrough", "Toggle passthrough (hold)");
+ mActionExit = boolAction("exit", "Leave VR");
+
+ XrActionCreateInfo stick{XR_TYPE_ACTION_CREATE_INFO};
+ stick.actionType = XR_ACTION_TYPE_VECTOR2F_INPUT;
+ std::strncpy(stick.actionName, "place", XR_MAX_ACTION_NAME_SIZE - 1);
+ std::strncpy(stick.localizedActionName, "Move the screen", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1);
+ if (!check(xrCreateAction(mActionSet, &stick, &mActionStick), "xrCreateAction(place)"))
+ {
+ return false;
+ }
+
+ // Analog, so the trigger and grip pull the panel in and push it out at a rate.
+ XrActionCreateInfo axis{XR_TYPE_ACTION_CREATE_INFO};
+ axis.actionType = XR_ACTION_TYPE_FLOAT_INPUT;
+ std::strncpy(axis.actionName, "nearer", XR_MAX_ACTION_NAME_SIZE - 1);
+ std::strncpy(axis.localizedActionName, "Pull the screen closer", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1);
+ if (!check(xrCreateAction(mActionSet, &axis, &mActionNearer), "xrCreateAction(nearer)"))
+ {
+ return false;
+ }
+ std::strncpy(axis.actionName, "farther", XR_MAX_ACTION_NAME_SIZE - 1);
+ std::strncpy(axis.localizedActionName, "Push the screen away", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1);
+ if (!check(xrCreateAction(mActionSet, &axis, &mActionFarther), "xrCreateAction(farther)"))
+ {
+ return false;
+ }
+
+ XrActionCreateInfo haptic{XR_TYPE_ACTION_CREATE_INFO};
+ haptic.actionType = XR_ACTION_TYPE_VIBRATION_OUTPUT;
+ std::strncpy(haptic.actionName, "haptic", XR_MAX_ACTION_NAME_SIZE - 1);
+ std::strncpy(haptic.localizedActionName, "Haptic feedback", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1);
+ if (!check(xrCreateAction(mActionSet, &haptic, &mActionHaptic), "xrCreateAction(haptic)"))
+ {
+ return false;
+ }
+
+ // Mirrored left and right, so a single controller in either hand is enough. A/B and
+ // X/Y sit in the same places on their respective controllers, so binding both to the
+ // same action gives the same thumb positions whichever one you picked up.
+ //
+ // thumbstick up/down screen up/down
+ // thumbstick left/right screen smaller/bigger
+ // trigger pull the screen closer
+ // grip push the screen away
+ // A / X (lower button) recenter
+ // B / Y (upper button) toggle passthrough
+ // thumbstick click toggle recording
+ //
+ // Head lock stays on a hand gesture and in the flat settings menu - it is set once,
+ // not adjusted in flight, and there is no button left that exists on both controllers.
+ const XrActionSuggestedBinding touchBindings[] = {
+ {mActionRecenter, path("/user/hand/right/input/a/click")},
+ {mActionRecenter, path("/user/hand/left/input/x/click")},
+ {mActionPassthrough, path("/user/hand/right/input/b/click")},
+ {mActionPassthrough, path("/user/hand/left/input/y/click")},
+ {mActionRecord, path("/user/hand/right/input/thumbstick/click")},
+ {mActionRecord, path("/user/hand/left/input/thumbstick/click")},
+ {mActionStick, path("/user/hand/right/input/thumbstick")},
+ {mActionStick, path("/user/hand/left/input/thumbstick")},
+ {mActionNearer, path("/user/hand/right/input/trigger/value")},
+ {mActionNearer, path("/user/hand/left/input/trigger/value")},
+ {mActionFarther, path("/user/hand/right/input/squeeze/value")},
+ {mActionFarther, path("/user/hand/left/input/squeeze/value")},
+ {mActionExit, path("/user/hand/left/input/menu/click")},
+ {mActionHaptic, path("/user/hand/left/output/haptic")},
+ {mActionHaptic, path("/user/hand/right/output/haptic")},
+ };
+ XrInteractionProfileSuggestedBinding touch{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING};
+ touch.interactionProfile = path("/interaction_profiles/oculus/touch_controller");
+ touch.suggestedBindings = touchBindings;
+ touch.countSuggestedBindings = (uint32_t) (sizeof(touchBindings) / sizeof(touchBindings[0]));
+ if (XR_FAILED(xrSuggestInteractionProfileBindings(mInstance, &touch)))
+ {
+ LOGW("touch controller bindings rejected, controls may be unavailable");
+ }
+
+ // Bare minimum so something still works on a runtime without Touch controllers.
+ const XrActionSuggestedBinding simpleBindings[] = {
+ {mActionRecenter, path("/user/hand/right/input/select/click")},
+ {mActionPassthrough, path("/user/hand/left/input/select/click")},
+ {mActionHaptic, path("/user/hand/left/output/haptic")},
+ {mActionHaptic, path("/user/hand/right/output/haptic")},
+ };
+ XrInteractionProfileSuggestedBinding simple{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING};
+ simple.interactionProfile = path("/interaction_profiles/khr/simple_controller");
+ simple.suggestedBindings = simpleBindings;
+ simple.countSuggestedBindings = (uint32_t) (sizeof(simpleBindings) / sizeof(simpleBindings[0]));
+ if (XR_FAILED(xrSuggestInteractionProfileBindings(mInstance, &simple)))
+ {
+ LOGW("simple controller bindings rejected");
+ }
+
+ // Hand tracking, for flying without controllers. A boolean action may be bound to a
+ // float source - the runtime does the thresholding - so the same four actions work.
+ // Pinch is index-to-thumb, grasp is a whole-hand squeeze, which are far enough apart
+ // not to trigger each other.
+ if (mHasHandInteraction)
+ {
+ // No grasp bindings: a whole-hand squeeze is what you do to pick up a drone, and
+ // having that flip passthrough mid-flight is worse than not having the shortcut.
+ // Pinch survives, but only when held - see heldActionFired().
+ std::vector handBindings = {
+ {mActionHandRecenter, path("/user/hand/right/input/pinch_ext/value")},
+ {mActionHandPassthrough, path("/user/hand/left/input/pinch_ext/value")},
+ };
+ // Thumb swipes along the index finger, for nudging the panel up and down without
+ // spending pinch or grasp on it.
+ const size_t withoutMicrogestures = handBindings.size();
+ if (mHasMicrogestures)
+ {
+ handBindings.push_back(
+ {mActionRaise, path("/user/hand/right/input/swipe_forward_meta")});
+ handBindings.push_back(
+ {mActionLower, path("/user/hand/right/input/swipe_backward_meta")});
+ handBindings.push_back({mActionExit, path("/user/hand/right/input/swipe_left_meta")});
+ }
+
+ XrInteractionProfileSuggestedBinding hands{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING};
+ hands.interactionProfile = path("/interaction_profiles/ext/hand_interaction_ext");
+ hands.suggestedBindings = handBindings.data();
+ hands.countSuggestedBindings = (uint32_t) handBindings.size();
+ XrResult r = xrSuggestInteractionProfileBindings(mInstance, &hands);
+ if (XR_FAILED(r) && handBindings.size() != withoutMicrogestures)
+ {
+ // Suggestions are atomic per profile, so one bad path would cost the working
+ // pinch/grasp bindings too. Drop the microgestures and keep those.
+ char name[XR_MAX_RESULT_STRING_SIZE] = {0};
+ xrResultToString(mInstance, r, name);
+ LOGW("microgesture bindings rejected (%s), retrying without them", name);
+ mHasMicrogestures = false;
+ hands.countSuggestedBindings = (uint32_t) withoutMicrogestures;
+ r = xrSuggestInteractionProfileBindings(mInstance, &hands);
+ }
+ if (XR_FAILED(r))
+ {
+ char name[XR_MAX_RESULT_STRING_SIZE] = {0};
+ xrResultToString(mInstance, r, name);
+ LOGW("hand interaction bindings rejected: %s", name);
+ }
+ else
+ {
+ LOGI("hand interaction bindings suggested (held pinch%s)",
+ mHasMicrogestures ? " + microgesture height" : "");
+ }
+ }
+
+ XrSessionActionSetsAttachInfo attach{XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO};
+ attach.countActionSets = 1;
+ attach.actionSets = &mActionSet;
+ return check(xrAttachSessionActionSets(mSession, &attach), "xrAttachSessionActionSets");
+}
+
+bool XrGoggleSession::createPassthrough()
+{
+ XrPassthroughCreateInfoFB info{XR_TYPE_PASSTHROUGH_CREATE_INFO_FB};
+ info.flags = 0;
+ if (!check(mXrCreatePassthroughFB(mSession, &info, &mPassthrough), "xrCreatePassthroughFB"))
+ {
+ return false;
+ }
+
+ XrPassthroughLayerCreateInfoFB layerInfo{XR_TYPE_PASSTHROUGH_LAYER_CREATE_INFO_FB};
+ layerInfo.passthrough = mPassthrough;
+ layerInfo.purpose = XR_PASSTHROUGH_LAYER_PURPOSE_RECONSTRUCTION_FB;
+ layerInfo.flags = 0;
+ if (!check(mXrCreatePassthroughLayerFB(mSession, &layerInfo, &mPassthroughLayer),
+ "xrCreatePassthroughLayerFB"))
+ {
+ return false;
+ }
+
+ if (mXrPassthroughLayerSetStyleFB != nullptr)
+ {
+ XrPassthroughStyleFB style{XR_TYPE_PASSTHROUGH_STYLE_FB};
+ style.textureOpacityFactor = 1.0f;
+ style.edgeColor = XrColor4f{0.0f, 0.0f, 0.0f, 0.0f};
+ mXrPassthroughLayerSetStyleFB(mPassthroughLayer, &style);
+ }
+ return true;
+}
+
+// ---------------------------------------------------------------------------------
+// settings
+// ---------------------------------------------------------------------------------
+
+void XrGoggleSession::setSwapchainSize(int width, int height)
+{
+ if (width >= 320 && height >= 240 && width <= 4096 && height <= 4096)
+ {
+ mSwapchainWidth = width;
+ mSwapchainHeight = height;
+ mVideoWidth = width;
+ mVideoHeight = height;
+ }
+}
+
+void XrGoggleSession::setVideoResolution(int width, int height)
+{
+ if (width > 0 && height > 0)
+ {
+ mVideoWidth = width;
+ mVideoHeight = height;
+ }
+}
+
+void XrGoggleSession::setQuadDistance(float meters)
+{
+ mQuadDistance = clampf(meters, kMinDistance, kMaxDistance);
+}
+
+void XrGoggleSession::setQuadWidth(float meters)
+{
+ mQuadWidth = clampf(meters, kMinWidth, kMaxWidth);
+}
+
+void XrGoggleSession::setQuadHeightOffset(float meters)
+{
+ mQuadHeightOffset = clampf(meters, -kMaxHeightOffset, kMaxHeightOffset);
+}
+
+void XrGoggleSession::setPassthrough(bool enabled)
+{
+ mPassthroughWanted = enabled;
+}
+
+void XrGoggleSession::setSharpening(bool enabled)
+{
+ mSharpening = enabled;
+}
+
+void XrGoggleSession::setHeadLocked(bool enabled)
+{
+ mHeadLocked = enabled;
+ if (!enabled)
+ {
+ // Anchor where the pilot is looking right now instead of at the origin.
+ mRecenterRequested = true;
+ }
+}
+
+void XrGoggleSession::requestRefreshRate(float hz)
+{
+ mWantedRefreshRate = hz;
+}
+
+std::vector XrGoggleSession::refreshRates()
+{
+ return mRefreshRates;
+}
+
+void XrGoggleSession::requestHaptic(float amplitude, int durationMs)
+{
+ mPendingHapticAmplitude = clampf(amplitude, 0.0f, 1.0f);
+ mPendingHapticMs = durationMs < 1 ? 1 : durationMs;
+}
+
+// ---------------------------------------------------------------------------------
+// frame loop
+// ---------------------------------------------------------------------------------
+
+void XrGoggleSession::runLoop(JNIEnv* env, jobject listener)
+{
+ bool exitLoop = false;
+ while (!exitLoop && !mStopRequested)
+ {
+ pollEvents(env, listener, &exitLoop);
+ if (exitLoop) break;
+
+ if (!mSessionRunning)
+ {
+ // Idle without burning a core while the runtime has us backgrounded.
+ struct timespec ts = {0, 20 * 1000 * 1000};
+ nanosleep(&ts, nullptr);
+ continue;
+ }
+
+ applyPendingRefreshRate();
+ syncActions(env, listener);
+ applyPassthroughState();
+ renderFrame(env, listener);
+ }
+
+ if (mSessionRunning && mSession != XR_NULL_HANDLE)
+ {
+ xrRequestExitSession(mSession);
+ // Drain until the runtime actually stops the session, so teardown is clean.
+ for (int i = 0; i < 200 && mSessionRunning; ++i)
+ {
+ bool ignored = false;
+ pollEvents(env, listener, &ignored);
+ struct timespec ts = {0, 10 * 1000 * 1000};
+ nanosleep(&ts, nullptr);
+ }
+ }
+}
+
+void XrGoggleSession::pollEvents(JNIEnv* env, jobject listener, bool* exitLoop)
+{
+ (void) env;
+ (void) listener;
+ XrEventDataBuffer event{XR_TYPE_EVENT_DATA_BUFFER};
+ while (true)
+ {
+ event.type = XR_TYPE_EVENT_DATA_BUFFER;
+ XrResult r = xrPollEvent(mInstance, &event);
+ if (r != XR_SUCCESS) break;
+
+ switch (event.type)
+ {
+ case XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED:
+ {
+ const auto* changed = reinterpret_cast(&event);
+ mSessionState = changed->state;
+ LOGI("session state -> %d", (int) mSessionState);
+ if (mSessionState == XR_SESSION_STATE_READY)
+ {
+ XrSessionBeginInfo begin{XR_TYPE_SESSION_BEGIN_INFO};
+ begin.primaryViewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO;
+ if (check(xrBeginSession(mSession, &begin), "xrBeginSession"))
+ {
+ mSessionRunning = true;
+ if (mHasSimultaneous)
+ {
+ XrSimultaneousHandsAndControllersTrackingResumeInfoMETA resume{
+ XR_TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META};
+ if (XR_SUCCEEDED(mXrResumeSimultaneous(mSession, &resume)))
+ {
+ LOGI("hands and controllers tracked simultaneously");
+ }
+ else
+ {
+ LOGW("xrResumeSimultaneousHandsAndControllersTrackingMETA failed");
+ }
+ }
+ logInteractionProfiles();
+ }
+ }
+ else if (mSessionState == XR_SESSION_STATE_STOPPING)
+ {
+ mSessionRunning = false;
+ xrEndSession(mSession);
+ }
+ else if (mSessionState == XR_SESSION_STATE_EXITING ||
+ mSessionState == XR_SESSION_STATE_LOSS_PENDING)
+ {
+ mSessionRunning = false;
+ *exitLoop = true;
+ }
+ break;
+ }
+ case XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING:
+ {
+ LOGW("instance loss pending");
+ mSessionRunning = false;
+ *exitLoop = true;
+ break;
+ }
+ case XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED:
+ {
+ // The only way to see which of the suggested profiles is actually live.
+ logInteractionProfiles();
+ break;
+ }
+ case XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING:
+ {
+ // The runtime recentered underneath us; re-anchor on the next frame.
+ if (!mHeadLocked) mRecenterRequested = true;
+ break;
+ }
+ default:
+ break;
+ }
+ }
+}
+
+// A hand action fires once, after being held for kHandHoldNs. Anything shorter is treated
+// as an accident.
+bool XrGoggleSession::heldActionFired(XrAction action, int slot, XrTime now)
+{
+ if (action == XR_NULL_HANDLE) return false;
+ HeldButton& held = mHeld[slot];
+
+ XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO};
+ get.action = action;
+ XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN};
+ if (XR_FAILED(xrGetActionStateBoolean(mSession, &get, &state)) || !state.isActive)
+ {
+ held = HeldButton{};
+ return false;
+ }
+
+ if (!state.currentState)
+ {
+ held = HeldButton{};
+ return false;
+ }
+ if (!held.down)
+ {
+ held.down = true;
+ held.fired = false;
+ held.downTime = now;
+ return false;
+ }
+ if (!held.fired && (now - held.downTime) >= kHandHoldNs)
+ {
+ held.fired = true;
+ return true;
+ }
+ return false;
+}
+
+void XrGoggleSession::logInteractionProfiles()
+{
+ for (const char* hand : {"/user/hand/left", "/user/hand/right"})
+ {
+ XrInteractionProfileState state{XR_TYPE_INTERACTION_PROFILE_STATE};
+ if (XR_FAILED(xrGetCurrentInteractionProfile(mSession, path(hand), &state)))
+ {
+ continue;
+ }
+ if (state.interactionProfile == XR_NULL_PATH)
+ {
+ LOGI("%s: no interaction profile active", hand);
+ continue;
+ }
+ char name[XR_MAX_PATH_LENGTH] = {0};
+ uint32_t written = 0;
+ if (XR_SUCCEEDED(
+ xrPathToString(mInstance, state.interactionProfile, sizeof(name), &written, name)))
+ {
+ LOGI("%s: %s", hand, name);
+ }
+ }
+}
+
+void XrGoggleSession::applyPendingRefreshRate()
+{
+ if (!mHasRefreshRate) return;
+ const float wanted = mWantedRefreshRate.load();
+ if (wanted <= 0.0f || std::fabs(wanted - mAppliedRefreshRate.load()) < 0.01f) return;
+
+ // Only ask for a rate the runtime actually offers, otherwise it returns an error.
+ bool supported = mRefreshRates.empty();
+ for (float hz : mRefreshRates)
+ {
+ if (std::fabs(hz - wanted) < 0.51f)
+ {
+ supported = true;
+ break;
+ }
+ }
+ if (!supported)
+ {
+ LOGW("refresh rate %.1f Hz not offered by the runtime", wanted);
+ mAppliedRefreshRate = wanted;
+ return;
+ }
+ if (XR_SUCCEEDED(mXrRequestDisplayRefreshRateFB(mSession, wanted)))
+ {
+ LOGI("display refresh rate set to %.1f Hz", wanted);
+ }
+ else
+ {
+ LOGW("xrRequestDisplayRefreshRateFB(%.1f) failed", wanted);
+ }
+ mAppliedRefreshRate = wanted;
+}
+
+// Passthrough keeps the cameras and reconstruction running, so only pay for it while it
+// is on screen.
+void XrGoggleSession::applyPassthroughState()
+{
+ if (!mHasPassthrough || mPassthrough == XR_NULL_HANDLE) return;
+ const bool wanted = mPassthroughWanted.load();
+ if (wanted == mPassthroughRunning) return;
+
+ if (wanted)
+ {
+ if (mXrPassthroughStartFB == nullptr) return;
+ if (!XR_SUCCEEDED(mXrPassthroughStartFB(mPassthrough)))
+ {
+ LOGW("xrPassthroughStartFB failed");
+ mPassthroughWanted = false;
+ return;
+ }
+ if (mXrPassthroughLayerResumeFB != nullptr)
+ {
+ mXrPassthroughLayerResumeFB(mPassthroughLayer);
+ }
+ mPassthroughRunning = true;
+ LOGI("passthrough started");
+ }
+ else
+ {
+ if (mXrPassthroughLayerPauseFB != nullptr)
+ {
+ mXrPassthroughLayerPauseFB(mPassthroughLayer);
+ }
+ if (mXrPassthroughPauseFB != nullptr)
+ {
+ mXrPassthroughPauseFB(mPassthrough);
+ }
+ mPassthroughRunning = false;
+ LOGI("passthrough paused");
+ }
+}
+
+void XrGoggleSession::applyPendingHaptic()
+{
+ const float amplitude = mPendingHapticAmplitude.exchange(0.0f);
+ const int ms = mPendingHapticMs.exchange(0);
+ if (amplitude <= 0.0f || ms <= 0 || mActionHaptic == XR_NULL_HANDLE) return;
+
+ XrHapticVibration vibration{XR_TYPE_HAPTIC_VIBRATION};
+ vibration.amplitude = amplitude;
+ vibration.frequency = XR_FREQUENCY_UNSPECIFIED;
+ vibration.duration = (XrDuration) ms * 1000000LL;
+
+ XrHapticActionInfo info{XR_TYPE_HAPTIC_ACTION_INFO};
+ info.action = mActionHaptic;
+ info.subactionPath = XR_NULL_PATH;
+ xrApplyHapticFeedback(mSession, &info, (const XrHapticBaseHeader*) &vibration);
+}
+
+void XrGoggleSession::syncActions(JNIEnv* env, jobject listener)
+{
+ if (mActionSet == XR_NULL_HANDLE) return;
+
+ XrActiveActionSet active{mActionSet, XR_NULL_PATH};
+ XrActionsSyncInfo sync{XR_TYPE_ACTIONS_SYNC_INFO};
+ sync.countActiveActionSets = 1;
+ sync.activeActionSets = &active;
+ if (XR_FAILED(xrSyncActions(mSession, &sync))) return;
+
+ jclass cls = env->GetObjectClass(listener);
+ jmethodID onButton = env->GetMethodID(cls, "onXrButton", "(I)V");
+
+ struct
+ {
+ XrAction action;
+ int button;
+ } buttons[] = {
+ {mActionRecenter, BUTTON_RECENTER},
+ {mActionPassthrough, BUTTON_PASSTHROUGH},
+ {mActionRecord, BUTTON_RECORD},
+ {mActionLockMode, BUTTON_LOCK_MODE},
+ {mActionRaise, BUTTON_RAISE},
+ {mActionLower, BUTTON_LOWER},
+ {mActionExit, BUTTON_EXIT},
+ };
+
+ for (const auto& b : buttons)
+ {
+ if (b.action == XR_NULL_HANDLE) continue;
+ XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO};
+ get.action = b.action;
+ XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN};
+ if (XR_FAILED(xrGetActionStateBoolean(mSession, &get, &state))) continue;
+ // Rising edge only.
+ if (state.isActive && state.changedSinceLastSync && state.currentState)
+ {
+ switch (b.button)
+ {
+ case BUTTON_RECENTER:
+ mRecenterRequested = true;
+ requestHaptic(0.4f, 30);
+ break;
+ case BUTTON_PASSTHROUGH:
+ mPassthroughWanted = !mPassthroughWanted.load();
+ requestHaptic(0.4f, 30);
+ break;
+ case BUTTON_LOCK_MODE:
+ setHeadLocked(!mHeadLocked.load());
+ requestHaptic(0.4f, 30);
+ break;
+ // Recenter and passthrough are dispatched from the held-gesture path too, so
+ // they are handled where the press is detected rather than here.
+ case BUTTON_RAISE:
+ setQuadHeightOffset(mQuadHeightOffset.load() + kHeightStep);
+ requestHaptic(0.25f, 20);
+ break;
+ case BUTTON_LOWER:
+ setQuadHeightOffset(mQuadHeightOffset.load() - kHeightStep);
+ requestHaptic(0.25f, 20);
+ break;
+ case BUTTON_EXIT:
+ // Handled on the Java side; just confirm it was registered.
+ requestHaptic(0.6f, 80);
+ break;
+ default:
+ break;
+ }
+ if (onButton != nullptr)
+ {
+ env->CallVoidMethod(listener, onButton, (jint) b.button);
+ if (env->ExceptionCheck()) env->ExceptionClear();
+ }
+ }
+ }
+
+ // Hand gestures, which need a deliberate hold before they count.
+ const XrTime nowNs = mLastPredictedDisplayTime;
+ if (heldActionFired(mActionHandRecenter, 0, nowNs))
+ {
+ mRecenterRequested = true;
+ requestHaptic(0.5f, 60);
+ if (onButton != nullptr)
+ {
+ env->CallVoidMethod(listener, onButton, (jint) BUTTON_RECENTER);
+ if (env->ExceptionCheck()) env->ExceptionClear();
+ }
+ }
+ if (heldActionFired(mActionHandPassthrough, 1, nowNs))
+ {
+ mPassthroughWanted = !mPassthroughWanted.load();
+ requestHaptic(0.5f, 60);
+ if (onButton != nullptr)
+ {
+ env->CallVoidMethod(listener, onButton, (jint) BUTTON_PASSTHROUGH);
+ if (env->ExceptionCheck()) env->ExceptionClear();
+ }
+ }
+
+ const float dt = 1.0f / 72.0f; // good enough for a manual adjustment ramp
+
+ XrActionStateGetInfo get{XR_TYPE_ACTION_STATE_GET_INFO};
+ if (mActionStick != XR_NULL_HANDLE)
+ {
+ get.action = mActionStick;
+ XrActionStateVector2f stickState{XR_TYPE_ACTION_STATE_VECTOR2F};
+ if (XR_SUCCEEDED(xrGetActionStateVector2f(mSession, &get, &stickState)) && stickState.isActive)
+ {
+ // Either thumbstick: up/down raises the panel, left/right resizes it.
+ const float y = stickState.currentState.y;
+ if (std::fabs(y) > kStickDeadzone)
+ {
+ setQuadHeightOffset(mQuadHeightOffset.load() + y * kHeightPerSec * dt);
+ }
+ const float x = stickState.currentState.x;
+ if (std::fabs(x) > kStickDeadzone)
+ {
+ setQuadWidth(mQuadWidth.load() + x * kWidthPerSec * dt);
+ }
+ }
+ }
+
+ // Trigger pulls the panel in, grip pushes it out, both analog.
+ float distanceRate = 0.0f;
+ for (int i = 0; i < 2; ++i)
+ {
+ const XrAction action = (i == 0) ? mActionNearer : mActionFarther;
+ if (action == XR_NULL_HANDLE) continue;
+ get.action = action;
+ XrActionStateFloat axisState{XR_TYPE_ACTION_STATE_FLOAT};
+ if (XR_SUCCEEDED(xrGetActionStateFloat(mSession, &get, &axisState)) && axisState.isActive &&
+ axisState.currentState > kTriggerDeadzone)
+ {
+ distanceRate += (i == 0 ? -1.0f : 1.0f) * axisState.currentState;
+ }
+ }
+ if (std::fabs(distanceRate) > 0.0f)
+ {
+ setQuadDistance(mQuadDistance.load() + distanceRate * kDistancePerSec * dt);
+ }
+}
+
+void XrGoggleSession::recenterAt(XrTime time)
+{
+ XrSpaceLocation location{XR_TYPE_SPACE_LOCATION};
+ if (XR_FAILED(xrLocateSpace(mViewSpace, mLocalSpace, time, &location)))
+ {
+ return;
+ }
+ if ((location.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) == 0 ||
+ (location.locationFlags & XR_SPACE_LOCATION_POSITION_VALID_BIT) == 0)
+ {
+ return;
+ }
+
+ const float yaw = yawOf(location.pose.orientation);
+ const float distance = mQuadDistance.load();
+ mAnchorPose.orientation = quatFromYaw(yaw);
+ mAnchorPose.position = XrVector3f{location.pose.position.x - std::sin(yaw) * distance,
+ location.pose.position.y,
+ location.pose.position.z - std::cos(yaw) * distance};
+ mAnchorValid = true;
+}
+
+void XrGoggleSession::renderFrame(JNIEnv* env, jobject listener)
+{
+ (void) env;
+ (void) listener;
+
+ XrFrameWaitInfo waitInfo{XR_TYPE_FRAME_WAIT_INFO};
+ XrFrameState frameState{XR_TYPE_FRAME_STATE};
+ if (XR_FAILED(xrWaitFrame(mSession, &waitInfo, &frameState))) return;
+
+ XrFrameBeginInfo beginInfo{XR_TYPE_FRAME_BEGIN_INFO};
+ if (XR_FAILED(xrBeginFrame(mSession, &beginInfo))) return;
+
+ mLastPredictedDisplayTime = frameState.predictedDisplayTime;
+
+ applyPendingHaptic();
+
+ if (mRecenterRequested.exchange(false) || (!mHeadLocked.load() && !mAnchorValid))
+ {
+ recenterAt(frameState.predictedDisplayTime);
+ }
+
+ const bool headLocked = mHeadLocked.load();
+ const bool wantPassthrough =
+ mHasPassthrough && mPassthroughRunning && mPassthroughWanted.load();
+
+ // Video quad. Width is in meters at the configured distance, height follows the
+ // stream's aspect ratio so nothing is stretched.
+ const int vw = mVideoWidth.load();
+ const int vh = mVideoHeight.load();
+ const float aspect = (vw > 0 && vh > 0) ? (float) vh / (float) vw : 9.0f / 16.0f;
+ const float widthM = mQuadWidth.load();
+
+ XrCompositionLayerSettingsFB layerSettings{XR_TYPE_COMPOSITION_LAYER_SETTINGS_FB};
+ layerSettings.layerFlags = XR_COMPOSITION_LAYER_SETTINGS_QUALITY_SHARPENING_BIT_FB;
+
+ // Everything that produces into an Android Surface - MediaCodec, and Canvas for the
+ // status screen - has its origin at the top left, while the compositor samples a
+ // swapchain image from the bottom left. Without this the picture is upside down.
+ XrCompositionLayerImageLayoutFB imageLayout{XR_TYPE_COMPOSITION_LAYER_IMAGE_LAYOUT_FB};
+ imageLayout.flags = XR_COMPOSITION_LAYER_IMAGE_LAYOUT_VERTICAL_FLIP_BIT_FB;
+
+ // Chain whatever applies onto the quad.
+ const void* layerChain = nullptr;
+ if (mHasLayerSettings && mSharpening.load())
+ {
+ layerSettings.next = layerChain;
+ layerChain = &layerSettings;
+ }
+ if (mHasImageLayout)
+ {
+ imageLayout.next = const_cast(layerChain);
+ layerChain = &imageLayout;
+ }
+
+ XrCompositionLayerQuad quad{XR_TYPE_COMPOSITION_LAYER_QUAD};
+ quad.next = layerChain;
+ quad.layerFlags = 0; // opaque: the video has to cover passthrough behind it
+ quad.space = headLocked ? mViewSpace : mLocalSpace;
+ quad.eyeVisibility = XR_EYE_VISIBILITY_BOTH;
+ quad.subImage.swapchain = mSwapchain;
+ quad.subImage.imageArrayIndex = 0;
+ // Clamp to the swapchain the runtime handed us; MediaCodec resizes the buffers to the
+ // stream resolution, which is normally the size this was created with.
+ quad.subImage.imageRect.offset = XrOffset2Di{0, 0};
+ quad.subImage.imageRect.extent =
+ XrExtent2Di{vw > 0 && vw <= mSwapchainWidth ? vw : mSwapchainWidth,
+ vh > 0 && vh <= mSwapchainHeight ? vh : mSwapchainHeight};
+ const float heightOffset = mQuadHeightOffset.load();
+ if (headLocked)
+ {
+ quad.pose =
+ XrPosef{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, heightOffset, -mQuadDistance.load()}};
+ }
+ else
+ {
+ quad.pose = mAnchorPose;
+ quad.pose.position.y += heightOffset;
+ }
+ quad.size = XrExtent2Df{widthM, widthM * aspect};
+
+ XrCompositionLayerPassthroughFB passthroughLayer{XR_TYPE_COMPOSITION_LAYER_PASSTHROUGH_FB};
+ passthroughLayer.flags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT;
+ passthroughLayer.space = XR_NULL_HANDLE;
+ passthroughLayer.layerHandle = mPassthroughLayer;
+
+ const XrCompositionLayerBaseHeader* layers[2];
+ uint32_t layerCount = 0;
+ if (wantPassthrough)
+ {
+ layers[layerCount++] = (const XrCompositionLayerBaseHeader*) &passthroughLayer;
+ }
+ layers[layerCount++] = (const XrCompositionLayerBaseHeader*) &quad;
+
+ XrFrameEndInfo endInfo{XR_TYPE_FRAME_END_INFO};
+ endInfo.displayTime = frameState.predictedDisplayTime;
+ endInfo.environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE;
+ // shouldRender false means the runtime wants nothing on screen (e.g. system overlay
+ // in front of us); submitting zero layers keeps the frame loop pacing intact.
+ endInfo.layerCount = frameState.shouldRender ? layerCount : 0;
+ endInfo.layers = frameState.shouldRender ? layers : nullptr;
+ xrEndFrame(mSession, &endInfo);
+}
+
+// ---------------------------------------------------------------------------------
+// teardown
+// ---------------------------------------------------------------------------------
+
+void XrGoggleSession::destroy(JNIEnv* env)
+{
+ if (mPassthroughLayer != XR_NULL_HANDLE && mXrDestroyPassthroughLayerFB != nullptr)
+ {
+ mXrDestroyPassthroughLayerFB(mPassthroughLayer);
+ mPassthroughLayer = XR_NULL_HANDLE;
+ }
+ if (mPassthrough != XR_NULL_HANDLE && mXrDestroyPassthroughFB != nullptr)
+ {
+ if (mPassthroughRunning && mXrPassthroughPauseFB != nullptr)
+ {
+ mXrPassthroughPauseFB(mPassthrough);
+ }
+ mXrDestroyPassthroughFB(mPassthrough);
+ mPassthrough = XR_NULL_HANDLE;
+ mPassthroughRunning = false;
+ }
+ if (mActionSet != XR_NULL_HANDLE)
+ {
+ xrDestroyActionSet(mActionSet);
+ mActionSet = XR_NULL_HANDLE;
+ }
+ if (mSwapchain != XR_NULL_HANDLE)
+ {
+ xrDestroySwapchain(mSwapchain);
+ mSwapchain = XR_NULL_HANDLE;
+ }
+ if (mVideoSurface != nullptr)
+ {
+ env->DeleteGlobalRef(mVideoSurface);
+ mVideoSurface = nullptr;
+ }
+ if (mViewSpace != XR_NULL_HANDLE)
+ {
+ xrDestroySpace(mViewSpace);
+ mViewSpace = XR_NULL_HANDLE;
+ }
+ if (mLocalSpace != XR_NULL_HANDLE)
+ {
+ xrDestroySpace(mLocalSpace);
+ mLocalSpace = XR_NULL_HANDLE;
+ }
+ if (mSession != XR_NULL_HANDLE)
+ {
+ xrDestroySession(mSession);
+ mSession = XR_NULL_HANDLE;
+ }
+ if (mEglDisplay != EGL_NO_DISPLAY)
+ {
+ eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+ if (mEglSurface != EGL_NO_SURFACE) eglDestroySurface(mEglDisplay, mEglSurface);
+ if (mEglContext != EGL_NO_CONTEXT) eglDestroyContext(mEglDisplay, mEglContext);
+ eglTerminate(mEglDisplay);
+ mEglSurface = EGL_NO_SURFACE;
+ mEglContext = EGL_NO_CONTEXT;
+ mEglDisplay = EGL_NO_DISPLAY;
+ }
+ if (mInstance != XR_NULL_HANDLE)
+ {
+ xrDestroyInstance(mInstance);
+ mInstance = XR_NULL_HANDLE;
+ }
+}
diff --git a/app/xr/src/main/cpp/XrGoggleSession.h b/app/xr/src/main/cpp/XrGoggleSession.h
new file mode 100644
index 00000000..8c18a24a
--- /dev/null
+++ b/app/xr/src/main/cpp/XrGoggleSession.h
@@ -0,0 +1,223 @@
+// Immersive OpenXR presentation of the decoded video stream.
+//
+// The video never touches this code: xrCreateSwapchainAndroidSurfaceKHR hands out an
+// Android Surface that is the producer side of a compositor swapchain, MediaCodec
+// decodes straight into it, and the runtime composites it as a quad layer. There is no
+// GL blit and no readback in the video path - the only reason an EGL context exists at
+// all is that OpenXR requires a graphics binding for the session.
+
+#ifndef PIXELPILOT_XRGOGGLESESSION_H
+#define PIXELPILOT_XRGOGGLESESSION_H
+
+#include
+#include
+#include
+#include
+
+#define XR_USE_PLATFORM_ANDROID
+#define XR_USE_GRAPHICS_API_OPENGL_ES
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+class XrGoggleSession
+{
+ public:
+ // Keep in sync with XrGoggleSession.java
+ enum Button
+ {
+ BUTTON_RECENTER = 0,
+ BUTTON_PASSTHROUGH = 1,
+ BUTTON_RECORD = 2,
+ BUTTON_LOCK_MODE = 3,
+ BUTTON_RAISE = 4,
+ BUTTON_LOWER = 5,
+ BUTTON_EXIT = 6,
+ };
+
+ XrGoggleSession() = default;
+ ~XrGoggleSession() = default;
+
+ // Brings up loader, instance, system, EGL, session, swapchain and actions.
+ // Returns false and leaves lastError() set on any failure; the caller is expected to
+ // fall back to the flat activity.
+ bool create(JNIEnv* env, jobject activity);
+
+ // Blocks until requestStop() or an unrecoverable error. Must be called from the same
+ // thread that called create(), and that thread must be attached to the JVM: button
+ // presses are dispatched to listener.onXrButton(int) from here.
+ void runLoop(JNIEnv* env, jobject listener);
+
+ void destroy(JNIEnv* env);
+
+ void requestStop() { mStopRequested = true; }
+
+ // The producer Surface for MediaCodec. Global ref, valid between create() and
+ // destroy().
+ jobject videoSurface() const { return mVideoSurface; }
+
+ std::string lastError();
+
+ std::vector refreshRates();
+
+ // --- live settings, read by the frame loop -------------------------------------
+ void setVideoResolution(int width, int height);
+ void setQuadDistance(float meters);
+ void setQuadWidth(float meters);
+ void setQuadHeightOffset(float meters);
+ void setPassthrough(bool enabled);
+ void setSharpening(bool enabled);
+ void setHeadLocked(bool enabled);
+ void requestRefreshRate(float hz);
+ void requestRecenter() { mRecenterRequested = true; }
+ // amplitude 0..1, duration in milliseconds
+ void requestHaptic(float amplitude, int durationMs);
+
+ float quadDistance() const { return mQuadDistance; }
+ float quadWidth() const { return mQuadWidth; }
+ float quadHeightOffset() const { return mQuadHeightOffset; }
+ bool headLocked() const { return mHeadLocked; }
+ bool passthroughEnabled() const { return mPassthroughWanted; }
+
+ // Swapchain geometry is fixed at create() time; the caller seeds it from the last
+ // known stream resolution.
+ void setSwapchainSize(int width, int height);
+
+ // Writable directory used to drop a runtime manifest, see resolveRuntime().
+ void setManifestDir(std::string dir) { mManifestDir = std::move(dir); }
+
+ private:
+ bool initLoader(JNIEnv* env, jobject activity);
+ bool resolveRuntime();
+ bool runtimeAnswers();
+ bool pointLoaderAt(const char* libraryPath);
+ bool createInstance(JNIEnv* env, jobject activity);
+ bool createEgl();
+ bool createSession();
+ bool createSpaces();
+ bool createSwapchain(JNIEnv* env);
+ bool createActions();
+ bool createPassthrough();
+
+ void pollEvents(JNIEnv* env, jobject listener, bool* exitLoop);
+ void renderFrame(JNIEnv* env, jobject listener);
+ void syncActions(JNIEnv* env, jobject listener);
+ void applyPendingHaptic();
+ void applyPassthroughState();
+ // Returns true on the frame a held action should fire, once per press.
+ bool heldActionFired(XrAction action, int slot, XrTime now);
+ void logInteractionProfiles();
+ void applyPendingRefreshRate();
+ void recenterAt(XrTime time);
+
+ bool extensionSupported(const char* name) const;
+ bool check(XrResult result, const char* what);
+ void setError(const std::string& what);
+ XrPath path(const char* str);
+ XrAction boolAction(const char* name, const char* localized);
+
+ // --- OpenXR handles -----------------------------------------------------------
+ XrInstance mInstance = XR_NULL_HANDLE;
+ XrSystemId mSystemId = XR_NULL_SYSTEM_ID;
+ XrSession mSession = XR_NULL_HANDLE;
+ XrSpace mViewSpace = XR_NULL_HANDLE;
+ XrSpace mLocalSpace = XR_NULL_HANDLE;
+ XrSwapchain mSwapchain = XR_NULL_HANDLE;
+ jobject mVideoSurface = nullptr;
+
+ XrPassthroughFB mPassthrough = XR_NULL_HANDLE;
+ XrPassthroughLayerFB mPassthroughLayer = XR_NULL_HANDLE;
+ bool mPassthroughRunning = false;
+
+ XrActionSet mActionSet = XR_NULL_HANDLE;
+ XrAction mActionRecenter = XR_NULL_HANDLE;
+ XrAction mActionPassthrough = XR_NULL_HANDLE;
+ XrAction mActionRecord = XR_NULL_HANDLE;
+ XrAction mActionLockMode = XR_NULL_HANDLE;
+ XrAction mActionStick = XR_NULL_HANDLE;
+ XrAction mActionNearer = XR_NULL_HANDLE;
+ XrAction mActionFarther = XR_NULL_HANDLE;
+ XrAction mActionHandRecenter = XR_NULL_HANDLE;
+ XrAction mActionHandPassthrough = XR_NULL_HANDLE;
+ XrAction mActionRaise = XR_NULL_HANDLE;
+ XrAction mActionLower = XR_NULL_HANDLE;
+ XrAction mActionExit = XR_NULL_HANDLE;
+ XrAction mActionHaptic = XR_NULL_HANDLE;
+
+ // --- EGL ----------------------------------------------------------------------
+ EGLDisplay mEglDisplay = EGL_NO_DISPLAY;
+ EGLConfig mEglConfig = nullptr;
+ EGLSurface mEglSurface = EGL_NO_SURFACE;
+ EGLContext mEglContext = EGL_NO_CONTEXT;
+
+ // --- extension entry points ---------------------------------------------------
+ PFN_xrCreateSwapchainAndroidSurfaceKHR mXrCreateSwapchainAndroidSurfaceKHR = nullptr;
+ PFN_xrCreatePassthroughFB mXrCreatePassthroughFB = nullptr;
+ PFN_xrDestroyPassthroughFB mXrDestroyPassthroughFB = nullptr;
+ PFN_xrPassthroughStartFB mXrPassthroughStartFB = nullptr;
+ PFN_xrPassthroughPauseFB mXrPassthroughPauseFB = nullptr;
+ PFN_xrCreatePassthroughLayerFB mXrCreatePassthroughLayerFB = nullptr;
+ PFN_xrDestroyPassthroughLayerFB mXrDestroyPassthroughLayerFB = nullptr;
+ PFN_xrPassthroughLayerResumeFB mXrPassthroughLayerResumeFB = nullptr;
+ PFN_xrPassthroughLayerPauseFB mXrPassthroughLayerPauseFB = nullptr;
+ PFN_xrPassthroughLayerSetStyleFB mXrPassthroughLayerSetStyleFB = nullptr;
+ PFN_xrEnumerateDisplayRefreshRatesFB mXrEnumerateDisplayRefreshRatesFB = nullptr;
+ PFN_xrRequestDisplayRefreshRateFB mXrRequestDisplayRefreshRateFB = nullptr;
+ PFN_xrResumeSimultaneousHandsAndControllersTrackingMETA mXrResumeSimultaneous = nullptr;
+
+ std::vector mAvailableExtensions;
+ bool mHasPassthrough = false;
+ bool mHasHandInteraction = false;
+ bool mHasImageLayout = false;
+ bool mHasMicrogestures = false;
+ bool mHasSimultaneous = false;
+ bool mHasLayerSettings = false;
+ bool mHasRefreshRate = false;
+
+ // --- state --------------------------------------------------------------------
+ XrTime mLastPredictedDisplayTime = 0;
+ XrSessionState mSessionState = XR_SESSION_STATE_UNKNOWN;
+ bool mSessionRunning = false;
+ std::atomic mStopRequested{false};
+ std::atomic mRecenterRequested{false};
+
+ std::atomic mVideoWidth{1920};
+ std::atomic mVideoHeight{1080};
+ int mSwapchainWidth = 1920;
+ int mSwapchainHeight = 1080;
+ std::atomic mQuadDistance{1.6f};
+ std::atomic mQuadWidth{2.2f};
+ std::atomic mQuadHeightOffset{0.0f};
+ std::atomic mHeadLocked{true};
+ std::atomic mPassthroughWanted{false};
+ std::atomic mSharpening{true};
+ std::atomic mWantedRefreshRate{0.0f};
+ std::atomic mAppliedRefreshRate{0.0f};
+
+ std::atomic mPendingHapticAmplitude{0.0f};
+ std::atomic mPendingHapticMs{0};
+
+ // Anchor for world-locked mode, in the LOCAL reference space.
+ XrPosef mAnchorPose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}};
+ bool mAnchorValid = false;
+
+ std::string mManifestDir;
+ std::string mRuntimeLibrary;
+ // Press bookkeeping for the hand actions, which need a deliberate hold.
+ struct HeldButton
+ {
+ bool down = false;
+ bool fired = false;
+ XrTime downTime = 0;
+ };
+ HeldButton mHeld[2];
+ std::vector mRefreshRates;
+ std::mutex mErrorMutex;
+ std::string mLastError;
+};
+
+#endif // PIXELPILOT_XRGOGGLESESSION_H
diff --git a/app/xr/src/main/cpp/jni_bindings.cpp b/app/xr/src/main/cpp/jni_bindings.cpp
new file mode 100644
index 00000000..12e68b76
--- /dev/null
+++ b/app/xr/src/main/cpp/jni_bindings.cpp
@@ -0,0 +1,205 @@
+#include
+
+#include "XrGoggleSession.h"
+
+#define JNI_METHOD(return_type, method_name) \
+ extern "C" JNIEXPORT return_type JNICALL Java_com_openipc_xr_XrGoggleSession_##method_name
+
+static inline XrGoggleSession* native(jlong handle)
+{
+ return reinterpret_cast(handle);
+}
+
+JNI_METHOD(jlong, nativeAlloc)(JNIEnv* env, jclass clazz)
+{
+ (void) env;
+ (void) clazz;
+ return reinterpret_cast(new XrGoggleSession());
+}
+
+JNI_METHOD(void, nativeFree)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ delete native(handle);
+}
+
+JNI_METHOD(void, nativeSetSwapchainSize)
+(JNIEnv* env, jclass clazz, jlong handle, jint width, jint height)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setSwapchainSize(width, height);
+}
+
+JNI_METHOD(void, nativeSetManifestDir)(JNIEnv* env, jclass clazz, jlong handle, jstring dir)
+{
+ (void) clazz;
+ if (dir == nullptr) return;
+ const char* chars = env->GetStringUTFChars(dir, nullptr);
+ if (chars != nullptr)
+ {
+ native(handle)->setManifestDir(std::string(chars));
+ env->ReleaseStringUTFChars(dir, chars);
+ }
+}
+
+JNI_METHOD(jboolean, nativeCreate)(JNIEnv* env, jclass clazz, jlong handle, jobject activity)
+{
+ (void) clazz;
+ return native(handle)->create(env, activity) ? JNI_TRUE : JNI_FALSE;
+}
+
+JNI_METHOD(jobject, nativeGetVideoSurface)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ return native(handle)->videoSurface();
+}
+
+JNI_METHOD(void, nativeRunLoop)(JNIEnv* env, jclass clazz, jlong handle, jobject listener)
+{
+ (void) clazz;
+ native(handle)->runLoop(env, listener);
+}
+
+JNI_METHOD(void, nativeRequestStop)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->requestStop();
+}
+
+JNI_METHOD(void, nativeDestroy)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) clazz;
+ native(handle)->destroy(env);
+}
+
+JNI_METHOD(jstring, nativeLastError)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) clazz;
+ const std::string error = native(handle)->lastError();
+ if (error.empty()) return nullptr;
+ return env->NewStringUTF(error.c_str());
+}
+
+JNI_METHOD(jfloatArray, nativeGetRefreshRates)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) clazz;
+ const std::vector rates = native(handle)->refreshRates();
+ jfloatArray out = env->NewFloatArray((jsize) rates.size());
+ if (out == nullptr) return nullptr;
+ if (!rates.empty())
+ {
+ env->SetFloatArrayRegion(out, 0, (jsize) rates.size(), rates.data());
+ }
+ return out;
+}
+
+JNI_METHOD(void, nativeSetVideoResolution)
+(JNIEnv* env, jclass clazz, jlong handle, jint width, jint height)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setVideoResolution(width, height);
+}
+
+JNI_METHOD(void, nativeSetQuadDistance)(JNIEnv* env, jclass clazz, jlong handle, jfloat meters)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setQuadDistance(meters);
+}
+
+JNI_METHOD(void, nativeSetQuadWidth)(JNIEnv* env, jclass clazz, jlong handle, jfloat meters)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setQuadWidth(meters);
+}
+
+JNI_METHOD(void, nativeSetQuadHeightOffset)(JNIEnv* env, jclass clazz, jlong handle, jfloat meters)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setQuadHeightOffset(meters);
+}
+
+JNI_METHOD(jfloat, nativeGetQuadHeightOffset)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ return native(handle)->quadHeightOffset();
+}
+
+JNI_METHOD(void, nativeSetPassthrough)(JNIEnv* env, jclass clazz, jlong handle, jboolean enabled)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setPassthrough(enabled == JNI_TRUE);
+}
+
+JNI_METHOD(void, nativeSetSharpening)(JNIEnv* env, jclass clazz, jlong handle, jboolean enabled)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setSharpening(enabled == JNI_TRUE);
+}
+
+JNI_METHOD(void, nativeSetHeadLocked)(JNIEnv* env, jclass clazz, jlong handle, jboolean enabled)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->setHeadLocked(enabled == JNI_TRUE);
+}
+
+JNI_METHOD(void, nativeRequestRefreshRate)(JNIEnv* env, jclass clazz, jlong handle, jfloat hz)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->requestRefreshRate(hz);
+}
+
+JNI_METHOD(void, nativeRequestRecenter)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->requestRecenter();
+}
+
+JNI_METHOD(void, nativeHaptic)
+(JNIEnv* env, jclass clazz, jlong handle, jfloat amplitude, jint durationMs)
+{
+ (void) env;
+ (void) clazz;
+ native(handle)->requestHaptic(amplitude, durationMs);
+}
+
+JNI_METHOD(jfloat, nativeGetQuadDistance)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ return native(handle)->quadDistance();
+}
+
+JNI_METHOD(jfloat, nativeGetQuadWidth)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ return native(handle)->quadWidth();
+}
+
+JNI_METHOD(jboolean, nativeIsHeadLocked)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ return native(handle)->headLocked() ? JNI_TRUE : JNI_FALSE;
+}
+
+JNI_METHOD(jboolean, nativeIsPassthroughEnabled)(JNIEnv* env, jclass clazz, jlong handle)
+{
+ (void) env;
+ (void) clazz;
+ return native(handle)->passthroughEnabled() ? JNI_TRUE : JNI_FALSE;
+}
diff --git a/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java
new file mode 100644
index 00000000..d78267ea
--- /dev/null
+++ b/app/xr/src/main/java/com/openipc/xr/XrGoggleSession.java
@@ -0,0 +1,320 @@
+package com.openipc.xr;
+
+import android.app.Activity;
+import android.content.Context;
+import android.content.pm.PackageManager;
+import android.os.Handler;
+import android.os.Looper;
+import android.util.Log;
+import android.view.Surface;
+
+import androidx.annotation.Keep;
+import androidx.annotation.Nullable;
+
+/**
+ * Immersive OpenXR presentation of the decoded video stream.
+ *
+ * The video is not rendered by this class. {@code xrCreateSwapchainAndroidSurfaceKHR}
+ * hands out a {@link Surface} that is the producer side of a compositor swapchain;
+ * MediaCodec decodes straight into it and the runtime composites it as a head locked
+ * quad layer. No GL blit, no readback, no view hierarchy.
+ *
+ *
All native calls happen on a dedicated thread owned by this class. Listener
+ * callbacks are posted to the main looper.
+ */
+@Keep
+public final class XrGoggleSession {
+ // Keep in sync with XrGoggleSession.h
+ public static final int BUTTON_RECENTER = 0;
+ public static final int BUTTON_PASSTHROUGH = 1;
+ public static final int BUTTON_RECORD = 2;
+ public static final int BUTTON_LOCK_MODE = 3;
+ public static final int BUTTON_RAISE = 4;
+ public static final int BUTTON_LOWER = 5;
+ public static final int BUTTON_EXIT = 6;
+
+ private static final String TAG = "pixelpilot-xr";
+
+ private static boolean libraryLoaded;
+
+ static {
+ try {
+ System.loadLibrary("XrGoggle");
+ libraryLoaded = true;
+ } catch (Throwable t) {
+ // A device without the OpenXR loader still has to be able to run the flat app.
+ Log.w(TAG, "libXrGoggle not available: " + t.getMessage());
+ libraryLoaded = false;
+ }
+ }
+
+ public interface Listener {
+ /** The compositor swapchain is ready; hand this Surface to the decoder. */
+ void onXrReady(Surface videoSurface);
+
+ /** One of the BUTTON_* constants was pressed on a controller. */
+ void onXrButton(int button);
+
+ /** Session ended. {@code error} is null for a normal stop. */
+ void onXrStopped(@Nullable String error);
+ }
+
+ private final Activity activity;
+ private final Listener listener;
+ private final Handler main = new Handler(Looper.getMainLooper());
+
+ private long handle;
+ private Thread thread;
+ private volatile boolean stopping;
+
+ public XrGoggleSession(Activity activity, Listener listener) {
+ this.activity = activity;
+ this.listener = listener;
+ }
+
+ /**
+ * Cheap pre-flight check used to decide whether to offer the mode at all. It says
+ * "this looks like a headset", not "a session will succeed" — only {@link #start()}
+ * can tell you that.
+ */
+ public static boolean isSupportedDevice(Context context) {
+ if (!libraryLoaded) {
+ return false;
+ }
+ PackageManager pm = context.getPackageManager();
+ if (pm.hasSystemFeature("android.hardware.vr.headtracking")
+ || pm.hasSystemFeature("android.software.xr.immersive")
+ || pm.hasSystemFeature("oculus.software.handtracking")) {
+ return true;
+ }
+ // Horizon OS shell, present on every Quest.
+ try {
+ pm.getPackageInfo("com.oculus.vrshell", 0);
+ return true;
+ } catch (PackageManager.NameNotFoundException ignored) {
+ return false;
+ }
+ }
+
+ public static boolean isLibraryLoaded() {
+ return libraryLoaded;
+ }
+
+ /** Seeds the swapchain geometry. Must be called before {@link #start()}. */
+ public void setSwapchainSize(int width, int height) {
+ ensureHandle();
+ if (handle != 0) {
+ nativeSetSwapchainSize(handle, width, height);
+ }
+ }
+
+ /**
+ * Creates the session on a background thread and starts the frame loop. The listener
+ * gets either {@link Listener#onXrReady} or {@link Listener#onXrStopped} with an
+ * error.
+ */
+ public void start() {
+ if (thread != null) {
+ return;
+ }
+ ensureHandle();
+ if (handle == 0) {
+ main.post(() -> listener.onXrStopped("OpenXR native library is not available"));
+ return;
+ }
+ thread = new Thread(this::threadBody, "xr-goggle");
+ // The frame loop drives the compositor; it must not lose to background work.
+ thread.setPriority(Thread.MAX_PRIORITY);
+ thread.start();
+ }
+
+ /** Asks the frame loop to exit. Returns immediately. */
+ public void stop() {
+ stopping = true;
+ if (handle != 0) {
+ nativeRequestStop(handle);
+ }
+ }
+
+ /** Stops and joins, then releases the native object. Safe to call twice. */
+ public void release() {
+ stop();
+ Thread t = thread;
+ if (t != null) {
+ try {
+ t.join(2000);
+ } catch (InterruptedException ignored) {
+ Thread.currentThread().interrupt();
+ }
+ thread = null;
+ }
+ if (handle != 0) {
+ nativeFree(handle);
+ handle = 0;
+ }
+ }
+
+ private void ensureHandle() {
+ if (libraryLoaded && handle == 0) {
+ handle = nativeAlloc();
+ // Somewhere writable for the runtime manifest the loader may need.
+ nativeSetManifestDir(handle, activity.getFilesDir().getAbsolutePath());
+ }
+ }
+
+ private void threadBody() {
+ final long h = handle;
+ if (!nativeCreate(h, activity)) {
+ final String error = errorOr("OpenXR session could not be created");
+ main.post(() -> listener.onXrStopped(error));
+ return;
+ }
+ final Surface surface = nativeGetVideoSurface(h);
+ if (surface == null) {
+ main.post(() -> listener.onXrStopped("OpenXR gave us no video surface"));
+ nativeDestroy(h);
+ return;
+ }
+ main.post(() -> listener.onXrReady(surface));
+
+ // Blocks until stop() or an unrecoverable runtime error. Button presses arrive
+ // through onXrButton() below, called from this thread.
+ nativeRunLoop(h, this);
+
+ final String error = stopping ? null : errorOr("OpenXR session ended unexpectedly");
+ nativeDestroy(h);
+ main.post(() -> listener.onXrStopped(error));
+ }
+
+ private String errorOr(String fallback) {
+ String error = handle != 0 ? nativeLastError(handle) : null;
+ return error != null ? error : fallback;
+ }
+
+ /** Called from the XR frame loop thread. */
+ @Keep
+ void onXrButton(int button) {
+ main.post(() -> listener.onXrButton(button));
+ }
+
+ // --- live settings ---------------------------------------------------------------
+
+ public void setVideoResolution(int width, int height) {
+ if (handle != 0) nativeSetVideoResolution(handle, width, height);
+ }
+
+ public void setQuadDistance(float meters) {
+ if (handle != 0) nativeSetQuadDistance(handle, meters);
+ }
+
+ public void setQuadWidth(float meters) {
+ if (handle != 0) nativeSetQuadWidth(handle, meters);
+ }
+
+ public void setQuadHeightOffset(float meters) {
+ if (handle != 0) nativeSetQuadHeightOffset(handle, meters);
+ }
+
+ public float quadHeightOffset() {
+ return handle != 0 ? nativeGetQuadHeightOffset(handle) : 0f;
+ }
+
+ public void setPassthrough(boolean enabled) {
+ if (handle != 0) nativeSetPassthrough(handle, enabled);
+ }
+
+ public void setSharpening(boolean enabled) {
+ if (handle != 0) nativeSetSharpening(handle, enabled);
+ }
+
+ public void setHeadLocked(boolean enabled) {
+ if (handle != 0) nativeSetHeadLocked(handle, enabled);
+ }
+
+ /** 0 leaves the runtime default alone. */
+ public void requestRefreshRate(float hz) {
+ if (handle != 0) nativeRequestRefreshRate(handle, hz);
+ }
+
+ public void recenter() {
+ if (handle != 0) nativeRequestRecenter(handle);
+ }
+
+ public void haptic(float amplitude, int durationMs) {
+ if (handle != 0) nativeHaptic(handle, amplitude, durationMs);
+ }
+
+ public float[] refreshRates() {
+ return handle != 0 ? nativeGetRefreshRates(handle) : new float[0];
+ }
+
+ public float quadDistance() {
+ return handle != 0 ? nativeGetQuadDistance(handle) : 0f;
+ }
+
+ public float quadWidth() {
+ return handle != 0 ? nativeGetQuadWidth(handle) : 0f;
+ }
+
+ public boolean isHeadLocked() {
+ return handle != 0 && nativeIsHeadLocked(handle);
+ }
+
+ public boolean isPassthroughEnabled() {
+ return handle != 0 && nativeIsPassthroughEnabled(handle);
+ }
+
+ // --- native ---------------------------------------------------------------------
+
+ private static native long nativeAlloc();
+
+ private static native void nativeFree(long handle);
+
+ private static native void nativeSetSwapchainSize(long handle, int width, int height);
+
+ private static native void nativeSetManifestDir(long handle, String dir);
+
+ private static native boolean nativeCreate(long handle, Activity activity);
+
+ private static native Surface nativeGetVideoSurface(long handle);
+
+ private static native void nativeRunLoop(long handle, XrGoggleSession listener);
+
+ private static native void nativeRequestStop(long handle);
+
+ private static native void nativeDestroy(long handle);
+
+ private static native String nativeLastError(long handle);
+
+ private static native float[] nativeGetRefreshRates(long handle);
+
+ private static native void nativeSetVideoResolution(long handle, int width, int height);
+
+ private static native void nativeSetQuadDistance(long handle, float meters);
+
+ private static native void nativeSetQuadWidth(long handle, float meters);
+
+ private static native void nativeSetQuadHeightOffset(long handle, float meters);
+
+ private static native float nativeGetQuadHeightOffset(long handle);
+
+ private static native void nativeSetPassthrough(long handle, boolean enabled);
+
+ private static native void nativeSetSharpening(long handle, boolean enabled);
+
+ private static native void nativeSetHeadLocked(long handle, boolean enabled);
+
+ private static native void nativeRequestRefreshRate(long handle, float hz);
+
+ private static native void nativeRequestRecenter(long handle);
+
+ private static native void nativeHaptic(long handle, float amplitude, int durationMs);
+
+ private static native float nativeGetQuadDistance(long handle);
+
+ private static native float nativeGetQuadWidth(long handle);
+
+ private static native boolean nativeIsHeadLocked(long handle);
+
+ private static native boolean nativeIsPassthroughEnabled(long handle);
+}
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index b1540ecf..d2075b43 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -2,18 +2,22 @@
agp = "8.5.2"
coreKtx = "1.13.1"
appcompat = "1.7.0"
+annotation = "1.8.2"
material = "1.12.0"
constraintlayout = "2.1.4"
mpandroidchartVersion = "3.1.0"
mediapipe = "0.10.29"
+openxr = "1.1.49"
[libraries]
core-ktx = { module = "androidx.core:core-ktx", version.ref = "coreKtx" }
appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" }
+annotation = { module = "androidx.annotation:annotation", version.ref = "annotation" }
material = { group = "com.google.android.material", name = "material", version.ref = "material" }
constraintlayout = { group = "androidx.constraintlayout", name = "constraintlayout", version.ref = "constraintlayout" }
philjay-mpandroidchart = { module = "com.github.Philjay:mpandroidchart", version.ref = "mpandroidchartVersion" }
mediapipe-tasks-vision = { module = "com.google.mediapipe:tasks-vision", version.ref = "mediapipe" }
+openxr-loader = { module = "org.khronos.openxr:openxr_loader_for_android", version.ref = "openxr" }
[plugins]
androidApplication = { id = "com.android.application", version.ref = "agp" }
diff --git a/settings.gradle b/settings.gradle
index b383549c..12d4a68b 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -27,4 +27,5 @@ rootProject.name = "PixelPilot"
include(":app")
include(":app:videonative")
include(":app:wfbngrtl8812")
+include(":app:xr")
include(":app:mavlink")