Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ aa-supervisor: $(SUPERVISOR_OBJ)
# iDrive PluginApp (installed in /opt/hmi/data/ro/Plugins/EntryAad, loaded by HmiMain3D at boot). Its objects are
# position independent and hidden: HmiMain3D only looks up pluginAppInfo, the one symbol the library exports.
HMI_SRC = hmiapp/entry_aa_app.cpp hmiapp/bmw_framework.cpp hmiapp/app_log.cpp hmiapp/app_guards.cpp \
hmiapp/service_link.cpp hmiapp/car_signals.cpp hmiapp/hold_machine.cpp hmiapp/controller_coding.cpp
hmiapp/service_link.cpp hmiapp/car_signals.cpp hmiapp/hold_machine.cpp hmiapp/idrive_coding.cpp
HMI_OBJ = $(HMI_SRC:.cpp=.o)
HMI_LIBDIRS = -L$(SYSROOT)/opt/hmi/lib -Wl,-rpath-link,$(SYSROOT)/opt/hmi/lib
hmiapp: hmiapp/libEntryAad.so
Expand Down
5 changes: 3 additions & 2 deletions core/aa_channels.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,8 @@ void AASession::onTlsHandshake(const Message& message) {
// The description of this car: Google's Desktop Head Unit template, adapted by core/sdr_builder.cpp.
void AASession::onDiscovery() {
printf("[AASession] >>> Received ServiceDiscoveryRequest from phone! Sending ServiceDiscoveryResponse...\n");
const Message response = message::discoveryResponse(
buildCarServiceDiscovery(kDhuSdpProtobuf, kDhuSdpProtobufLen, vehicleFile_, softwareVersion_, display_));
const Message response = message::discoveryResponse(buildCarServiceDiscovery(
kDhuSdpProtobuf, kDhuSdpProtobufLen, vehicleFile_, softwareVersion_, display_, carMake_));
if (sendSealed(channel::kControl, response)) {
printf("[AASession] >>> Sent ServiceDiscoveryResponse (%u bytes)\n", static_cast<unsigned>(response.size()));
}
Expand Down Expand Up @@ -253,6 +253,7 @@ void AASession::onSetup(uint8_t channel) {
if (channel == channel::kVideo) {
sendVideoFocus(true, false);
videoReady_ = true;
keyframe_.videoStarted(locationClockMs());
}
}

Expand Down
35 changes: 35 additions & 0 deletions core/aa_session.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ const unsigned kFramesLoggedEvery = 300;
const unsigned kKeysLogged = 200; // evo_knob logs every key anyway
const unsigned kMovesLogged = 40;
const unsigned kMicrophoneLogEvery = 250;
// Encrypted frames in a row that do not decrypt: the TLS stream is broken (a record lost or cut on the way), and no
// later frame will decrypt either. The session ends rather than wait for the silence limit with a frozen picture.
const unsigned kUnreadableLimit = 16;
const int kTouchMove = 2;

uint64_t monotonicUs() {
Expand All @@ -61,9 +64,12 @@ AASession::AASession()
thread_(0),
vehicleFile_(nullptr),
softwareVersion_(nullptr),
carMake_(nullptr),
longestSilenceMs_(0),
lastReceivedMs_(0),
videoReady_(false),
projected_(false),
unreadableInARow_(0),
mediaMessages_(0),
microphoneChunks_(0) {
pthread_mutex_init(&lifecycleLock_, nullptr);
Expand Down Expand Up @@ -125,6 +131,10 @@ void AASession::postVideoFocus(bool projected) {
inbox_.postFocus(projected);
}

void AASession::postKeyframeRequest() {
inbox_.postKeyframeRequest();
}

void AASession::postNightMode(bool night) {
inbox_.postNight(night);
}
Expand Down Expand Up @@ -211,6 +221,9 @@ bool AASession::start(int client_fd) {
assembler_.clear();
inbox_.reset();
videoReady_ = false;
projected_ = false;
keyframe_.reset();
unreadableInARow_ = 0;
microphoneChunks_ = 0;
mediaMessages_ = 0;
memset(mediaSessions_, 0, sizeof(mediaSessions_));
Expand Down Expand Up @@ -441,7 +454,14 @@ AASession::Received AASession::receive(uint8_t& channel, Message& message) {
} else if (!tls_.open(&payload[0], payload.size(), plain)) {
printf("[AASession] Failed to decrypt encrypted frame (%u bytes).\n", header.length);
assembler_.drop(channel);
if (++unreadableInARow_ >= kUnreadableLimit) {
printf("[AASession] %u encrypted frames in a row did not decrypt: the stream is broken, session ends.\n",
unreadableInARow_);
return kLost;
}
return kNothing;
} else {
unreadableInARow_ = 0;
}
switch (assembler_.add(header, plain, message)) {
case MessageAssembler::kComplete:
Expand Down Expand Up @@ -497,6 +517,9 @@ void AASession::sendCarInput() {
printf("[AASession] video focus -> phone: %s (unrequested) %s\n",
projected ? "FOCUSED/projected" : "UNFOCUSED/native", sent ? "sent" : "SEND FAILED");
}
if (batch.keyframe) {
askKeyframe();
}
for (size_t i = 0; i < batch.keys.size(); ++i) {
const CarKey& key = batch.keys[i];
const Message event = key.rotary ? message::rotaryEvent(monotonicUs(), key.keycode, key.delta)
Expand Down Expand Up @@ -566,10 +589,22 @@ bool AASession::sendSensorEvent(uint32_t type) {
bool AASession::sendVideoFocus(bool projected, bool unrequested) {
const bool sent = sendSealed(channel::kVideo, message::videoFocusIndication(projected, unrequested));
if (sent) {
projected_ = projected;
printf("[AASession] >>> Sent VideoFocusIndication (%s%s) on Channel %u!\n", projected ? "FOCUSED" : "UNFOCUSED",
unrequested ? ", unrequested" : "", channel::kVideo);
}
return sent;
}

// A lost frame leaves the picture broken until the phone's next keyframe: the focus is taken away and given back at
// once, which restarts the phone's encoder with one (keyframe_request.hpp). Only while the picture is projected.
void AASession::askKeyframe() {
if (!videoReady_ || !projected_ || !keyframe_.allow(locationClockMs())) {
return;
}
const bool sent = sendVideoFocus(false, true) && sendVideoFocus(true, true);
printf("[AASession] video frame lost: keyframe asked of the phone (focus off and on) %s\n",
sent ? "sent" : "SEND FAILED");
}

} // namespace aa
12 changes: 11 additions & 1 deletion core/aa_session.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "car_inbox.hpp"
#include "car_location.hpp"
#include "display_plan.hpp"
#include "keyframe_request.hpp"

namespace aa {

Expand Down Expand Up @@ -65,13 +66,17 @@ class AASession {
void setSoftwareVersion(const char* version) { softwareVersion_ = version; }
// The panel and the canvas asked of the phone for it (display_plan.hpp). The reference car's by default.
void setDisplay(const DisplayPlan& display) { display_ = display; }
// The maker announced to the phone, as iDrive is coded ("MINI"); kept by the caller. Null: "BMW".
void setCarMake(const char* make) { carMake_ = make; }

// Car input for the phone, from any thread: queued, then sent by the session thread.
void postTouchEvent(const TouchEventOut& ev);
void postKeyEvent(uint32_t keycode, bool down); // Android key codes, as declared on the input channel
void postRelativeEvent(uint32_t keycode, int delta); // rotary steps (keycode 65536, ROTARY_CONTROLLER)
// The car takes (false) or gives back (true) the screen: an unrequested video focus indication.
void postVideoFocus(bool projected);
// The decoder lost a video frame. Any thread; the session asks the phone for a keyframe when it may.
void postKeyframeRequest();
// The car's day or night, kept across sessions: a NIGHT_MODE event once the phone started that sensor.
void postNightMode(bool night);
// The car's latest fix (monotonic capture time); an invalid one when the car's feed expired.
Expand Down Expand Up @@ -114,6 +119,7 @@ class AASession {
bool sendLocation();
bool sendSensorEvent(uint32_t type);
bool sendVideoFocus(bool projected, bool unrequested);
void askKeyframe();

// One handler per route (aa_channels.cpp); dispatch() is false when the session must end.
bool dispatch(uint8_t channel, const Message& message);
Expand Down Expand Up @@ -150,6 +156,7 @@ class AASession {
Callback<NavStatusCallback> navigation_;
const char* vehicleFile_;
const char* softwareVersion_;
const char* carMake_;
DisplayPlan display_;

CarInbox inbox_;
Expand All @@ -158,7 +165,10 @@ class AASession {
uint64_t lastReceivedMs_; // when the phone last sent anything, 0 before its first byte
TlsClient tls_;
MessageAssembler assembler_;
bool videoReady_; // the phone set the video channel up: a focus can go
bool videoReady_; // the phone set the video channel up: a focus can go
bool projected_; // the last video focus sent was FOCUSED
KeyframeRequest keyframe_;
unsigned unreadableInARow_; // encrypted frames that did not decrypt since the last one that did
uint32_t mediaSessions_[32]; // per channel, the session id of its start indication, echoed in our acks
uint32_t mediaMessages_; // media messages received, every channel
uint32_t audioMessages_[2]; // guidance (8), media (9)
Expand Down
16 changes: 16 additions & 0 deletions core/car_inbox.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ CarInbox::CarInbox()
keyDrops_(0),
rotaryDrops_(0),
focus_(kFocusNone),
keyframe_(false),
night_(-1),
nightStarted_(false),
nightChanged_(false),
Expand Down Expand Up @@ -134,6 +135,18 @@ void CarInbox::postFocus(bool projected) {
wake();
}

void CarInbox::postKeyframeRequest() {
bool first;
{
MutexLock lock(mutex_);
first = !keyframe_;
keyframe_ = true;
}
if (first) { // a stalled decoder loses frame after frame: one wake is enough
wake();
}
}

void CarInbox::postNight(bool night) {
bool due;
{
Expand Down Expand Up @@ -188,6 +201,8 @@ void CarInbox::take(CarBatch& batch) {
batch.keys.swap(keys_);
batch.focus = focus_;
focus_ = kFocusNone;
batch.keyframe = keyframe_;
keyframe_ = false;
batch.nightChanged = nightChanged_ && nightStarted_;
nightChanged_ = false;
batch.microphone.swap(microphone_);
Expand Down Expand Up @@ -248,6 +263,7 @@ void CarInbox::reset() {
keys_.clear();
keyDrops_ = rotaryDrops_ = 0;
focus_ = kFocusNone;
keyframe_ = false;
nightStarted_ = nightChanged_ = false;
location_.reset();
microphoneOpen_ = false;
Expand Down
3 changes: 3 additions & 0 deletions core/car_inbox.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ struct CarBatch {
std::vector<TouchEventOut> touches;
std::vector<CarKey> keys;
FocusRequest focus;
bool keyframe; // the decoder lost a video frame since the last turn
bool nightChanged; // the phone started night mode and the car changed since the last event
std::vector<std::vector<uint8_t>> microphone;
};
Expand All @@ -50,6 +51,7 @@ class CarInbox {
void postKey(uint32_t keycode, bool down);
void postRotary(uint32_t keycode, int delta);
void postFocus(bool projected);
void postKeyframeRequest(); // a video frame was lost: the session decides whether to ask the phone
void postNight(bool night); // kept across sessions
void postLocation(const CarLocation& location);
void postMicrophone(const uint8_t* pcm, size_t size);
Expand Down Expand Up @@ -83,6 +85,7 @@ class CarInbox {
std::vector<CarKey> keys_;
unsigned keyDrops_, rotaryDrops_;
FocusRequest focus_;
bool keyframe_;
int night_;
bool nightStarted_, nightChanged_;
LocationFeed location_;
Expand Down
9 changes: 7 additions & 2 deletions core/display_plan.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
// asked for; the phone is told to keep its UI in the top-left part the panel shows (insets, core/sdr_builder.cpp),
// and the video surface shows that part only (platform/evo_video.cpp). The reference car is one case of the rule: a
// 1280x480 panel in the 1280x720 canvas, its top 480 rows shown.
// A panel partly hidden by its surround (the MINI's round one, platform/car_brand.hpp) keeps the whole picture; the
// phone is only told to keep its UI away from the hidden sides.
#ifndef ENTRY_AAD_CORE_DISPLAY_PLAN_HPP
#define ENTRY_AAD_CORE_DISPLAY_PLAN_HPP

Expand All @@ -17,10 +19,12 @@ struct DisplayPlan {
uint32_t width, height; // the panel, and the part of the canvas that is shown
uint32_t canvasWidth, canvasHeight; // what the phone streams
uint32_t resolution; // the canvas as the protocol numbers it: 1, 2 or 3
uint32_t hiddenSide; // panel columns hidden on each side by the surround
// The reference car.
DisplayPlan() : width(1280), height(480), canvasWidth(1280), canvasHeight(720), resolution(2) {}
DisplayPlan() : width(1280), height(480), canvasWidth(1280), canvasHeight(720), resolution(2), hiddenSide(0) {}
uint32_t bottomInset() const { return canvasHeight - height; }
uint32_t rightInset() const { return canvasWidth - width; }
uint32_t leftInset() const { return hiddenSide; }
uint32_t rightInset() const { return canvasWidth - width + hiddenSide; }
bool reference() const { return width == 1280 && height == 480; }
};

Expand All @@ -37,6 +41,7 @@ inline bool planDisplay(uint32_t panelWidth, uint32_t panelHeight, DisplayPlan&
out.canvasWidth = kCanvas[i][0];
out.canvasHeight = kCanvas[i][1];
out.resolution = i + 1;
out.hiddenSide = 0;
return true;
}
}
Expand Down
50 changes: 50 additions & 0 deletions core/keyframe_request.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright (c) 2026 Jiraiya43
// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
// When the head unit may ask the phone for a new keyframe after the decoder lost a video frame.
#ifndef ENTRY_AAD_CORE_KEYFRAME_REQUEST_HPP
#define ENTRY_AAD_CORE_KEYFRAME_REQUEST_HPP

#include <stdint.h>

namespace aa {

// H.264 frames between two keyframes refer to the ones before them: once one is lost, the picture stays broken
// until the phone's next keyframe, which can be seconds away. Android Auto has no message to ask for one, but a
// phone that gets the video focus back starts its encoder again with a keyframe; the session takes the focus away
// and gives it back at once (aa_session.cpp).
// Not during the first seconds of the video, when the phone's own keyframes are still coming, and seldom: a decoder
// short of buffers would otherwise turn every frame into a keyframe, each one larger than the frames it replaces.
// Session thread only; the clock is the caller's, in milliseconds.
class KeyframeRequest {
public:
static const uint64_t kSettleMs = 5000; // after the video was set up
static const uint64_t kSpacingMs = 10000; // between two requests

KeyframeRequest() : videoSinceMs_(0), lastMs_(0), video_(false), asked_(false) {}

// The phone set the video channel up: a new stream, nothing asked yet.
void videoStarted(uint64_t nowMs) {
video_ = true;
videoSinceMs_ = nowMs;
asked_ = false;
}
// A new session.
void reset() { video_ = asked_ = false; }
// A frame was lost: true when a request may go now, and it then counts as made.
bool allow(uint64_t nowMs) {
if (!video_ || nowMs - videoSinceMs_ < kSettleMs || (asked_ && nowMs - lastMs_ < kSpacingMs)) {
return false;
}
asked_ = true;
lastMs_ = nowMs;
return true;
}

private:
uint64_t videoSinceMs_, lastMs_;
bool video_, asked_;
};

} // namespace aa

#endif // ENTRY_AAD_CORE_KEYFRAME_REQUEST_HPP
10 changes: 7 additions & 3 deletions core/sdr_builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,9 @@ bool tuneVideoConfig(const pb::Field& config, Bytes& out, const DisplayPlan& dis
return false;
}
Bytes insets;
pb::appendUint(insets, 1, 0);
pb::appendUint(insets, 1, 0); // top
pb::appendUint(insets, 2, display.bottomInset());
pb::appendUint(insets, 3, 0);
pb::appendUint(insets, 3, display.leftInset());
pb::appendUint(insets, 4, display.rightInset());
Bytes rebuilt;
pb::appendMessage(rebuilt, kInsetsField, insets);
Expand Down Expand Up @@ -318,11 +318,15 @@ void setDiscoveryTrim(DiscoveryTrim trim) {
}

std::vector<uint8_t> buildCarServiceDiscovery(const uint8_t* base, size_t baseLen, const char* vehicleFile,
const char* softwareVersion, const DisplayPlan& display) {
const char* softwareVersion, const DisplayPlan& display,
const char* make) {
SdrIdentity identity;
if (vehicleFile) {
identity.loadVehicleFile(vehicleFile);
}
if (make && *make) {
identity.carMake = identity.huMake = make;
}
if (softwareVersion && *softwareVersion) {
identity.swVersion = softwareVersion;
}
Expand Down
6 changes: 4 additions & 2 deletions core/sdr_builder.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,11 @@ void setDiscoveryTrim(DiscoveryTrim trim);
// What the phone is told about this car: the template adapted by every step above. vehicleFile: the model
// and year (SdrIdentity::loadVehicleFile), given by the platform; null keeps the default ones. softwareVersion: the
// version of the program, announced as the head unit's software version; null keeps "0.0.0". display: the panel
// and the canvas chosen for it; the reference car's when not given.
// and the canvas chosen for it; the reference car's when not given. make: the car's and the head unit's maker as
// iDrive is coded ("MINI"); null keeps "BMW".
std::vector<uint8_t> buildCarServiceDiscovery(const uint8_t* base, size_t baseLen, const char* vehicleFile,
const char* softwareVersion, const DisplayPlan& display = DisplayPlan());
const char* softwareVersion, const DisplayPlan& display = DisplayPlan(),
const char* make = nullptr);

} // namespace aa

Expand Down
4 changes: 3 additions & 1 deletion docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ Le protocole Android Auto côté voiture, sans rien de propre à BMW.
| `car_location.hpp` | Position de la voiture vers le capteur Android Auto |
| `crash_handler` | Trace d'un plantage, puis le nettoyage enregistré par la plateforme |
| `display_plan.hpp` | Image demandée au téléphone selon la taille de l'écran, et zone montrée |
| `keyframe_request.hpp` | Image vidéo perdue : quand redemander une image complète au téléphone (pas dans les 5 premières secondes, au plus une fois par 10 s) |
| `include/` | Modèle de description de la voiture, certificat de l'unité |

## `platform/` : l'EntryNav2
Expand All @@ -62,6 +63,7 @@ Le protocole Android Auto côté voiture, sans rien de propre à BMW.
| `vmost_client` | Accès au bus vMOST de la voiture (molette, touches, Wi-Fi), chargé à l'exécution |
| `evo_ccs.hpp`, `evo_touch_frame.hpp` | Messages de la molette BMW (deux générations de contrôleur) et trames du tactile |
| `controller_layout.hpp` | Quelle touche est chaque bouton, selon le codage de la voiture, comme l'iDrive (contrôleur type G ou F) |
| `car_brand.hpp` | Marque codée dans l'iDrive (BMW ou MINI) : marque annoncée au téléphone, bords de l'écran cachés par le rond des MINI |
| `evo_gps` | Lecture passive du GPS de la voiture |
| `evo_mic` | Micro de la voiture pour l'Assistant |
| `evo_wlan`, `evo_ap` | Wi-Fi de la voiture, allumé comme le fait CarPlay |
Expand All @@ -88,7 +90,7 @@ Le protocole Android Auto côté voiture, sans rien de propre à BMW.
| `pending_restart.hpp` | Redémarrage d'Android Auto quand la case du câble USB change en marche (testé sur PC) |
| `service_link` | Lancement du superviseur, commandes au service, arrêt |
| `car_signals` | Marche arrière, appels, jour et nuit, mode de conduite, lus dans l'iDrive |
| `controller_coding` | Codage des touches de la voiture, lu dans l'iDrive au démarrage de l'appli, écrit pour le service |
| `idrive_coding` | Codage de la voiture lu dans l'iDrive au démarrage de l'appli et écrit pour le service : touches du contrôleur, marque (BMW ou MINI) |
| `app_guards` | Interrupteur, garde-fou contre les plantages, démarrage automatique |
| `app_log`, `log_profile.hpp` | Journal de l'appli, profil maintenance |
| `popup_peek.hpp`, `cds_parse.hpp` | Fenêtres de l'iDrive par-dessus Android Auto, lecture des données de l'iDrive |
Expand Down
Loading