diff --git a/config.h b/config.h index a235042b..588d9a83 100644 --- a/config.h +++ b/config.h @@ -13,7 +13,7 @@ // Minimal build for bring-up //#define BRINGUP // Disable display configuration / bringup on desktop devices -#define NO_DISPLAY +//#define NO_DISPLAY // Print RTKit logs to the console //#define RTKIT_SYSLOG diff --git a/src/afk.c b/src/afk.c index a8645a0a..9277217c 100644 --- a/src/afk.c +++ b/src/afk.c @@ -8,21 +8,13 @@ #include "dcp/parser.h" -struct afk_rb_hdr { - u32 bufsz; - u32 unk; - u32 _pad1[14]; - u32 rptr; - u32 _pad2[15]; - u32 wptr; - u32 _pad3[15]; -}; - struct afk_rb { bool ready; - struct afk_rb_hdr *hdr; + u32 *rptr; + u32 *wptr; void *buf; size_t bufsz; + size_t block_size; }; enum EPICType { @@ -104,6 +96,11 @@ struct afk_epic_ep { struct rtkit_buffer rxbuf; bool started; + bool compact; + u8 compact_seq, compact_next_seq; + u16 compact_interface; + u32 compact_length, compact_received, compact_token; + void *compact_message; u16 seq; u32 num_channels; @@ -124,6 +121,7 @@ enum RBEP_MSG { RBEP_START = 0xa3, RBEP_START_ACK = 0x86, RBEP_SEND = 0xa2, + RBEP_SPACE_AVAILABLE = 0xa7, RBEP_RECV = 0x85, RBEP_SHUTDOWN = 0xc0, RBEP_SHUTDOWN_ACK = 0xc1, @@ -148,15 +146,32 @@ enum RBEP_MSG { bool afk_rb_init(afk_epic_ep_t *epic, struct afk_rb *rb, u64 base, u64 size) { - rb->hdr = epic->buf.bfr + base; + if (base > epic->buf.sz || size > epic->buf.sz - base || size < 3 * (1 << BLOCK_SHIFT)) { + printf("AFK: ring buffer lies outside its allocation\n"); + return false; + } - if (rb->hdr->bufsz + sizeof(*rb->hdr) != size) { - printf("AFK: ring buffer size mismatch\n"); + dma_rmb(); + void *hdr = epic->buf.bfr + base; + u32 bufsz = *(u32 *)hdr; + if (!bufsz || bufsz >= size || (size - bufsz) % 3) { + printf("AFK: invalid ring buffer size\n"); return false; } - rb->buf = rb->hdr + 1; - rb->bufsz = rb->hdr->bufsz; + /* As in the Python AFK client, size/rptr/wptr occupy three equal blocks. + * Firmware chooses their stride; it is not necessarily one 64-byte block. */ + size_t stride = (size - bufsz) / 3; + if (stride % (1 << BLOCK_SHIFT) || (stride & (stride - 1))) { + printf("AFK: invalid ring buffer header stride\n"); + return false; + } + + rb->rptr = hdr + stride; + rb->wptr = hdr + 2 * stride; + rb->buf = hdr + 3 * stride; + rb->bufsz = bufsz; + rb->block_size = stride; rb->ready = true; return true; @@ -243,7 +258,7 @@ static int afk_epic_poll(afk_epic_t *afk, int endpoint, bool block) case RBEP_RECV: { dma_rmb(); struct afk_rb *rb = &epic->rx; - if (rb->hdr->rptr != rb->hdr->wptr) { + if (rb->ready && *rb->rptr != *rb->wptr) { if (endpoint == epic->ep) return EPIC_DATA_READY; else if (epic->recv_handler) @@ -251,6 +266,9 @@ static int afk_epic_poll(afk_epic_t *afk, int endpoint, bool block) } break; } + case RBEP_SPACE_AVAILABLE: + break; + case RBEP_START_ACK: epic->started = true; break; @@ -271,22 +289,23 @@ static int afk_epic_poll(afk_epic_t *afk, int endpoint, bool block) static int afk_epic_rx(afk_epic_ep_t *epic, struct afk_qe **qe) { struct afk_rb *rb = &epic->rx; - u32 rptr = rb->hdr->rptr; + u32 rptr = *rb->rptr; struct afk_qe *hdr = rb->buf + rptr; if (hdr->magic != QE_MAGIC) { - printf("EPIC: bad queue entry magic!\n"); + printf("EPIC: bad queue entry magic 0x%x (rptr=%x wptr=%x bufsz=%lx) at %p\n", hdr->magic, + rptr, *rb->wptr, rb->bufsz, hdr); return -1; } - if (rptr + hdr->size > rb->bufsz) { + if (rptr + sizeof(*hdr) + hdr->size > rb->bufsz) { rptr = 0; hdr = rb->buf + rptr; if (hdr->magic != QE_MAGIC) { printf("EPIC: bad queue entry magic!\n"); return -1; } - rb->hdr->rptr = rptr; + *rb->rptr = rptr; } *qe = hdr; @@ -298,10 +317,11 @@ static int afk_epic_tx(afk_epic_ep_t *epic, u32 channel, u32 type, void *data, s { struct afk_rb *rb = &epic->tx; - u32 rptr = rb->hdr->rptr; - u32 wptr = rb->hdr->wptr; + u32 rptr = *rb->rptr; + u32 wptr = *rb->wptr; + u32 old_wptr = wptr; struct afk_qe *hdr = rb->buf + wptr; - size_t buf_advance = ALIGN_UP(sizeof(struct afk_qe) + size, 1 << BLOCK_SHIFT); + size_t buf_advance = ALIGN_UP(sizeof(struct afk_qe) + size, rb->block_size); if (wptr < rptr && buf_advance >= rptr - wptr) goto buffer_full; @@ -327,14 +347,14 @@ static int afk_epic_tx(afk_epic_ep_t *epic, u32 channel, u32 type, void *data, s } wptr += size; - wptr = ALIGN_UP(wptr, 1 << BLOCK_SHIFT); + wptr = ALIGN_UP(wptr, rb->block_size); if (wptr >= rb->bufsz) wptr = 0; memcpy(hdr + 1, data, size); dma_mb(); - rb->hdr->wptr = wptr; + *rb->wptr = wptr; dma_wmb(); struct rtkit_message msg = { @@ -342,6 +362,9 @@ static int afk_epic_tx(afk_epic_ep_t *epic, u32 channel, u32 type, void *data, s FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr), }; + if (epic->compact) + msg.msg = FIELD_PREP(RBEP_TYPE, RBEP_RECV) | ((u64)(rptr & 0xffff) << 32) | + ((u64)(wptr & 0xffff) << 16) | (old_wptr & 0xffff); if (!rtkit_send(epic->afk->rtk, &msg)) { printf("EPIC: failed to send TX WPTR message\n"); return -1; @@ -357,7 +380,7 @@ static int afk_epic_tx(afk_epic_ep_t *epic, u32 channel, u32 type, void *data, s static void afk_epic_rx_ack(afk_epic_ep_t *epic) { struct afk_rb *rb = &epic->rx; - u32 rptr = rb->hdr->rptr; + u32 rptr = *rb->rptr; struct afk_qe *hdr = rb->buf + rptr; if (hdr->magic != QE_MAGIC) { @@ -366,11 +389,193 @@ static void afk_epic_rx_ack(afk_epic_ep_t *epic) dma_mb(); - rptr = ALIGN_UP(rptr + sizeof(*hdr) + hdr->size, 1 << BLOCK_SHIFT); + rptr = ALIGN_UP(rptr + sizeof(*hdr) + hdr->size, rb->block_size); assert(rptr <= rb->bufsz); if (rptr == rb->bufsz) rptr = 0; - rb->hdr->rptr = rptr; + *rb->rptr = rptr; +} + +/* Compact EPIC wire format and fragment limit follow Gravity Linux's clean + * drm/apple AFK implementation (Cody Ho, based on Sven Peter's AFK driver). + * The bootloader uses synchronous commands and bounds each logical message. */ +#define COMPACT_FRAGMENT_SIZE 1028 +#define COMPACT_MESSAGE_MAX (64 * 1024) +#define COMPACT_ANNOUNCE 0x11 +#define COMPACT_START 0x12 + +struct compact_hdr { + u8 seq, flags; + u16 interface; + u32 length; +} PACKED; + +struct compact_body { + u64 timestamp; + u8 type, category, unk, flags; + u32 reserved; + u8 payload[]; +} PACKED; + +struct compact_desc { + u32 token; + u32 length; /* Request capacity; replies may leave this zero. */ + u8 payload[]; +} PACKED; + +static bool afk_is_compact(struct afk_qe *msg) +{ + struct compact_hdr *hdr = (void *)msg->data; + struct compact_body *body = (void *)(hdr + 1); + return !msg->channel && !msg->type && msg->size >= sizeof(*hdr) + sizeof(*body) && + hdr->length >= sizeof(*body) && hdr->length <= COMPACT_MESSAGE_MAX && + msg->size - sizeof(*hdr) <= hdr->length && body->category == 0 && + body->type == COMPACT_ANNOUNCE; +} + +/* Copy before publishing the consumer pointer, including on fragmented RX. */ +static int afk_compact_rx(afk_epic_ep_t *epic, struct afk_qe *msg) +{ + if (msg->channel || msg->type || msg->size <= sizeof(struct compact_hdr)) + return -1; + struct compact_hdr *hdr = (void *)msg->data; + u32 size = msg->size - sizeof(*hdr); + if (hdr->length < sizeof(struct compact_body) || hdr->length > COMPACT_MESSAGE_MAX || + size > hdr->length) + return -1; + + if (!epic->compact_received) { + free(epic->compact_message); + epic->compact_message = malloc(hdr->length); + if (!epic->compact_message) + return -1; + epic->compact_length = hdr->length; + epic->compact_interface = hdr->interface; + } else if (hdr->length != epic->compact_length || hdr->interface != epic->compact_interface || + hdr->seq != epic->compact_next_seq) { + return -1; + } + if (size > epic->compact_length - epic->compact_received) + return -1; + memcpy(epic->compact_message + epic->compact_received, hdr + 1, size); + epic->compact_received += size; + epic->compact_next_seq = hdr->seq + 1; + afk_epic_rx_ack(epic); + if (epic->compact_received != epic->compact_length) + return 0; + epic->compact_received = 0; + return 1; +} + +static int afk_compact_send(afk_epic_ep_t *epic, u16 interface, u8 type, u8 category, void *payload, + size_t payload_size) +{ + size_t total = sizeof(struct compact_body) + payload_size; + if (total > COMPACT_MESSAGE_MAX) + return -1; + struct compact_body *body = calloc(1, total); + if (!body) + return -1; + body->type = type; + body->category = category; + body->unk = type == COMPACT_START; + if (payload_size) + memcpy(body->payload, payload, payload_size); + + int ret = 0; + u64 deadline = timeout_calculate(5 * USEC_PER_SEC); + for (size_t offset = 0; offset < total;) { + size_t chunk = min(total - offset, COMPACT_FRAGMENT_SIZE); + u8 fragment[sizeof(struct compact_hdr) + COMPACT_FRAGMENT_SIZE]; + struct compact_hdr *hdr = (void *)fragment; + *hdr = (struct compact_hdr){ + .seq = epic->compact_seq++, .interface = interface, .length = total}; + memcpy(hdr + 1, (void *)body + offset, chunk); + struct afk_rb *rb = &epic->tx; + size_t advance = ALIGN_UP(sizeof(struct afk_qe) + sizeof(*hdr) + chunk, rb->block_size); + while (true) { + dma_rmb(); + u32 rptr = *rb->rptr, wptr = *rb->wptr; + bool fits = + wptr < rptr + ? advance < rptr - wptr + : ((advance < rb->bufsz - wptr || (advance == rb->bufsz - wptr && rptr)) || + advance < rptr); + if (fits) + break; + if (timeout_expired(deadline)) { + ret = -1; + goto out; + } + /* Service mailbox progress without consuming an eventual reply. */ + if (afk_epic_work(epic->afk, epic->ep) < 0) { + ret = -1; + goto out; + } + udelay(10); + } + ret = afk_epic_tx(epic, 0, 0, fragment, sizeof(*hdr) + chunk); + if (ret < 0) + goto out; + offset += chunk; + } + ret = 0; +out: + free(body); + return ret; +} + +static int afk_compact_command(afk_epic_ep_t *epic, u16 interface, u8 type, void *txbuf, + size_t txsize, void *rxbuf, size_t *rxsize) +{ + size_t wire_size = max(txsize, rxsize ? *rxsize : 0); + if (wire_size > COMPACT_MESSAGE_MAX - sizeof(struct compact_body) - sizeof(struct compact_desc)) + return -1; + struct compact_desc *desc = calloc(1, sizeof(*desc) + wire_size); + if (!desc) + return -1; + u32 token = epic->compact_token++ << 8; + desc->token = token; + desc->length = wire_size; + memcpy(desc->payload, txbuf, txsize); + int ret = afk_compact_send(epic, interface, type, 1, desc, sizeof(*desc) + wire_size); + free(desc); + if (ret < 0) + return ret; + + u64 deadline = timeout_calculate(5 * USEC_PER_SEC); + while (!timeout_expired(deadline)) { + ret = afk_epic_work(epic->afk, epic->ep); + if (ret < 0) + return ret; + if (ret != EPIC_DATA_READY) + continue; + struct afk_qe *msg; + if (afk_epic_rx(epic, &msg) < 0) + return -1; + ret = afk_compact_rx(epic, msg); + if (ret < 0) + return ret; + if (!ret) + continue; + struct compact_body *body = epic->compact_message; + if (body->category != 2 || body->type != type || epic->compact_interface != interface) + continue; + size_t payload_size = epic->compact_length - sizeof(*body); + if (payload_size < sizeof(struct compact_desc)) + return -1; + struct compact_desc *reply = (void *)body->payload; + if (reply->token != token) + continue; + size_t available = payload_size - sizeof(*reply); + if (rxsize) { + *rxsize = min(*rxsize, available); + memcpy(rxbuf, reply->payload, *rxsize); + } + return 0; + } + printf("AFK: compact command timed out\n"); + return -1; } int afk_epic_work(afk_epic_t *afk, int endpoint) @@ -381,7 +586,7 @@ int afk_epic_work(afk_epic_t *afk, int endpoint) afk_epic_ep_t *cur = afk->endpoint[i++]; if (cur) { struct afk_rb *rb = &cur->rx; - if (rb->hdr->rptr != rb->hdr->wptr) { + if (rb->ready && *rb->rptr != *rb->wptr) { if (cur->ep == endpoint) { return EPIC_DATA_READY; } @@ -496,6 +701,9 @@ static int afk_epic_handle_std_service(afk_epic_ep_t *epic, int channel, u8 cate int afk_epic_command(afk_epic_ep_t *epic, int channel, u16 sub_type, void *txbuf, size_t txsize, void *rxbuf, size_t *rxsize) { + if (epic->compact) + return afk_compact_command(epic, channel, sub_type, txbuf, txsize, rxbuf, rxsize); + struct { struct epic_hdr hdr; struct epic_sub_hdr sub; @@ -681,6 +889,7 @@ int afk_epic_shutdown_ep(afk_epic_ep_t *epic) epic->afk->endpoint[epic->ep - 0x20] = NULL; + free(epic->compact_message); free(epic); return 0; } @@ -739,6 +948,39 @@ int afk_epic_start_interface(afk_epic_ep_t *epic, void *intf, int expected, size if (ret < 0) return ret; + if (epic->compact || afk_is_compact(msg)) { + epic->compact = true; + ret = afk_compact_rx(epic, msg); + if (ret < 0) + return ret; + if (!ret) + continue; + struct compact_body *body = epic->compact_message; + if (body->category || body->type != COMPACT_ANNOUNCE || + epic->compact_length < sizeof(*body) + 32) + continue; + char name[33]; + memcpy(name, body->payload, 32); + name[32] = 0; + const afk_epic_service_ops_t *ops = afk_match_service(epic, name); + if (!ops || epic->num_channels >= AFK_MAX_CHANNEL) + continue; + afk_epic_service_t *service = &epic->services[epic->num_channels++]; + service->enabled = true; + service->ops = ops; + service->intf = intf; + service->epic = epic; + service->channel = epic->compact_interface; + ops->init(service, name, name, -1); + if (afk_compact_send(epic, service->channel, COMPACT_START, 0, NULL, 0) < 0) + return -1; + printf("AFK[ep:%02x]: compact service %s on interface %u\n", epic->ep, name, + service->channel); + if (++services >= expected) + break; + continue; + } + if (msg->type != TYPE_NOTIFY && msg->type != TYPE_REPLY) { dprintf("AFK[ep:%02x]: got unexpected message type %d during iface start\n", epic->ep, msg->type); @@ -822,6 +1064,9 @@ int afk_epic_start_interface(afk_epic_ep_t *epic, void *intf, int expected, size return -1; } + if (epic->compact) + return 0; + if (!rtkit_alloc_buffer(epic->afk->rtk, &epic->rxbuf, rxsize)) { printf("AFK[ep:%02x]: failed to allocate rx buffer\n", epic->ep); return -1; diff --git a/src/cpufreq.c b/src/cpufreq.c index ed541939..1e899a23 100644 --- a/src/cpufreq.c +++ b/src/cpufreq.c @@ -40,6 +40,7 @@ struct cluster_t { bool pcluster; uint32_t apsc_pstate; uint32_t default_pstate; + int (*init)(const struct cluster_t *cluster); }; struct feat_t { @@ -78,6 +79,7 @@ static u32 pstate_reg_to_pstate(u64 val) case T6034: case T8122: case T8132: + case T8142: return FIELD_GET(CLUSTER_PSTATE_DESIRED1, val); default: printf("cpufreq: Chip 0x%x is unsupported\n", chip_id); @@ -459,6 +461,32 @@ static struct cluster_t t8132_clusters[] = { {}, }; +/* Modern request interface: serialize against the previous request, then + * leave completion asynchronous, as in the Linux T8112 cpufreq driver. */ +static int cpufreq_init_request(const struct cluster_t *cluster) +{ + u64 command = cluster->base + CLUSTER_PSTATE; + if (poll64(command, CLUSTER_PSTATE_BUSY, 0, CLUSTER_SWITCH_TIMEOUT) < 0) { + printf("cpufreq: Timed out waiting for cluster %s request\n", cluster->name); + return -1; + } + mask64(command, CLUSTER_PSTATE_DESIRED1, + CLUSTER_PSTATE_SET | FIELD_PREP(CLUSTER_PSTATE_DESIRED1, cluster->default_pstate)); + printf("cpufreq: %s initialized at P%u\n", cluster->name, cluster->default_pstate); + return 0; +} + +/* No legacy APSC/throttle feature is advertised on this platform. */ +static const struct cluster_t t8142_clusters[] = { + {.name = "ECPU", .base = 0x210e00000, .default_pstate = 5, .init = cpufreq_init_request}, + {.name = "PCPU", + .base = 0x211e00000, + .pcluster = true, + .default_pstate = 6, + .init = cpufreq_init_request}, + {}, +}; + static const struct cluster_t *cpufreq_get_t8132_clusters(void) { int cpus = adt_path_offset(adt, "/cpus"); @@ -554,6 +582,8 @@ const struct cluster_t *cpufreq_get_clusters(void) return t6031_clusters; case T8132: return cpufreq_get_t8132_clusters(); + case T8142: + return t8142_clusters; default: printf("cpufreq: Chip 0x%x is unsupported\n", chip_id); return NULL; @@ -653,6 +683,10 @@ static const struct feat_t t8132_features[] = { {}, }; +static const struct feat_t t8142_features[] = { + {}, +}; + const struct feat_t *cpufreq_get_features(void) { switch (chip_id) { @@ -689,6 +723,8 @@ const struct feat_t *cpufreq_get_features(void) return t6030_features; case T8132: return t8132_features; + case T8142: + return t8142_features; default: printf("cpufreq: Chip 0x%x is unsupported\n", chip_id); return NULL; @@ -711,7 +747,8 @@ int cpufreq_init(void) bool err = false; while (cluster->base) { - err |= cpufreq_init_cluster(cluster++, features); + err |= cluster->init ? cluster->init(cluster) : cpufreq_init_cluster(cluster, features); + cluster++; } if (chip_id == T8132) diff --git a/src/dart.c b/src/dart.c index 7401668a..27e03028 100644 --- a/src/dart.c +++ b/src/dart.c @@ -67,6 +67,7 @@ #define DART_T8110_TCR_OFF 0x1000 #define DART_T8110_TCR_REMAP GENMASK(11, 8) #define DART_T8110_TCR_REMAP_EN BIT(7) +#define DART_T8110_TCR_FOUR_LEVELS BIT(3) #define DART_T8110_TCR_BYPASS_DAPF BIT(2) #define DART_T8110_TCR_BYPASS_DART BIT(1) #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) @@ -112,11 +113,13 @@ struct dart_params { struct dart_dev { bool locked; bool keep; + bool extended_tables; uintptr_t regs; u8 device; enum dart_type_t type; const struct dart_params *params; u64 vm_base; + u64 dva_base; u64 *l1[DART_MAX_TTBR_COUNT]; }; @@ -141,7 +144,7 @@ static void dart_t8110_tlb_invalidate(dart_dev_t *dart) FIELD_PREP(DART_T8110_TLB_CMD_OP, DART_T8110_TLB_CMD_OP_FLUSH_SID) | FIELD_PREP(DART_T8110_TLB_CMD_STREAM, dart->device)); - if (poll32(dart->regs + DART_T8110_TLB_CMD_OP, DART_T8110_TLB_CMD_BUSY, 0, 100)) + if (poll32(dart->regs + DART_T8110_TLB_CMD, DART_T8110_TLB_CMD_BUSY, 0, 100)) printf("dart: DART_T8110_TLB_CMD_BUSY did not clear.\n"); } @@ -236,6 +239,8 @@ dart_dev_t *dart_init(uintptr_t base, u8 device, bool keep_pts, enum dart_type_t } dart->keep = keep_pts; + dart->extended_tables = type == DART_T8110 && (dart->locked || keep_pts) && + (read32(DART_TCR(dart)) & DART_T8110_TCR_FOUR_LEVELS); if (dart->locked || keep_pts) { for (int i = 0; i < dart->params->ttbr_count; i++) { @@ -246,6 +251,15 @@ dart_dev_t *dart_init(uintptr_t base, u8 device, bool keep_pts, enum dart_type_t } } + /* Extended roots are inherited read-only; do not allocate or reconfigure them. */ + if (dart->extended_tables) { + if (!dart->l1[0]) { + free(dart); + return NULL; + } + return dart; + } + for (int i = 0; i < dart->params->ttbr_count; i++) { if (dart->l1[i]) continue; @@ -335,8 +349,12 @@ dart_dev_t *dart_init_adt(const char *path, int instance, int device, bool keep_ printf("dart: unexpected length of vm-base property: %u\n", len); } } - if (dart->locked) + if (dart->locked) { + /* Preserve the device-visible address prefix while allocating and + * indexing page tables in the inherited 36-bit DART aperture. */ + dart->dva_base = dart->vm_base & ~GENMASK(35, 0); dart->vm_base &= (1LLU << 36) - 1; + } return dart; } @@ -405,11 +423,21 @@ dart_dev_t *dart_init_fdt(void *dt, u32 phandle, int device, bool keep_pts) printf("dart: dart %s at 0x%lx is a %s%s\n", name, base, type_s, dart->locked ? " (locked)" : ""); + int len; + const fdt64_t *range = fdt_getprop(dt, node, "apple,dma-range", &len); + if (range && len == 2 * sizeof(*range)) { + u64 base = fdt64_ld(range); + dart->vm_base = base & GENMASK(35, 0); + dart->dva_base = base & ~GENMASK(35, 0); + } return dart; } int dart_setup_pt_region(dart_dev_t *dart, const char *path, int device, u64 vm_base) { + if (dart->extended_tables) + return -1; + int node = adt_path_offset(adt, path); if (node < 0) { printf("dart: Error getting DART node %s\n", path); @@ -541,6 +569,9 @@ static int dart_map_page(dart_dev_t *dart, uintptr_t iova, uintptr_t paddr, u32 int dart_map_flags(dart_dev_t *dart, uintptr_t iova, void *bfr, size_t len, u32 flags) { + if (dart->extended_tables) + return -1; + uintptr_t paddr = (uintptr_t)bfr; u64 offset = 0; @@ -586,6 +617,9 @@ static void dart_unmap_page(dart_dev_t *dart, uintptr_t iova) void dart_unmap(dart_dev_t *dart, uintptr_t iova, size_t len) { + if (dart->extended_tables) + return; + if (len % SZ_16K) return; if (iova % SZ_16K) @@ -603,6 +637,9 @@ void dart_unmap(dart_dev_t *dart, uintptr_t iova, size_t len) void dart_free_l2(dart_dev_t *dart, uintptr_t iova) { + if (dart->extended_tables) + return; + if (iova & ((1 << 25) - 1)) { printf("dart: %08lx is not at the start of L2 table\n", iova); return; @@ -628,42 +665,25 @@ void dart_free_l2(dart_dev_t *dart, uintptr_t iova) static void *dart_translate_internal(dart_dev_t *dart, uintptr_t iova, int silent) { - u32 ttbr = (iova >> 36) & 0x3; - u32 l1_index = (iova >> 25) & 0x7ff; - - if ((int)ttbr >= dart->params->ttbr_count) { - printf("dart[%lx %u]: ttbr out of range: %d\n", dart->regs, dart->device, ttbr); - return NULL; - } - - if (!dart->l1[ttbr]) { - printf("dart[%lx %u]: l1[%u] is not set\n", dart->regs, dart->device, ttbr); - return NULL; - } - - if (!(dart->l1[ttbr][l1_index] & DART_PTE_VALID)) { - if (!silent) - printf("dart[%lx %u]: l1 translation failure %x %lx\n", dart->regs, dart->device, - l1_index, iova); - return NULL; - } - - u32 l2_index = (iova >> 14) & 0x7ff; - u64 *l2 = (u64 *)(FIELD_GET(dart->params->offset_mask, dart->l1[ttbr][l1_index]) - << DART_PTE_OFFSET_SHIFT); - - if (!(l2[l2_index] & DART_PTE_VALID)) { - if (!silent) - printf("dart[%lx %u]: l2 translation failure %x:%x %lx\n", dart->regs, dart->device, - l1_index, l2_index, iova); + u32 ttbr = dart->extended_tables ? 0 : ((iova >> 36) & 0x3); + if (ttbr >= (u32)dart->params->ttbr_count || !dart->l1[ttbr]) return NULL; + u64 *table = dart->l1[ttbr]; + /* T8110 FOUR_LEVELS mode adds a root above the usual two tables. + * Table/leaf entries retain the same format. Follow the configured mode, + * including address bits above 36, rather than folding every DVA. */ + for (int level = dart->extended_tables ? 2 : 1; level >= 0; level--) { + u32 index = (iova >> (14 + 11 * level)) & 0x7ff; + u64 pte = table[index]; + if (!(pte & DART_PTE_VALID)) { + if (!silent) + printf("dart[%lx %u]: L%d translation failure at %lx\n", dart->regs, dart->device, + level, iova); + return NULL; + } + table = (void *)(FIELD_GET(dart->params->offset_mask, pte) << DART_PTE_OFFSET_SHIFT); } - - u32 offset = iova & 0x3fff; - void *base = - (void *)(FIELD_GET(dart->params->offset_mask, l2[l2_index]) << DART_PTE_OFFSET_SHIFT); - - return base + offset; + return (void *)((u64)table | (iova & (SZ_16K - 1))); } void *dart_translate(dart_dev_t *dart, uintptr_t iova) @@ -678,6 +698,9 @@ void *dart_translate_silent(dart_dev_t *dart, uintptr_t iova) u64 dart_search(dart_dev_t *dart, void *paddr) { + if (dart->extended_tables) + return DART_PTR_ERR; + for (int ttbr = 0; ttbr < dart->params->ttbr_count; ++ttbr) { if (!dart->l1[ttbr]) continue; @@ -703,6 +726,9 @@ u64 dart_search(dart_dev_t *dart, void *paddr) u64 dart_find_iova(dart_dev_t *dart, s64 start, size_t len) { + if (dart->extended_tables) + return DART_PTR_ERR; + if (len % SZ_16K) return -1; if (start < 0 || start % SZ_16K) @@ -732,6 +758,12 @@ u64 dart_find_iova(dart_dev_t *dart, s64 start, size_t len) void dart_shutdown(dart_dev_t *dart) { + if (dart->extended_tables) { + /* These roots and all child tables belong to resident firmware. */ + free(dart); + return; + } + if (!dart->locked && !dart->keep) write32(DART_TCR(dart), dart->params->tcr_disabled); @@ -763,3 +795,8 @@ u64 dart_vm_base(dart_dev_t *dart) { return dart->vm_base; } + +u64 dart_dva_base(dart_dev_t *dart) +{ + return dart->dva_base; +} diff --git a/src/dart.h b/src/dart.h index 739d9454..88b599ce 100644 --- a/src/dart.h +++ b/src/dart.h @@ -31,5 +31,7 @@ u64 dart_search(dart_dev_t *dart, void *paddr); u64 dart_find_iova(dart_dev_t *dart, s64 start, size_t len); void dart_shutdown(dart_dev_t *dart); u64 dart_vm_base(dart_dev_t *dart); +/* Device-visible address bits removed before the inherited DART translation. */ +u64 dart_dva_base(dart_dev_t *dart); #endif diff --git a/src/dcp.c b/src/dcp.c index 38bf34fd..e431721a 100644 --- a/src/dcp.c +++ b/src/dcp.c @@ -258,10 +258,12 @@ int dcp_shutdown(dcp_dev_t *dcp, bool sleep) free(dcp->phy); afk_epic_shutdown(dcp->afk); if (sleep) { - rtkit_sleep(dcp->rtkit); + if (!rtkit_sleep(dcp->rtkit)) + return -1; pmgr_reset(dcp_die, dcp_pmgr_dev); } else { - rtkit_quiesce(dcp->rtkit); + if (!rtkit_quiesce(dcp->rtkit)) + return -1; } rtkit_free(dcp->rtkit); dart_shutdown(dcp->dart_disp); diff --git a/src/firmware.c b/src/firmware.c index 18465c80..b402f0ba 100644 --- a/src/firmware.c +++ b/src/firmware.c @@ -203,6 +203,8 @@ const struct fw_version_info fw_versions[NUM_FW_VERSIONS] = { [V26_4] = {V26_4, "26.4", {26, 4, 0}, 3, "mBoot-18000.101.7"}, [V26_6] = {V26_6, "26.6", {26, 6, 0}, 3, "mBoot-18000.160.17"}, [V26_6_2] = {V26_6_2, "26.6.2", {26, 6, 2}, 3, "mBoot-18000.161.10"}, + [V27_2] = {V27_2, "27.2", {27, 2, 0}, 3, "mBoot-20457.41.3"}, + [V27_3B1] = {V27_3B1, "27.3 beta", {27, 2, 98, 1}, 4, "mBoot-20457.60.65.0.1"}, // clang-format on }; diff --git a/src/firmware.h b/src/firmware.h index 80955ef9..8c163483 100644 --- a/src/firmware.h +++ b/src/firmware.h @@ -186,6 +186,8 @@ enum fw_version { V26_4, V26_6, V26_6_2, + V27_2, + V27_3B1, NUM_FW_VERSIONS, }; diff --git a/src/hv_exc.c b/src/hv_exc.c index 02e575ce..dd47bbfe 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -853,6 +853,7 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) } return true; /* pass through PMU handling */ + SYSREG_PASS(SYS_IMP_APL_UPMCR0) SYSREG_PASS(SYS_IMP_APL_PMCR1) SYSREG_PASS(SYS_IMP_APL_PMCR2) SYSREG_PASS(SYS_IMP_APL_PMCR3) @@ -873,6 +874,12 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_PASS(SYS_IMP_APL_PMC8) SYSREG_PASS(SYS_IMP_APL_PMC9) + /* Keep the normal Python passthrough behavior local for frequent MTE accesses. */ + SYSREG_PASS(sys_reg(3, 0, 1, 0, 5)) // RGSR_EL1 + SYSREG_PASS(sys_reg(3, 0, 1, 0, 6)) // GCR_EL1 + SYSREG_PASS(sys_reg(3, 0, 5, 6, 0)) // TFSR_EL1 + SYSREG_PASS(sys_reg(3, 0, 5, 6, 1)) // TFSRE0_EL1 + /* Outer Sharable TLB maintenance instructions */ SYSREG_PASS(sys_reg(1, 0, 8, 1, 0)) // TLBI VMALLE1OS SYSREG_PASS(sys_reg(1, 0, 8, 1, 1)) // TLBI VAE1OS diff --git a/src/isp.c b/src/isp.c index ddce629f..b14c6bb7 100644 --- a/src/isp.c +++ b/src/isp.c @@ -23,6 +23,28 @@ #define ISP_PMGR_T6020 0x4008 #define ISP_PMGR_T6031 0x4030 +/* Firmware owns these ISP power domains at handoff. Reserve its host heap + * without probing or resetting the processor; Linux takes power ownership. */ +struct isp_heap_layout { + u32 chip; + enum fw_version firmware; + u64 shared_base; +}; + +static const struct isp_heap_layout isp_heap_layouts[] = { + {T8142, V27_2, 0x1700000}, + {T8142, V27_3B1, 0x1700000}, +}; + +static const struct isp_heap_layout *isp_find_heap_layout(void) +{ + for (size_t i = 0; i < ARRAY_SIZE(isp_heap_layouts); i++) + if (isp_heap_layouts[i].chip == chip_id && + isp_heap_layouts[i].firmware == os_firmware.version) + return &isp_heap_layouts[i]; + return NULL; +} + static bool isp_initialized = false; static u64 heap_phys, heap_iova, heap_size, heap_top; @@ -39,6 +61,14 @@ int isp_get_heap(u64 *phys, u64 *iova, u64 *size) u64 isp_iova_base(void) { + /* New handoffs describe the device-visible aperture on the DART. */ + int node = adt_path_offset(adt, "/arm-io/dart-isp"); + if (node < 0) + node = adt_path_offset(adt, "/arm-io/dart-isp0"); + u64 base; + if (node >= 0 && ADT_GETPROP(adt, node, "vm-base", &base) == sizeof(base)) + return base; + switch (chip_id) { case 0x6020 ... 0x6fff: case 0x8122: @@ -51,7 +81,10 @@ u64 isp_iova_base(void) int isp_init(void) { + if (isp_initialized) + return 0; int err = 0; + bool power_enabled = false; const char *isp_path = "/arm-io/isp"; const char *dart_path = "/arm-io/dart-isp"; @@ -68,6 +101,12 @@ int isp_init(void) if (node < 0) return 0; + const struct isp_heap_layout *layout = isp_find_heap_layout(); + if (layout) { + heap_top = layout->shared_base; + goto setup_heap; + } + /* * T8132 (M4) has the M4 Max's H16 ISP and runs the same macOS 26.6.2 * firmware, whose shared allocations start at 32 MiB (the M4 Max value); @@ -83,6 +122,8 @@ int isp_init(void) if (pmgr_adt_power_enable(isp_path) < 0) return -1; + power_enabled = true; + u64 isp_base; u64 pmgr_base; err = adt_get_reg(adt, adt_isp_path, "reg", 0, &isp_base, NULL); @@ -194,7 +235,12 @@ setup_heap:; } unsigned int count = segments_len / sizeof(*seg); - heap_iova = seg[count - 1].iova + seg[count - 1].size; + heap_iova = 0; + for (unsigned int i = 0; i < count; i++) { + if (seg[i].iova == UINT64_MAX || seg[i].size > UINT64_MAX - seg[i].iova) + continue; + heap_iova = max(heap_iova, seg[i].iova + seg[i].size); + } if (heap_iova >= heap_top) { printf("isp: firmware segments end above the heap top\n"); return -1; @@ -210,7 +256,7 @@ setup_heap:; isp_initialized = true; - if (chip_id != T8132) + if (power_enabled) pmgr_adt_power_disable(isp_path); return err; } diff --git a/src/kboot.c b/src/kboot.c index 33d0b42d..c28596b6 100644 --- a/src/kboot.c +++ b/src/kboot.c @@ -1021,6 +1021,9 @@ static int dt_set_wifi(void) { int anode = adt_path_offset(adt, "/arm-io/wlan"); + if (anode < 0) + anode = adt_path_offset(adt, "/arm-io/centauri"); + if (anode < 0) { /* * t8012-based system does not have wifi, and on some other @@ -1042,15 +1045,14 @@ static int dt_set_wifi(void) if (node < 0) return 0; - char antenna[8]; - memcpy(antenna, &info[8], sizeof(antenna)); + char antenna[9] = {0}; + memcpy(antenna, &info[8], 8); fdt_setprop_string(dt, node, "apple,antenna-sku", antenna); u32 len; const u8 *cal_blob = adt_getprop(adt, anode, "wifi-calibration-msf", &len); if (!cal_blob || !len) { - printf("ADT: Failed to get wifi-calibration-msf. Replaced module?\n"); return 0; } @@ -1059,6 +1061,21 @@ static int dt_set_wifi(void) return 0; } +/* The control function consumes the host's manifest identity; Linux owns + * firmware loading, endpoint reset, DMA state and radio initialization. */ +static int dt_set_centauri(void) +{ + int node = fdt_path_offset(dt, "wifi-control"); + if (node < 0 || adt_path_offset(adt, "/arm-io/centauri") < 0) + return 0; + int chosen = adt_path_offset(adt, "/chosen"); + u32 board, chip; + if (chosen < 0 || ADT_GETPROP(adt, chosen, "board-id", &board) < 0 || + ADT_GETPROP(adt, chosen, "chip-id", &chip) < 0 || board > UINT16_MAX || chip > UINT16_MAX) + bail("ADT: invalid host identity for Centauri\n"); + return fdt_setprop_u32(dt, node, "apple,host-platform-id", (board << 16) | chip); +} + static void dt_set_uboot_dm_preloc(int node) { // Tell U-Boot to bind this node early @@ -1414,7 +1431,7 @@ static u64 dart_get_mapping(dart_dev_t *dart, const char *path, u64 paddr, size_ return DART_PTR_ERR; } - return iova; + return iova | dart_dva_base(dart); } static int dt_device_set_reserved_mem(int node, const char *name, uint32_t phandle, u64 iova, @@ -1516,6 +1533,92 @@ static int dt_device_add_mem_region(const char *alias, uint32_t phandle, const c return 0; } +/* Replace an optional board-DT placeholder or append a new named region. */ +static int dt_device_set_named_mem_region(const char *alias, u32 phandle, const char *name) +{ + int node = fdt_path_offset(dt, alias); + if (node < 0) + return node; + int index = fdt_stringlist_search(dt, node, "memory-region-names", name); + if (index < 0) + return dt_device_add_mem_region(alias, phandle, name); + fdt32_t value = cpu_to_fdt32(phandle); + return fdt_setprop_inplace_namelen_partial(dt, node, "memory-region", 13, index * sizeof(value), + &value, sizeof(value)); +} + +/* Export only RAM segments still mapped by this consumer's actual DART. + * A second host may map different segments of the same resident firmware. */ +static int dt_reserve_mapped_asc(const char *adt_path, const char *alias, u64 excluded_base, + u64 excluded_size) +{ + int device = fdt_path_offset(dt, alias); + if (device < 0) + return 0; + int source = adt_path_offset(adt, adt_path); + if (source < 0) + return 0; + u32 bytes; + const struct adt_segment_ranges *segments = adt_getprop(adt, source, "segment-ranges", &bytes); + if (!segments || bytes % sizeof(*segments)) + bail("ADT: invalid resident segment table for %s\n", alias); + u32 owner = fdt_get_phandle(dt, device); + if (!owner && + (fdt_generate_phandle(dt, &owner) || fdt_setprop_u32(dt, device, "phandle", owner))) + return -1; + dart_dev_t *dart = dt_init_dart_by_node(device, 0); + if (!dart) + return -1; + int chosen = adt_path_offset(adt, "/chosen"); + u64 dram_base = ALIGN_DOWN(cur_boot_args.phys_base, BIT(32)); + if (chosen >= 0) + ADT_GETPROP(adt, chosen, "dram-base", &dram_base); + int ret = 0; + for (size_t i = 0; i < bytes / sizeof(*segments); i++) { + const struct adt_segment_ranges *seg = &segments[i]; + if (seg->phys < dram_base || !seg->size || + (excluded_size && seg->phys >= excluded_base && + seg->phys - excluded_base < excluded_size)) + continue; + u64 iova = seg->remap; + u64 address = iova; + size_t size = ALIGN_UP(seg->size, SZ_16K); + bool mapped = true; + for (size_t offset = 0; offset < size; offset += SZ_16K) { + if ((u64)dart_translate_silent(dart, address + offset) != seg->phys + offset) { + mapped = false; + break; + } + } + if (!mapped) + continue; + char node_name[64], region_name[32]; + snprintf(node_name, sizeof(node_name), "asc-firmware@%lx", seg->phys); + snprintf(region_name, sizeof(region_name), "firmware-%zu", i); + int mem = dt_get_or_add_reserved_mem(node_name, "apple,asc-mem", true, seg->phys, size); + if (mem < 0) { + ret = mem; + break; + } + u32 phandle = fdt_get_phandle(dt, mem); + ret = dt_device_set_reserved_mem(mem, node_name, owner, iova, size); + if (ret) + break; + ret = dt_device_set_named_mem_region(alias, phandle, region_name); + if (ret) + break; + } + dart_shutdown(dart); + return ret; +} + +static int dt_setup_aop(void) +{ + if (dt_reserve_mapped_asc("/arm-io/aop", "aop", 0, 0)) + return -1; + return dt_reserve_mapped_asc("/arm-io/aop", "aop-secure", 0, 0); +} + static int dt_set_dcp_firmware(const char *alias) { const char *path = fdt_get_alias(dt, alias); @@ -1770,8 +1873,49 @@ static int dt_vram_reserved_region(const char *dcp_alias, const char *disp_alias if (ret < 0) bail("ADT: failed to read /vram/reg\n"); - return dt_add_reserved_regions(dcp_alias, disp_alias, NULL, "framebuffer", - disp_reserved_regions_vram, ®ion, 1); + int pool = fdt_path_offset(dt, "scanout-pool"); + if (pool >= 0) { + u64 fb_size = ALIGN_UP(cur_boot_args.video.stride * cur_boot_args.video.height, SZ_16K); + if (!fb_size || fb_size >= region.size) + bail("FDT: VRAM has no room for a scanout pool\n"); + fdt64_t reg[2] = {cpu_to_fdt64(region.paddr + fb_size), + cpu_to_fdt64(region.size - fb_size)}; + u32 phandle = fdt_get_phandle(dt, pool); + if (fdt_setprop(dt, pool, "reg", reg, sizeof(reg)) || + fdt_setprop_string(dt, pool, "status", "okay")) + return -1; + if (dt_device_set_named_mem_region(disp_alias, phandle, "scanout-pool")) + return -1; + if (dt_device_set_named_mem_region(dcp_alias, phandle, "scanout-pool")) + return -1; + + /* Firmware may still access boot-time surfaces outside the active + * framebuffer while IOMFB starts. Preserve those translations as + * well as making the physical pages available for Linux scanout. */ + const char *consumers[] = {dcp_alias, disp_alias}; + for (size_t i = 0; i < ARRAY_SIZE(consumers); i++) { + int device = fdt_path_offset(dt, consumers[i]); + u32 owner = fdt_get_phandle(dt, device); + dart_dev_t *dart = dt_init_dart_by_node(device, 0); + if (!dart) + return -1; + pool = fdt_path_offset(dt, "scanout-pool"); + ret = dt_device_set_reserved_mem_from_dart( + pool, dart, "scanout-pool", owner, region.paddr + fb_size, region.size - fb_size); + dart_shutdown(dart); + if (ret) + return ret; + } + region.size = fb_size; + } + ret = dt_add_reserved_regions(dcp_alias, disp_alias, NULL, "framebuffer", + disp_reserved_regions_vram, ®ion, 1); + if (ret) + return ret; + int display_node = fdt_path_offset(dt, disp_alias); + if (display_node >= 0 && fdt_setprop_string(dt, display_node, "status", "okay")) + return -1; + return 0; } static int dt_reserve_asc_firmware(const char *adt_path, const char *adt_path_alt, @@ -1828,7 +1972,12 @@ static int dt_reserve_asc_firmware(const char *adt_path, const char *adt_path_al if (ret < 0) return ret; - ret = dt_device_add_mem_region(fdt_path, mem_phandle, NULL); + char name[24]; + if (i) + snprintf(name, sizeof(name), "firmware-%u", i); + else + strcpy(name, "firmware"); + ret = dt_device_set_named_mem_region(fdt_path, mem_phandle, name); if (ret < 0) return ret; @@ -2040,6 +2189,16 @@ static int dt_set_display(void) ARRAY_SIZE(disp_reserved_regions_t602x)); if (ret) return ret; + } else if (!fdt_node_check_compatible(dt, 0, "apple,t8142")) { + u64 phys, size; + if (display_get_vram(&phys, &size)) + return -1; + ret = dt_reserve_mapped_asc("/arm-io/dcp/iop-dcp-nub", "dcp", phys, size); + if (ret) + return ret; + ret = dt_set_dcp_firmware("dcp"); + if (ret) + return ret; } else if (!fdt_node_check_compatible(dt, 0, "apple,t6022")) { /* noop */ } else { @@ -2292,6 +2451,9 @@ static int dt_set_isp_fwdata(void) const struct fw_version_info *compat; switch (os_firmware.version) { + case V27_3B1: + compat = &fw_versions[V27_2]; + break; case V13_6_1: compat = &fw_versions[V13_5]; break; @@ -2338,7 +2500,11 @@ static int dt_set_isp_fwdata(void) if (ret < 0) return ret; - return 0; + ret = dt_device_set_named_mem_region(fdt_path, fdt_get_phandle(dt, mem_node), "heap"); + if (ret) + return ret; + fdt_node = fdt_path_offset(dt, fdt_path); + return fdt_setprop_string(dt, fdt_node, "status", "okay"); } static int dt_disable_missing_devs(const char *adt_prefix, const char *dt_prefix, int max_devs) @@ -2842,6 +3008,8 @@ int kboot_prepare_dt(void *fdt) return -1; if (dt_set_wifi()) return -1; + if (dt_set_centauri()) + return -1; if (dt_set_bluetooth()) return -1; if (dt_set_uboot()) @@ -2872,6 +3040,8 @@ int kboot_prepare_dt(void *fdt) return -1; if (dt_setup_sio()) return -1; + if (dt_setup_aop()) + return -1; if (dt_reserve_asc_firmware("/arm-io/isp", "/arm-io/isp0", "isp", false, isp_iova_base())) return -1; if (dt_set_isp_fwdata()) diff --git a/src/mcc.c b/src/mcc.c index b7939108..5d43b92d 100644 --- a/src/mcc.c +++ b/src/mcc.c @@ -147,6 +147,7 @@ struct mcc_regs { int cache_ways; u32 cache_status_mask; u32 cache_status_val; + u32 cache_status_reg; u32 cache_disable; bool has_cache_ctrl; @@ -188,11 +189,12 @@ int mcc_enable_cache(void) use it */ for (int mcc = 0; mcc < mcc_count; mcc++) { for (int plane = 0; plane < mcc_regs[mcc].plane_count; plane++) { + u32 status_reg = mcc_regs[mcc].cache_status_reg ?: PLANE_CACHE_STATUS; plane_write32(mcc, plane, PLANE_CACHE_ENABLE, mcc_regs[mcc].cache_enable_val); - if (plane_poll32(mcc, plane, PLANE_CACHE_STATUS, mcc_regs[mcc].cache_status_mask, + if (plane_poll32(mcc, plane, status_reg, mcc_regs[mcc].cache_status_mask, mcc_regs[mcc].cache_status_val, CACHE_ENABLE_TIMEOUT)) { printf("MCC: timeout while enabling cache for MCC %d plane %d: 0x%x\n", mcc, plane, - plane_read32(mcc, plane, PLANE_CACHE_STATUS)); + plane_read32(mcc, plane, status_reg)); ret = -1; } else if (mcc_regs[mcc].cache_disable) { plane_write32(mcc, plane, mcc_regs[mcc].cache_disable, 0); @@ -523,7 +525,7 @@ int mcc_init_m3(int node, int *path) return 0; } -int mcc_init_t8140(int node, int *path) +int mcc_init_adt(int node, int *path, u32 cache_enable_val) { u32 amcc_count = 1; u32 amcc_reg_idx = 0; @@ -531,8 +533,9 @@ int mcc_init_t8140(int node, int *path) u32 plane_count = 0; u32 plane_stride = T8140_PLANE_STRIDE; int lock_node; + u32 status_reg = 0, status_mask = 0, status_val = 0; - printf("MCC: Initializing T8140 AMCCs...\n"); + printf("MCC: Initializing AMCCs from ADT...\n"); if (ADT_GETPROP(adt, node, "amcc-reg-idx", &amcc_reg_idx) < 0) { printf("MCC: Failed to get amcc-reg-idx property!\n"); @@ -561,6 +564,20 @@ int mcc_init_t8140(int node, int *path) if (!plane_count) plane_count = 4; + if (cache_enable_val) { + /* The old per-way counters at 0x1c04 have changed. The handoff ADT + * supplies the cache-ready predicate for all AMCC planes. */ + if (lock_node < 0 || + ADT_GETPROP(adt, lock_node, "cache-status-reg-offset", &status_reg) < 0 || + ADT_GETPROP(adt, lock_node, "cache-status-reg-mask", &status_mask) < 0 || + ADT_GETPROP(adt, lock_node, "cache-status-reg-value", &status_val) < 0 || + !status_mask || (status_val & ~status_mask) || (status_reg & 3) || + plane_stride < sizeof(u32) || status_reg > plane_stride - sizeof(u32)) { + printf("MCC: Invalid cache status description\n"); + return -1; + } + } + mcc_count = amcc_count; if (mcc_count > MAX_MCC_INSTANCES) { printf("MCC: Too many instances, increase MAX_MCC_INSTANCES!\n"); @@ -576,11 +593,15 @@ int mcc_init_t8140(int node, int *path) mcc_regs[i].plane_stride = plane_stride; mcc_regs[i].plane_count = plane_count; mcc_regs[i].dcs_count = dcs_count; - mcc_regs[i].has_cache_ctrl = false; + mcc_regs[i].has_cache_ctrl = !!cache_enable_val; + mcc_regs[i].cache_enable_val = cache_enable_val; + mcc_regs[i].cache_status_reg = status_reg; + mcc_regs[i].cache_status_mask = status_mask; + mcc_regs[i].cache_status_val = status_val; mcc_regs[i].tz = NULL; } - printf("MCC: Initialized T8140 AMCCs (%d instances, %d planes, %d channels)\n", mcc_count, + printf("MCC: Initialized AMCCs (%d instances, %d planes, %d channels)\n", mcc_count, mcc_regs[0].plane_count, mcc_regs[0].dcs_count); mcc_initialized = true; @@ -615,7 +636,9 @@ int mcc_init(void) } else if (adt_is_compatible(adt, node, "mcc,t6031")) { return mcc_init_m3(node, path); } else if (adt_is_compatible(adt, node, "mcc,t8140")) { - return mcc_init_t8140(node, path); + return mcc_init_adt(node, path, 0); + } else if (adt_is_compatible(adt, node, "mcc,t8142")) { + return mcc_init_adt(node, path, 1); } else { printf("MCC: Unsupported version:%s\n", adt_get_property(adt, node, "compatible")->value); return -1; diff --git a/src/rtkit.c b/src/rtkit.c index aca64d6a..442eb040 100644 --- a/src/rtkit.c +++ b/src/rtkit.c @@ -40,8 +40,8 @@ #define MSG_SYSLOG_LOG 5 #define MSG_SYSLOG_LOG_INDEX GENMASK(7, 0) -#define MSG_OSLOG_INIT 0x10 -#define MSG_OSLOG_ACK 0x30 +#define MSG_OSLOG_TYPE GENMASK(63, 56) +#define MSG_OSLOG_NOTIFY 2 #define MGMT_MSG_HELLO 1 #define MGMT_MSG_HELLO_ACK 2 @@ -164,7 +164,10 @@ rtkit_dev_t *rtkit_init(const char *name, asc_dev_t *asc, dart_dev_t *dart, rtk->dva_base = 0; int iop_node = asc_get_iop_node(asc); - ADT_GETPROP(adt, iop_node, "asc-dram-mask", &rtk->dva_base); + /* Recent DCP handoffs put the device-visible aperture in the DART's + * vm-base instead of providing asc-dram-mask on the IOP nub. */ + if (ADT_GETPROP(adt, iop_node, "asc-dram-mask", &rtk->dva_base) < 0 && dart) + rtk->dva_base = dart_dva_base(dart); return rtk; @@ -461,6 +464,13 @@ int rtkit_recv(rtkit_dev_t *rtk, struct rtkit_message *msg) } break; case RTKIT_EP_OSLOG: + /* Notifications must be acknowledged even without a log + * consumer, or the firmware cannot complete AP quiesce. */ + if (FIELD_GET(MSG_OSLOG_TYPE, msg->msg) == MSG_OSLOG_NOTIFY) { + if (!rtkit_send(rtk, msg)) + return -1; + break; + } rtkit_printf("unknown oslog message %lx\n", msg->msg); break; default: @@ -634,7 +644,12 @@ bool rtkit_boot(rtkit_dev_t *rtk) if (has_oslog && !rtkit_start_ep(rtk, RTKIT_EP_OSLOG)) return false; + u64 deadline = timeout_calculate(USEC_PER_SEC); while (rtk->iop_power != RTKIT_POWER_ON) { + if (timeout_expired(deadline)) { + rtkit_printf("timed out waiting for IOP power on\n"); + return false; + } struct rtkit_message rtk_msg; int ret = rtkit_recv(rtk, &rtk_msg); if (ret == 1) @@ -672,7 +687,12 @@ static bool rtkit_switch_power_state(rtkit_dev_t *rtk, enum rtkit_power_state ta return false; } + u64 deadline = timeout_calculate(USEC_PER_SEC); while (rtk->ap_power != RTKIT_POWER_QUIESCED) { + if (timeout_expired(deadline)) { + rtkit_printf("AP quiesce timed out (AP=%x)\n", rtk->ap_power); + return false; + } struct rtkit_message rtk_msg; int ret = rtkit_recv(rtk, &rtk_msg); @@ -692,7 +712,12 @@ static bool rtkit_switch_power_state(rtkit_dev_t *rtk, enum rtkit_power_state ta return false; } + deadline = timeout_calculate(USEC_PER_SEC); while (rtk->iop_power != target) { + if (timeout_expired(deadline)) { + rtkit_printf("IOP power timed out (IOP=%x target=%x)\n", rtk->iop_power, target); + return false; + } struct rtkit_message rtk_msg; int ret = rtkit_recv(rtk, &rtk_msg); @@ -716,10 +741,9 @@ bool rtkit_quiesce(rtkit_dev_t *rtk) bool rtkit_sleep(rtkit_dev_t *rtk) { - int ret = rtkit_switch_power_state(rtk, RTKIT_POWER_SLEEP); - if (ret < 0) - return ret; + if (!rtkit_switch_power_state(rtk, RTKIT_POWER_SLEEP)) + return false; asc_cpu_stop(rtk->asc); - return 0; + return true; }