qualcomm/qbox
Loading...
Searching...
No Matches
smmuv3.h
1/*
2 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause-Clear
5 */
6
7#ifndef SMMUV3_H
8#define SMMUV3_H
9namespace gs {
10const void* smmuv3_zip_archive_data() noexcept;
11unsigned int smmuv3_zip_archive_size() noexcept;
12} // namespace gs
13
14#include <systemc>
15#include <cci_configuration>
16#include <cciutils.h>
17#include <scp/report.h>
18#include <tlm>
19#include <module_factory_registery.h>
20#include <ports/initiator-signal-socket.h>
21#include <ports/target-signal-socket.h>
22#include <tlm_utils/simple_initiator_socket.h>
23#include <tlm_utils/simple_target_socket.h>
24#include <tlm-extensions/underlying-dmi.h>
25#include <tlm-extensions/qemu-memtx-attrs.h>
26#include <registers.h>
27#include <reg_model_maker/reg_model_maker.h>
28#include <zip_loader.h>
29
30#include <algorithm>
31#include <array>
32#include <cassert>
33#include <cstdint>
34#include <cstring>
35#include <memory>
36#include <vector>
37#include <unordered_map>
38#include <list>
39
40#include "smmuv3_gen.h"
41#include "smmuv3_memory_attrs.h"
42#include "smmuv3_extensions.h"
43
44namespace gs {
45
46constexpr uint64_t SMMUV3_PAGESIZE = 4096;
47constexpr uint64_t SMMUV3_PAGEMASK = SMMUV3_PAGESIZE - 1;
48constexpr uint32_t
49 SMMUV3_MAX_TBU = 16; // TBU = Translation Buffer Unit: per-master front-end that owns one upstream port
50
51// Per-record sizes for the structures the SMMU walks/produces in memory.
52constexpr size_t SMMUV3_STE_SIZE = 64; // STE = Stream Table Entry (one per upstream device, indexed by StreamID)
53constexpr size_t
54 SMMUV3_CD_SIZE = 64; // CD = Context Descriptor (one per address space behind a device, holds page-table roots)
55constexpr size_t SMMUV3_CMD_SIZE = 16; // command-queue entry size
56constexpr size_t SMMUV3_EVENT_SIZE = 32; // event-queue entry size
57constexpr size_t SMMUV3_PRI_SIZE = 16; // PRI = Page Request Interface queue entry size
58
59// EVT_* = EVenT type codes the SMMU writes into the event queue when something happens.
60// Prefix F_ = Fault (translation-time error against a transaction);
61// Prefix C_ = Configuration error (bad table contents the driver gave us).
62constexpr uint32_t SMMUV3_EVT_F_UUT = 0x01; // UUT = Unsupported Upstream Transaction
63constexpr uint32_t SMMUV3_EVT_C_BAD_STREAMID = 0x02; // transaction came in with a StreamID outside the configured table
64constexpr uint32_t SMMUV3_EVT_F_STE_FETCH = 0x03; // bus error while reading the Stream Table Entry
65constexpr uint32_t SMMUV3_EVT_C_BAD_STE = 0x04; // Stream Table Entry contents are malformed
66constexpr uint32_t SMMUV3_EVT_C_BAD_SUBSTREAMID = 0x08; // SubstreamID is outside the range the STE allows
67constexpr uint32_t SMMUV3_EVT_F_CD_FETCH = 0x09; // bus error while reading the Context Descriptor
68constexpr uint32_t SMMUV3_EVT_C_BAD_CD = 0x0A; // Context Descriptor contents are malformed
69constexpr uint32_t SMMUV3_EVT_F_WALK_EABT = 0x0B; // EABT = External AborT: bus error during page-table walk
70constexpr uint32_t SMMUV3_EVT_F_TRANSLATION = 0x10; // VA had no valid mapping
71constexpr uint32_t SMMUV3_EVT_F_ADDR_SIZE = 0x11; // an address (input or output) didn't fit the configured size
72constexpr uint32_t SMMUV3_EVT_F_ACCESS = 0x12; // Access-Flag was clear and HW couldn't set it
73constexpr uint32_t SMMUV3_EVT_F_PERMISSION = 0x13; // page exists but read/write permission was denied
74constexpr uint32_t SMMUV3_EVT_F_STALL = 0x29; // synthetic event used when a fault is being stalled (driver must resume)
75
76// CERROR = Command-queue ERROR codes the SMMU writes back to CMDQ_CONS.ERR when it rejects a command.
77constexpr uint8_t SMMUV3_CERROR_NONE = 0; // no error
78constexpr uint8_t SMMUV3_CERROR_ILL = 1; // ILL = ILLegal/unrecognised opcode
79constexpr uint8_t SMMUV3_CERROR_ABT = 2; // ABT = bus AborT while fetching the command word from memory
80constexpr uint8_t SMMUV3_CERROR_ATC_INV_SYNC = 3; // ATC = Address Translation Cache: invalidation sync timed out
81
82// CLASS = which subsystem caused the fault (used in event records).
83constexpr uint32_t SMMUV3_CLASS_TT = 0; // TT = Translation Table walk
84constexpr uint32_t SMMUV3_CLASS_IN = 1; // IN = INput-side (e.g. stage-2 fault on an input address)
85constexpr uint32_t SMMUV3_CLASS_CD = 2; // CD = Context-Descriptor fetch
86
87// EVT_W0_*/EVT_W1_* = bit positions/masks within Word 0 and Word 1 of an event-queue entry.
88constexpr uint32_t EVT_W0_TYPE_MASK = 0xFFu;
89constexpr uint32_t EVT_W0_SSV_BIT = 12; // SSV = SubStream Valid (the SSID field below is meaningful)
90constexpr uint32_t EVT_W0_SSID_SHIFT = 13; // SSID = SubStream ID (sub-context within a StreamID)
91constexpr uint32_t EVT_W0_SSID_MASK = 0xFFFFFu;
92constexpr uint32_t EVT_W0_STREAMID_SHIFT = 32;
93constexpr uint32_t
94 EVT_W1_STAG_MASK = 0xFFFFu; // STAG = Stall TAG: handle the driver uses to resume a stalled transaction
95constexpr uint32_t EVT_W1_RNW_BIT = 17; // RNW = Read-Not-Write (1 = read, 0 = write)
96constexpr uint32_t EVT_W1_CLASS_SHIFT = 20;
97constexpr uint32_t EVT_W1_CLASS_MASK = 0x3u;
98constexpr uint32_t EVT_W1_STALL_BIT = 23; // 1 if this fault was delivered as stallable (driver must RESUME)
99constexpr uint32_t EVT_W1_REASON_SHIFT = 24; // REASON = page-table level / sub-reason of the fault
100constexpr uint32_t EVT_W1_REASON_MASK = 0x7u;
101
102// GERROR_BIT_* = bit positions inside the GERROR (Global Error) register. MSI variants flag a failure
103// to deliver the interrupt itself; non-MSI ones flag the underlying queue/error condition.
104constexpr uint32_t GERROR_BIT_MSI_CMDQ_ABT = 4;
105constexpr uint32_t GERROR_BIT_MSI_EVENTQ_ABT = 5;
106constexpr uint32_t GERROR_BIT_MSI_PRIQ_ABT = 6;
107constexpr uint32_t GERROR_BIT_MSI_GERROR_ABT = 7;
108
109// S1FMT = Stage-1 stream-table FMT (format) encodings: how the per-StreamID Context Descriptor table is laid out.
110constexpr uint32_t S1FMT_LINEAR = 0; // single flat array of CDs indexed by SubstreamID
111constexpr uint32_t S1FMT_4KB_L2 = 1; // two-level table whose leaf pages are 4 KiB
112constexpr uint32_t S1FMT_64KB_L2 = 2; // two-level table whose leaf pages are 64 KiB
113constexpr uint32_t S1FMT_4KB_L2_SPLIT = 4; // SPLIT = how many low SubstreamID bits index the L2 (4 KiB case)
114constexpr uint32_t S1FMT_64KB_L2_SPLIT = 8; // SPLIT for the 64 KiB-leaf case
115
116// DESC_BIT_* = bit positions inside a 64-bit page-table DESCriptor word.
117constexpr uint32_t DESC_BIT_AF = 10; // AF = Access Flag: HW sets this on first use of a page
118constexpr uint32_t
119 DESC_BIT_DBM = 51; // DBM = Dirty Bit Modifier: writeable page that should be marked dirty on first write
120
121// load_le / store_le: little-endian (LE) read/write of an unaligned byte buffer. Stream-table entries,
122// command-queue entries and so on are LE-packed in memory, and these helpers do the byte-safe conversion.
123template <typename T>
124static inline T load_le(const uint8_t* buf)
125{
126 T v{};
127 std::memcpy(&v, buf, sizeof(T));
128 return v;
129}
130
131template <typename T>
132static inline void store_le(uint8_t* buf, T v)
133{
134 std::memcpy(buf, &v, sizeof(T));
135}
136
137static inline unsigned clamp_shift_u64(unsigned shift)
138{
139 constexpr unsigned MAX_SHIFT = 63;
140 return shift > MAX_SHIFT ? MAX_SHIFT : shift;
141}
142
143static inline uint64_t safe_shl1_u64(unsigned shift) { return 1ULL << clamp_shift_u64(shift); }
144
145static inline uint32_t clamp_queue_log2size(uint32_t log2size)
146{
147 constexpr uint32_t QUEUE_LOG2_MAX = 19;
148 return log2size > QUEUE_LOG2_MAX ? QUEUE_LOG2_MAX : log2size;
149}
150
151// STE_CONFIG_* = decoded values of the STE.CONFIG field. They tell the SMMU what to do for this device.
152constexpr uint32_t STE_CONFIG_ABORT = 0; // refuse all traffic for this StreamID
153constexpr uint32_t STE_CONFIG_BYPASS = 4; // pass through untranslated (1:1 VA = PA)
154constexpr uint32_t STE_CONFIG_S1 = 5; // S1 = Stage-1 only translation (per-process page tables)
155constexpr uint32_t STE_CONFIG_S2 = 6; // S2 = Stage-2 only translation (hypervisor page tables)
156constexpr uint32_t STE_CONFIG_NESTED = 7; // NESTED = Stage-1 then Stage-2 (guest VA -> guest PA -> host PA)
157
158// CD_TG_* = Translation Granule encoding inside a Context Descriptor (page size of S1 page tables).
159constexpr uint8_t CD_TG_4K = 0;
160constexpr uint8_t CD_TG_64K = 1;
161constexpr uint8_t CD_TG_16K = 2;
162
163// TLBI_TG_* = Translation Granule encoding used in TLBI (TLB Invalidate) commands. Note: this encoding
164// differs from CD_TG_* — DC means "Default/Currently-active", and 4K/16K/64K reorder vs CD_TG_*.
165constexpr uint8_t TLBI_TG_DC = 0;
166constexpr uint8_t TLBI_TG_4K = 1;
167constexpr uint8_t TLBI_TG_16K = 2;
168constexpr uint8_t TLBI_TG_64K = 3;
169
170// PAGE_SHIFT_* = log2 of the page size, i.e. how many low bits of an address are the offset within a page.
171constexpr uint32_t PAGE_SHIFT_4K = 12;
172constexpr uint32_t PAGE_SHIFT_16K = 14;
173constexpr uint32_t PAGE_SHIFT_64K = 16;
174
175// GRAINSIZE_* = log2 of the page-table "grain" (same numeric value as the page shift; "grain" is the table-walk term).
176constexpr uint32_t GRAINSIZE_4K = 12;
177constexpr uint32_t GRAINSIZE_16K = 14;
178constexpr uint32_t GRAINSIZE_64K = 16;
179
180// STRIDE_* = number of address bits resolved by one page-table level (i.e. log2 of entries-per-table).
181constexpr uint32_t STRIDE_4K = 9;
182constexpr uint32_t STRIDE_16K = 11;
183constexpr uint32_t STRIDE_64K = 13;
184
185// More GERROR (Global Error) bit positions; ABT = aBorT (bus error talking to that queue).
186constexpr uint32_t GERROR_BIT_CMDQ_ERR = 0;
187constexpr uint32_t GERROR_BIT_EVENTQ_ABT = 2;
188constexpr uint32_t GERROR_BIT_PRIQ_ABT = 3;
189
190constexpr uint32_t GERROR_BIT_SFM_ERR = 8; // SFM = Stream Fault Mode: a fatal stream-table walk error
191
192// DESC_TYPE_* = the low 2 bits of a page-table descriptor say what kind of entry it is.
193constexpr uint32_t DESC_TYPE_INVALID = 0; // entry is unused
194constexpr uint32_t DESC_TYPE_BLOCK = 1; // BLOCK = a large (multi-page) leaf mapping at this level
195constexpr uint32_t DESC_TYPE_RESERVED = 2;
196constexpr uint32_t DESC_TYPE_TABLE_OR_PAGE = 3; // mid-levels: pointer to next-level table; leaf level: a single page
197
198// DESC_ATTR_* = bit positions inside the low-attribute byte of a descriptor.
199constexpr uint32_t DESC_ATTR_AP1_BIT = 4; // AP1 = Access Permissions bit 1 (User access allowed)
200constexpr uint32_t DESC_ATTR_AP2_BIT = 5; // AP2 = Access Permissions bit 2 (Read-only)
201constexpr uint32_t DESC_ATTR_AF_BIT = 8; // AF = Access Flag (page has been touched)
202
203// S2AP = Stage-2 Access Permissions encoding (read/write rights granted by the hypervisor).
204constexpr uint32_t S2AP_NONE = 0;
205constexpr uint32_t S2AP_RO = 1; // RO = Read-Only
206constexpr uint32_t S2AP_WO = 2; // WO = Write-Only
207constexpr uint32_t S2AP_RW = 3; // RW = Read+Write
208
209// Address-field masks used to strip control bits out of register values that hold a base address.
210constexpr uint64_t STE_BASE_ADDR_MASK = 0x000FFFFFFFFFFFC0ULL; // Stream-table base / S1 context pointer
211constexpr uint64_t S2TTB_ADDR_MASK = 0x000FFFFFFFFFFFF0ULL; // S2TTB = Stage-2 Translation Table Base
212constexpr uint64_t
213 IPA_ADDR_MASK = 0x000FFFFFFFFFF000ULL; // IPA = Intermediate Physical Address (output of stage-1 / input to stage-2)
214constexpr uint64_t CMDQ_BASE_HI_MASK = 0x000FFFFFULL;
215
216// CMD_OP_* = OPcode byte that identifies a command-queue command.
217// PREFETCH_* - hint commands; can be ignored.
218// CFGI_* = ConFiGuration Invalidate: drop cached copies of stream-table / context-descriptor entries.
219// TLBI_* = TLB Invalidate: drop cached translations.
220// NH = Non-Hyp (EL1/EL0 stage-1 traffic), NS = Non-Secure, NSNH = Non-Secure Non-Hyp,
221// EL2/EL3 = the corresponding privileged exception levels,
222// S12 = both stages, S2 = stage-2 only,
223// VA = by Virtual Address, VAA = by VA, All ASIDs, IPA = by Intermediate PA, ASID = by Address-Space
224// ID, VMALL = all entries belonging to a VMID.
225// ATC_INV = Address Translation Cache Invalidate (tells PCIe ATS clients to drop a cached translation).
226// PRI_RESP = Page Request Interface RESPonse (driver answering a previously-queued PRI request).
227// RESUME = release a stalled transaction (after the OS made the page available).
228// STALL_TERM = TERMinate stall (kill the transaction with a fault).
229// SYNC = barrier: all earlier commands complete before this one is acknowledged.
230constexpr uint8_t CMD_OP_PREFETCH_CONFIG = 0x01;
231constexpr uint8_t CMD_OP_PREFETCH_ADDR = 0x02;
232constexpr uint8_t CMD_OP_CFGI_STE = 0x03;
233constexpr uint8_t CMD_OP_CFGI_STE_RANGE = 0x04;
234constexpr uint8_t CMD_OP_CFGI_CD = 0x05;
235constexpr uint8_t CMD_OP_CFGI_CD_ALL = 0x06;
236constexpr uint8_t CMD_OP_CFGI_ALL = 0x07;
237constexpr uint8_t CMD_OP_TLBI_NH_ALL = 0x10;
238constexpr uint8_t CMD_OP_TLBI_NH_ASID = 0x11;
239constexpr uint8_t CMD_OP_TLBI_NH_VA = 0x12;
240constexpr uint8_t CMD_OP_TLBI_NH_VAA = 0x13;
241constexpr uint8_t CMD_OP_TLBI_EL3_ALL = 0x18;
242constexpr uint8_t CMD_OP_TLBI_EL3_VA = 0x1A;
243constexpr uint8_t CMD_OP_TLBI_EL2_ALL = 0x20;
244constexpr uint8_t CMD_OP_TLBI_EL2_ASID = 0x21;
245constexpr uint8_t CMD_OP_TLBI_EL2_VA = 0x22;
246constexpr uint8_t CMD_OP_TLBI_EL2_VAA = 0x23;
247constexpr uint8_t CMD_OP_TLBI_S12_VMALL = 0x28;
248constexpr uint8_t CMD_OP_TLBI_S2_IPA = 0x2A;
249constexpr uint8_t CMD_OP_TLBI_NSNH_ALL = 0x30;
250constexpr uint8_t CMD_OP_ATC_INV = 0x40;
251constexpr uint8_t CMD_OP_PRI_RESP = 0x41;
252constexpr uint8_t CMD_OP_RESUME = 0x44;
253constexpr uint8_t CMD_OP_STALL_TERM = 0x45;
254constexpr uint8_t CMD_OP_SYNC = 0x46;
255
256// SYNC_CS = Completion Signal field of the SYNC command: how the SMMU should notify the driver when the sync is done.
257constexpr uint32_t SYNC_CS_NONE = 0; // no signal (driver will poll the consumer pointer)
258constexpr uint32_t SYNC_CS_IRQ = 1; // raise an interrupt (and optionally an MSI write)
259constexpr uint32_t SYNC_CS_SEV = 2; // SEV = Send EVent (SystemC: not modelled, treated as no-op)
260
261constexpr uint32_t GATOS_PAR_FAULT_TRANSLATION = 0x10; // GATOS PAR fault-status code meaning "translation fault"
262
263constexpr uint32_t IDR0_TTF_AARCH64 = 0x2; // TTF (Translation Table Format) value advertising AArch64 page tables
264constexpr uint32_t IDR0_STLEVEL_2LVL = 0x2; // Stream-table level value advertising the 2-level format
265constexpr uint32_t IDR1_DEFAULT_QUEUE_LOG2 = 19;
266
267// OAS = Output Address Size encoding (3-bit field, expands to a number of physical-address bits).
268constexpr uint32_t OAS_32 = 0;
269constexpr uint32_t OAS_36 = 1;
270constexpr uint32_t OAS_40 = 2;
271constexpr uint32_t OAS_42 = 3;
272constexpr uint32_t OAS_44 = 4;
273constexpr uint32_t OAS_48 = 5;
274
275constexpr uint32_t PA_BITS = 48; // PA = Physical Address: largest physical address width this model supports
276constexpr uint32_t
277 VA_SIGN_BIT_SHIFT = 63; // VA = Virtual Address: bit 63 selects the upper/lower half of the address space
278constexpr uint32_t IPA_PAGE_SHIFT = 12; // page shift used when treating an address as an IPA
279
280constexpr uint32_t INPUT_SIZE_MIN = 25;
281constexpr uint32_t INPUT_SIZE_MAX = 48;
282
283// Translates an OAS/IPS (Intermediate Physical Size) 3-bit code into the actual number of address bits.
284constexpr std::array<uint32_t, 8> OUTPUT_SIZE_MAP = { 32, 36, 40, 42, 44, 48, 48, 48 };
285
286// Twelve bytes of fixed identification that appear in the top-of-page ID registers
287// (vendor/part/revision/Class). These values are consumed by the IDREGS array; they are not arbitrary.
288constexpr std::array<uint32_t, 12> PRIMECELL_IDS = {
289 0x04, 0x00, 0x00, 0x00, 0x34, 0xB0, 0x0B, 0x00, 0x0D, 0xF0, 0x05, 0xB1,
290};
291
292// IOMMU = I/O Memory Management Unit (the role this whole module plays). AccessFlags describe what
293// the translation grants downstream: RO = Read-Only, WO = Write-Only, RW = both, NONE = no access.
294enum class IOMMUAccessFlags : uint8_t {
295 NONE = 0,
296 RO = 1,
297 WO = 2,
298 RW = 3,
299};
300
301constexpr IOMMUAccessFlags operator&(IOMMUAccessFlags a, IOMMUAccessFlags b)
302{
303 return static_cast<IOMMUAccessFlags>(static_cast<uint8_t>(a) & static_cast<uint8_t>(b));
304}
305
306constexpr IOMMUAccessFlags operator|(IOMMUAccessFlags a, IOMMUAccessFlags b)
307{
308 return static_cast<IOMMUAccessFlags>(static_cast<uint8_t>(a) | static_cast<uint8_t>(b));
309}
310
311constexpr IOMMUAccessFlags operator~(IOMMUAccessFlags a)
312{
313 return static_cast<IOMMUAccessFlags>(~static_cast<uint8_t>(a) & 0x3u);
314}
315
316inline IOMMUAccessFlags& operator&=(IOMMUAccessFlags& a, IOMMUAccessFlags b)
317{
318 a = a & b;
319 return a;
320}
321
322inline IOMMUAccessFlags& operator|=(IOMMUAccessFlags& a, IOMMUAccessFlags b)
323{
324 a = a | b;
325 return a;
326}
327
328template <unsigned int BUSWIDTH = 32>
329class smmuv3_tbu;
330
331// SMMUv3 = System Memory Management Unit, version 3. The "core" of the model: holds the
332// register file (smmuv3_gen), translates transactions on behalf of one or more TBUs, and
333// runs the command/event/PRI queues that the OS driver talks to.
334template <unsigned int BUSWIDTH = 32>
335class smmuv3 : public sc_core::sc_module, public smmuv3_gen
336{
337 SCP_LOGGER();
338 cci::cci_broker_handle m_broker;
340 bool m_loaded_ok;
342
343public:
344 // IOMMUTLBEntry = the result of one translation. "TLB" (Translation Lookaside Buffer) is the
345 // historical CPU-side name for the cached-translation structure; here it just means "one mapping".
347 uint64_t iova; // IOVA = I/O Virtual Address: the address the upstream device used
348 uint64_t translated_addr; // resulting physical address (page-aligned)
349 uint64_t addr_mask; // low-bit mask of the page (page_size - 1); used to add the offset back in
350 IOMMUAccessFlags perm; // permissions granted by this translation (RO/WO/RW/NONE)
351 uint64_t descriptor; // raw leaf-level page-table descriptor (kept so memory attributes can be re-derived)
352 uint32_t table_attrs; // accumulated table-walk attributes (PXN/UXN/AP propagated from intermediate levels)
353 uint32_t fault_type; // fault reason code if perm == NONE (one of SMMUV3_EVT_*)
354 uint32_t fault_level; // page-table level at which the fault occurred
355 bool stallable_fault; // true if this fault should be reported as stallable (driver may RESUME)
356 };
357
358 // Configuration parameters (CCI = Configuration & Communication Interface — the SystemC parameter system):
359 cci::cci_param<uint32_t> p_pamax; // PAMAX = max Physical Address bits this instance advertises (32..48)
360 cci::cci_param<uint16_t> p_sidsize; // SIDSIZE = number of StreamID bits this instance accepts
361 cci::cci_param<bool> p_ato; // ATO = Address Translation Operations: enable the GATOS debug channel
362 cci::cci_param<uint8_t> p_num_tbu; // number of Translation Buffer Units (upstream front-ends) to expose
363 cci::cci_param<uint32_t> p_iotlb_size; // IOTLB = I/O TLB: max number of cached translations to keep
364 cci::cci_param<uint32_t> p_iidr; // IIDR = Implementation ID Register value to advertise
365
366 tlm_utils::multi_passthrough_target_socket<smmuv3, BUSWIDTH>
367 socket; // MMIO target: register-file accesses arrive here
368 tlm_utils::simple_initiator_socket<smmuv3>
369 dma_socket; // DMA = Direct Memory Access port; SMMU uses it for table walks and queue accesses
370 InitiatorSignalSocket<bool> irq_eventq; // event-queue interrupt line
371 InitiatorSignalSocket<bool> irq_priq; // page-request-interface interrupt line
372 InitiatorSignalSocket<bool> irq_cmd_sync; // pulse fired when a CMD_SYNC's IRQ-completion-signal triggers
373 InitiatorSignalSocket<bool> irq_gerror; // GERROR (Global Error) interrupt line
374 InitiatorSignalSocket<bool> irq_s_gerror; // Secure-world GERROR line
375 InitiatorSignalSocket<bool> irq_s_eventq; // Secure-world event-queue line
376 TargetSignalSocket<bool> reset; // hardware reset input
377
378 std::vector<smmuv3_tbu<BUSWIDTH>*> tbus; // every TBU registers itself here at construction time
379
380 smmuv3(sc_core::sc_module_name name);
381
382private:
383 // STE = Stream Table Entry: the in-memory record that describes one upstream device.
384 // Decoded into this struct after fetching/parsing it from the stream table.
385 struct STE {
386 bool valid = false; // entry has its V (Valid) bit set
387 uint32_t config = 0; // CONFIG: ABORT/BYPASS/S1/S2/NESTED (see STE_CONFIG_*)
388 uint64_t s1contextptr = 0; // S1ContextPtr = pointer to the Context Descriptor table for this device
389 uint64_t s2ttb = 0; // S2TTB = Stage-2 Translation Table Base (root of the hypervisor page tables)
390 uint32_t s2t0sz = 0; // S2T0SZ = Stage-2 TTBR0 input SiZe (number of high bits ignored on input)
391 uint32_t s2sl0 = 0; // S2SL0 = Stage-2 Start Level for TTBR0 (which level the walk begins at)
392 uint32_t s2tg = 0; // S2TG = Stage-2 Translation Granule (page size of the S2 tables)
393 uint32_t s2ps = 0; // S2PS = Stage-2 Physical-address Size advertised by this STE
394 uint32_t s1fmt = 0; // S1FMT = Stage-1 stream-table format (linear vs 2-level)
395 bool s1stalld = false; // S1STALLD = Stage-1 STALL Disabled (faults are reported as terminating, not stallable)
396 uint32_t s1cdmax = 0; // S1CDMAX = log2 of max number of CDs reachable through this STE
397 uint16_t vmid = 0; // VMID = Virtual Machine ID this device belongs to
398 bool s2affd = false; // S2AFFD = Stage-2 Access-Flag Fault Disable (don't fault on AF=0, ignore the flag)
399 bool s2hd = false; // S2HD = Stage-2 Hardware Dirty management enabled
400 bool eats = false; // EATS = Enable ATS for this device (PCIe Address Translation Service)
401 uint8_t strw = 0; // STRW = STReam World (STE.W1[30:31]): which CPU translation regime this device's
402 // transactions belong to (NS-EL1 / NS-EL2 / EL3). Combined with CR2.E2H and the
403 // Secure tag to disambiguate IOTLB entries that would otherwise alias.
404 };
405
406 // CD = Context Descriptor: per-(StreamID, SubstreamID) record holding the Stage-1 page-table roots.
407 struct CD {
408 bool valid = false;
409 bool aarch64 = false; // selects AArch64 vs AArch32 long-format page tables
410 std::array<uint64_t, 2> ttb{}; // TTB[0/1] = Translation Table Base for low-half / high-half VAs (TTBR0/TTBR1)
411 std::array<uint32_t, 2> tsz{}; // TSZ[0/1] = Translation SiZe: number of high VA bits ignored for each half
412 std::array<uint32_t, 2> tg{}; // TG[0/1] = Translation Granule for each half (page size)
413 uint32_t ips = 0; // IPS = Intermediate Physical-address Size (output width of stage 1)
414 uint16_t asid = 0; // ASID = Address Space ID (tags TLB entries belonging to this address space)
415 std::array<bool, 2> epd{}; // EPD[0/1] = page-table walk Disable for each half (1 = unmapped half)
416 bool affd = false; // AFFD = Access-Flag Fault Disable (stage 1)
417 bool ha = false; // HA = Hardware-managed Access-flag updates allowed
418 bool hd = false; // HD = Hardware-managed Dirty-flag updates allowed
419 };
420
421 // PtwCtx = Page-Table Walk Context: input state bundled up and passed into the walker.
422 struct PtwCtx {
423 std::array<uint64_t, 2> ttb{}; // see CD::ttb
424 std::array<uint32_t, 2> tsz{};
425 std::array<uint32_t, 2> tg{};
426 uint32_t ips = 0;
427 uint32_t sl0 = 0; // SL0 = Start Level 0 (which page-table level to begin walking at; stage-2 only)
428 std::array<bool, 2> epd{};
429 int stage = 0; // 1 = walk stage-1 tables, 2 = walk stage-2 tables
430 uint64_t iova = 0; // input address for this walk (an IOVA for stage-1, an IPA for stage-2)
431 IOMMUAccessFlags access = IOMMUAccessFlags::NONE; // intended access (used for permission checks)
432 bool s2_enabled = false; // stage-1 walk may need each table fetch to be S2-translated first (nested)
433 STE* ste = nullptr; // pointer back to the originating STE (needed when nested)
434 bool affd = false;
435 bool ha = false;
436 bool hd = false;
437 };
438
439 // TransReq = TRANSlation REQuest: scratch state carried through the translation pipeline. Holds inputs
440 // (iova, access, sid, ssid, ste, cd) and is filled in as the walk progresses (pa, prot, fault info, ...).
441 struct TransReq {
442 uint64_t iova = 0; // input I/O Virtual Address
443 IOMMUAccessFlags access = IOMMUAccessFlags::NONE; // requested access (read/write)
444 uint32_t sid = 0; // SID = StreamID (which device originated the transaction)
445 uint32_t substream_id = 0; // sub-context within that device
446 STE ste{}; // resolved Stream Table Entry
447 CD cd{}; // resolved Context Descriptor (for stage-1)
448 uint64_t pa = 0; // resulting Physical Address
449 IOMMUAccessFlags prot = IOMMUAccessFlags::NONE; // permissions granted by the walk
450 uint64_t page_size = 0; // log2 of resulting page size
451 bool err = false; // walk produced a fault
452 uint32_t event_type = 0; // fault type to report (one of SMMUV3_EVT_*)
453 uint32_t tableattrs = 0; // accumulated TABLE-level attributes from intermediate descriptors
454 int fault_level = 0; // page-table level at which the fault occurred
455 uint64_t descriptor = 0; // raw leaf descriptor word
456 uint64_t leaf_desc_addr = 0; // memory address of the leaf descriptor (used for HW AF/DBM updates)
457 };
458
459 // IOTLBKey = lookup key for the I/O TLB cache. Identifies one cached translation by
460 // (vmid, asid, iova-page, granule, level, secure, regime). The "regime" byte folds in the
461 // CPU translation regime (StreamWorld + CR2.E2H) so that two STEs with the same VMID/ASID/IOVA
462 // but different regimes (e.g. NS-EL1 vs NS-EL2-E2H) do not alias.
463 struct IOTLBKey {
464 uint16_t vmid; // VMID = Virtual Machine ID
465 uint16_t asid; // ASID = Address Space ID
466 uint64_t iova; // IOVA, masked to the page boundary
467 uint8_t tg; // TG = Translation Granule (page size of the cached entry)
468 uint8_t level; // page-table level at which the leaf was found
469 bool secure; // true = Secure-world entry, isolated from Non-Secure lookups
470 uint8_t regime; // CPU translation regime (see compute_iotlb_regime()): EL1 vs EL2-classic vs EL2-E2H
471
472 bool operator==(const IOTLBKey& o) const
473 {
474 return vmid == o.vmid && asid == o.asid && iova == o.iova && tg == o.tg && level == o.level &&
475 secure == o.secure && regime == o.regime;
476 }
477 };
478
479 struct IOTLBKeyHash {
480 static constexpr uint32_t VMID_HASH_SHIFT = 48;
481 static constexpr uint32_t ASID_HASH_SHIFT = 32;
482
483 static constexpr uint64_t SECURE_HASH_SALT = 0x9E3779B97F4A7C15ULL;
484 static constexpr uint64_t REGIME_HASH_MULT = 0xBF58476D1CE4E5B9ULL; // mixes the regime byte into the hash
485 std::size_t operator()(const IOTLBKey& k) const
486 {
487 uint64_t base = ((uint64_t)k.vmid << VMID_HASH_SHIFT) | ((uint64_t)k.asid << ASID_HASH_SHIFT) | k.iova;
488 if (k.secure) base ^= SECURE_HASH_SALT;
489 base ^= static_cast<uint64_t>(k.regime) * REGIME_HASH_MULT;
490 return std::hash<uint64_t>()(base);
491 }
492 };
493
494 std::unordered_map<IOTLBKey, IOMMUTLBEntry, IOTLBKeyHash> m_iotlb;
495 std::list<IOTLBKey> m_iotlb_lru; // LRU = Least Recently Used: ordering used to pick a victim when the IOTLB is full
496
497 // StalledTxn = a TLM transaction (Txn) parked here while a stallable fault is outstanding;
498 // it stays here until the driver issues a RESUME command quoting our stag (Stall Tag).
499 struct StalledTxn {
500 tlm::tlm_generic_payload txn; // the original transaction object we're holding
501 sc_core::sc_time delay;
502 uint32_t stag; // STAG = Stall TAG that uniquely identifies this stalled txn
503 uint32_t sid; // originating StreamID
504 uint32_t substream_id;
505 sc_core::sc_event resume_event; // signalled when the driver responds (resume or abort)
506 bool aborted; // set by the driver via the resume command if the txn must be killed
507 };
508 std::unordered_map<uint32_t, StalledTxn> m_stalled_txns;
509 uint32_t m_next_stag; // monotonic stag generator
510
511 // Local caches for previously-decoded STEs/CDs and previously-granted DMI regions.
512 // DMI = Direct Memory Interface (TLM's fast-path mechanism for grabbing a raw pointer into a target).
513 static constexpr size_t STE_CACHE_MAX = 1024;
514 static constexpr size_t CD_CACHE_MAX = 1024;
515 static constexpr size_t DMA_DMI_CACHE_MAX = 64;
516
517 std::unordered_map<uint32_t, STE> m_ste_cache;
518 std::list<uint32_t> m_ste_cache_lru;
519 std::unordered_map<uint64_t, CD> m_cd_cache;
520 std::list<uint64_t> m_cd_cache_lru;
521
522 std::unordered_map<uint32_t, STE> m_ste_cache_secure;
523 std::unordered_map<uint64_t, CD> m_cd_cache_secure;
524
525 bool m_prev_gerror_pending = false;
526 bool m_prev_eventq_pending = false;
527 bool m_prev_priq_pending = false;
528 bool m_prev_s_gerror_pending = false;
529 bool m_prev_s_eventq_pending = false;
530 bool m_last_irq_gerror_level = false;
531 bool m_last_irq_eventq_level = false;
532 bool m_last_irq_priq_level = false;
533 bool m_last_irq_s_gerror_level = false;
534 bool m_last_irq_s_eventq_level = false;
535
536 static uint32_t extract32(uint32_t val, int start, int length) { return (val >> start) & ((1u << length) - 1); }
537
538 static uint64_t extract64(uint64_t val, int start, int length) { return (val >> start) & ((1ULL << length) - 1); }
539
540 // compute_iotlb_regime: collapse (STE.STRW, CR2.E2H, secure) into one byte used to tag IOTLB entries.
541 // The four reachable regimes get distinct codes so cached translations cannot alias across them:
542 // IOTLB_REGIME_NS_EL1 = NS-EL1 traffic (typical guest/host stage-1)
543 // IOTLB_REGIME_NS_EL2 = NS-EL2 classic hypervisor traffic (CR2.E2H=0)
544 // IOTLB_REGIME_NS_EL2_E2H= NS-EL2-Host (VHE) traffic (CR2.E2H=1)
545 // IOTLB_REGIME_SECURE = any Secure-world traffic; the "secure" flag in the key still applies
546 static constexpr uint8_t IOTLB_REGIME_NS_EL1 = 0;
547 static constexpr uint8_t IOTLB_REGIME_NS_EL2 = 1;
548 static constexpr uint8_t IOTLB_REGIME_NS_EL2_E2H = 2;
549 static constexpr uint8_t IOTLB_REGIME_SECURE = 3;
550 static uint8_t compute_iotlb_regime(uint8_t strw, bool e2h, bool is_secure)
551 {
552 if (is_secure) return IOTLB_REGIME_SECURE;
553 // STRW raw encoding: 0 = NS-EL1, 2 = NS-EL2; other values are reserved/secure-only in the spec.
554 if ((strw & 0x3) == 0x2) return e2h ? IOTLB_REGIME_NS_EL2_E2H : IOTLB_REGIME_NS_EL2;
555 return IOTLB_REGIME_NS_EL1;
556 }
557
558 bool check_s2_startlevel(unsigned int pamax_val, int level, int inputsize, int stride)
559 {
560 if (level < 0) return false;
561 switch (stride) {
562 case STRIDE_64K:
563 if (level == 0 || (level == 1 && pamax_val <= 42)) return false;
564 break;
565 case STRIDE_16K:
566 if (level == 0 || (level == 1 && pamax_val <= 40)) return false;
567 break;
568 case STRIDE_4K:
569 if (level == 0 && pamax_val <= 42) return false;
570 break;
571 default:
572 return false;
573 }
574 return true;
575 }
576
577 bool check_out_addr(uint64_t addr, unsigned int outputsize)
578 {
579 if (outputsize >= PA_BITS) return true;
580 return extract64(addr, outputsize, PA_BITS - outputsize) == 0;
581 }
582
583 // One cached DMI (Direct Memory Interface) grant: the SystemC target told us we may read/write
584 // [start..end] directly via "ptr" instead of issuing a TLM transaction every time. Used to
585 // accelerate the SMMU's own page-table walks and queue accesses.
586 struct DmaDmiRegion {
587 sc_dt::uint64 start;
588 sc_dt::uint64 end;
589 unsigned char* ptr;
590 bool read_allowed;
591 bool write_allowed;
592 };
593 std::vector<DmaDmiRegion> m_dma_dmi_cache;
594
595 const DmaDmiRegion* find_dma_dmi(uint64_t addr, size_t len, bool write) const
596 {
597 if (len == 0) return nullptr;
598 sc_dt::uint64 last = addr + len - 1;
599 for (const auto& r : m_dma_dmi_cache) {
600 if (addr >= r.start && last <= r.end) {
601 if ((write && r.write_allowed) || (!write && r.read_allowed)) {
602 return &r;
603 }
604 }
605 }
606 return nullptr;
607 }
608
609 bool grant_dma_dmi(uint64_t addr, tlm::tlm_command cmd)
610 {
611 tlm::tlm_generic_payload probe;
612 probe.set_command(cmd);
613 probe.set_address(addr);
614 uint8_t scratch = 0;
615 probe.set_data_ptr(&scratch);
616 probe.set_data_length(1);
617 probe.set_streaming_width(1);
618 probe.set_byte_enable_length(0);
619 probe.set_dmi_allowed(true);
620 probe.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
621 tlm::tlm_dmi dmi;
622 if (!dma_socket->get_direct_mem_ptr(probe, dmi)) {
623 return false;
624 }
625 if (!dmi.is_read_allowed() && !dmi.is_write_allowed()) {
626 return false;
627 }
628 DmaDmiRegion r;
629 r.start = dmi.get_start_address();
630 r.end = dmi.get_end_address();
631 r.ptr = dmi.get_dmi_ptr();
632 r.read_allowed = dmi.is_read_allowed();
633 r.write_allowed = dmi.is_write_allowed();
634
635 if (m_dma_dmi_cache.size() >= DMA_DMI_CACHE_MAX) {
636 m_dma_dmi_cache.erase(m_dma_dmi_cache.begin());
637 }
638 m_dma_dmi_cache.push_back(r);
639 return true;
640 }
641
642 void dma_dmi_invalidate(sc_dt::uint64 start, sc_dt::uint64 end)
643 {
644 m_dma_dmi_cache.erase(
645 std::remove_if(m_dma_dmi_cache.begin(), m_dma_dmi_cache.end(),
646 [start, end](const DmaDmiRegion& r) { return !(r.end < start || r.start > end); }),
647 m_dma_dmi_cache.end());
648 }
649
650 bool dma_read(uint64_t addr, void* data, size_t len)
651 {
652 const DmaDmiRegion* r = find_dma_dmi(addr, len, false);
653 if (!r && grant_dma_dmi(addr, tlm::TLM_READ_COMMAND)) {
654 r = find_dma_dmi(addr, len, false);
655 }
656 if (r) {
657 std::memcpy(data, r->ptr + (addr - r->start), len);
658 return true;
659 }
660 tlm::tlm_generic_payload txn;
661 txn.set_command(tlm::TLM_READ_COMMAND);
662 txn.set_address(addr);
663 txn.set_data_ptr(reinterpret_cast<unsigned char*>(data));
664 txn.set_data_length(len);
665 txn.set_streaming_width(len);
666 txn.set_byte_enable_length(0);
667 txn.set_dmi_allowed(false);
668 txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
669 sc_core::sc_time delay = sc_core::SC_ZERO_TIME;
670 dma_socket->b_transport(txn, delay);
671 return txn.get_response_status() == tlm::TLM_OK_RESPONSE;
672 }
673
674 bool dma_write(uint64_t addr, const void* data, size_t len)
675 {
676 const DmaDmiRegion* r = find_dma_dmi(addr, len, true);
677 if (!r && grant_dma_dmi(addr, tlm::TLM_WRITE_COMMAND)) {
678 r = find_dma_dmi(addr, len, true);
679 }
680 if (r) {
681 std::memcpy(r->ptr + (addr - r->start), data, len);
682 return true;
683 }
684 tlm::tlm_generic_payload txn;
685 txn.set_command(tlm::TLM_WRITE_COMMAND);
686 txn.set_address(addr);
687 txn.set_data_ptr(const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(data)));
688 txn.set_data_length(len);
689 txn.set_streaming_width(len);
690 txn.set_byte_enable_length(0);
691 txn.set_dmi_allowed(false);
692 txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
693 sc_core::sc_time delay = sc_core::SC_ZERO_TIME;
694 dma_socket->b_transport(txn, delay);
695 return txn.get_response_status() == tlm::TLM_OK_RESPONSE;
696 }
697
698 bool find_ste(uint32_t sid, STE& ste); // walks the stream table and decodes the STE for a StreamID
699 bool read_cd(uint32_t sid, const STE& ste, uint32_t substream_id,
700 CD& cd); // fetches and decodes the Context Descriptor
701 bool smmuv3_ptw64(PtwCtx& ctx, TransReq& req); // PTW64 = 64-bit Page-Table Walker (the heart of translation)
702 IOMMUTLBEntry smmuv3_translate(tlm::tlm_generic_payload& txn, uint32_t sid, uint32_t substream_id,
703 bool is_secure = false,
704 bool is_ats_tr = false); // is_ats_tr = is this an ATS Translation request
705
706 // IOTLB management: insert/lookup, plus the invalidation helpers used by TLBI commands.
707 void iotlb_insert(const IOTLBKey& key, const IOMMUTLBEntry& entry);
708 bool iotlb_lookup(const IOTLBKey& key, IOMMUTLBEntry& entry);
709 void iotlb_inv_all(); // INV = INValidate
710 void iotlb_inv_asid(uint16_t asid); // drop entries matching one ASID
711 void iotlb_inv_vmid(uint16_t vmid); // drop entries matching one VMID
712 void iotlb_inv_iova(uint16_t vmid, uint16_t asid, uint64_t iova, uint8_t tg,
713 uint64_t addr_mask); // drop one IOVA range
714
715 void consume_cmdq(); // drain pending entries from the (non-secure) command queue
716 void consume_secure_cmdq(); // same for the Secure-world command queue
717 void record_event(uint32_t type, uint32_t sid, uint64_t iova,
718 uint32_t info); // append a simple entry to the event queue
719 void record_event_full(uint32_t type, uint32_t sid, uint32_t ssid, uint64_t iova, tlm::tlm_generic_payload& txn,
720 uint32_t class_, uint32_t fault_level, uint32_t stag, bool stall); // ssid = SubstreamID
721 void record_secure_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info);
722 void record_pri(uint32_t sid, uint64_t iova, uint32_t flags); // PRI = Page Request Interface entry
723 bool queue_full(uint32_t prod, uint32_t cons, uint32_t size); // PROD/CONS = producer/consumer pointers
724
725 // Per-opcode command-queue handlers; naming mirrors the CMD_OP_* opcodes above.
726 void handle_cmd_prefetch_config(const uint8_t* cmd);
727 void handle_cmd_prefetch_addr(const uint8_t* cmd);
728 void handle_cmd_cfgi_ste(const uint8_t* cmd); // CFGI = ConFiGuration Invalidate; STE = Stream Table Entry
729 void handle_cmd_cfgi_ste_range(const uint8_t* cmd);
730 void handle_cmd_cfgi_cd(const uint8_t* cmd); // CD = Context Descriptor
731 void handle_cmd_cfgi_cd_all(const uint8_t* cmd);
732 void handle_cmd_cfgi_all(const uint8_t* cmd);
733 void handle_cmd_tlbi_nh_all(const uint8_t* cmd); // TLBI = TLB Invalidate; NH = Non-Hyp scope
734 void handle_cmd_tlbi_nh_asid(const uint8_t* cmd);
735 void handle_cmd_tlbi_nh_va(const uint8_t* cmd); // VA = by Virtual Address
736 void handle_cmd_tlbi_nh_vaa(const uint8_t* cmd); // VAA = by VA, All ASIDs
737 void handle_cmd_atc_inv(const uint8_t* cmd); // ATC_INV = Address-Translation-Cache Invalidate (ATS clients)
738 void handle_cmd_tlbi_s12_vmall(const uint8_t* cmd); // S12 = stages 1+2; VMALL = all entries for one VMID
739 void handle_cmd_tlbi_s2_ipa(const uint8_t* cmd); // S2 by IPA = Intermediate Physical Address
740 void handle_cmd_tlbi_nsnh_all(const uint8_t* cmd); // NSNH = Non-Secure Non-Hyp
741 void handle_cmd_sync(const uint8_t* cmd); // SYNC = command-queue barrier
742 void handle_cmd_resume(const uint8_t* cmd); // release a stalled transaction quoted by its stag
743 void handle_cmd_pri_resp(const uint8_t* cmd); // PRI_RESP = driver's response to a queued page request
744
745 // stall_and_wait: park a translation-faulted transaction and block until the driver answers (RESUME/abort).
746 // class_ = SMMUV3_CLASS_* (which subsystem caused the fault), reported back in the event record.
747 bool stall_and_wait(uint32_t sid, uint32_t ssid, uint64_t iova, uint32_t fault_type, uint32_t fault_level,
748 uint32_t class_, tlm::tlm_generic_payload& txn);
749
750 void do_gatos(); // run one Global Address Translation Operation (debug translate-this-VA channel; see GATOS_*)
751
752 // Pulse a wired interrupt line: high then low (edge-triggered receivers see one event).
753 void trigger_irq(InitiatorSignalSocket<bool>& irq)
754 {
755 if (irq.size() > 0) {
756 irq->write(true);
757 irq->write(false);
758 }
759 }
760
761 // Set one bit in GERROR (Global Error). The driver later acknowledges by writing the same bit
762 // into GERRORN; until then the bit position remains "live" and edge-triggers no further IRQs.
763 void set_gerror(uint32_t bit)
764 {
765 uint32_t err = static_cast<uint32_t>(GERROR);
766 uint32_t ack = static_cast<uint32_t>(GERRORN);
767 if (((err ^ ack) >> bit) & 1u) {
768 return;
769 }
770 GERROR = err ^ (1u << bit);
771 update_irq_levels();
772 }
773
774 void set_secure_gerror(uint32_t bit)
775 {
776 uint32_t err = static_cast<uint32_t>(S_GERROR);
777 uint32_t ack = static_cast<uint32_t>(S_GERRORN);
778 if (((err ^ ack) >> bit) & 1u) {
779 return;
780 }
781 S_GERROR = err ^ (1u << bit);
782 update_secure_irq_levels();
783 }
784
785 void set_cmdq_err(uint8_t cerror)
786 {
787 CMDQ_CONS[CMDQ_CONS_ERR] = cerror;
788 set_gerror(GERROR_BIT_CMDQ_ERR);
789 }
790
791 void set_secure_cmdq_err(uint8_t cerror)
792 {
793 S_CMDQ_CONS[S_CMDQ_CONS_ERR] = cerror;
794 set_secure_gerror(GERROR_BIT_CMDQ_ERR);
795 }
796
797 void drive_irq_level(InitiatorSignalSocket<bool>& irq, bool& last_level, bool new_level)
798 {
799 if (last_level == new_level) return;
800 last_level = new_level;
801 if (irq.size() > 0) {
802 irq->write(new_level);
803 }
804 }
805
806 // Send one Message-Signalled Interrupt: a 32-bit "data" word is DMA-written to the address built
807 // from cfg0_hi:cfg0_lo. If the write fails, set the matching abort bit in GERROR.
808 void emit_msi(uint32_t cfg0_lo, uint32_t cfg0_hi, uint32_t data, uint32_t abort_bit_on_fail)
809 {
810 uint64_t addr = (static_cast<uint64_t>(cfg0_hi) << 32) | static_cast<uint64_t>(cfg0_lo);
811 if (addr == 0) return;
812 uint32_t payload = data;
813 if (!dma_write(addr, &payload, sizeof(payload))) {
814 set_gerror(abort_bit_on_fail);
815 }
816 }
817
818 void update_irq_levels()
819 {
820 bool gerror_pending = (static_cast<uint32_t>(GERROR) ^ static_cast<uint32_t>(GERRORN)) != 0;
821 bool eventq_pending = (static_cast<uint32_t>(EVENTQ_PROD) != static_cast<uint32_t>(EVENTQ_CONS));
822 bool priq_pending = (static_cast<uint32_t>(PRIQ_PROD) != static_cast<uint32_t>(PRIQ_CONS));
823
824 bool gerror_irq = gerror_pending && static_cast<uint32_t>(IRQ_CTRL[IRQ_CTRL_GERROR_IRQEN]);
825 bool eventq_irq = eventq_pending && static_cast<uint32_t>(IRQ_CTRL[IRQ_CTRL_EVENTQ_IRQEN]);
826 bool priq_irq = priq_pending && static_cast<uint32_t>(IRQ_CTRL[IRQ_CTRL_PRI_IRQEN]);
827
828 if (gerror_pending && !m_prev_gerror_pending && static_cast<uint32_t>(IRQ_CTRL[IRQ_CTRL_GERROR_IRQEN])) {
829 emit_msi(static_cast<uint32_t>(GERROR_IRQ_CFG0_LO), static_cast<uint32_t>(GERROR_IRQ_CFG0_HI),
830 static_cast<uint32_t>(GERROR_IRQ_CFG2), GERROR_BIT_MSI_GERROR_ABT);
831 }
832 if (eventq_pending && !m_prev_eventq_pending && static_cast<uint32_t>(IRQ_CTRL[IRQ_CTRL_EVENTQ_IRQEN])) {
833 emit_msi(static_cast<uint32_t>(EVENTQ_IRQ_CFG0_LO), static_cast<uint32_t>(EVENTQ_IRQ_CFG0_HI),
834 static_cast<uint32_t>(EVENTQ_IRQ_CFG2), GERROR_BIT_MSI_EVENTQ_ABT);
835 }
836 if (priq_pending && !m_prev_priq_pending && static_cast<uint32_t>(IRQ_CTRL[IRQ_CTRL_PRI_IRQEN])) {
837 emit_msi(static_cast<uint32_t>(PRIQ_IRQ_CFG0_LO), static_cast<uint32_t>(PRIQ_IRQ_CFG0_HI),
838 static_cast<uint32_t>(PRIQ_IRQ_CFG2), GERROR_BIT_MSI_PRIQ_ABT);
839 }
840
841 m_prev_gerror_pending = gerror_pending;
842 m_prev_eventq_pending = eventq_pending;
843 m_prev_priq_pending = priq_pending;
844
845 drive_irq_level(irq_gerror, m_last_irq_gerror_level, gerror_irq);
846 drive_irq_level(irq_eventq, m_last_irq_eventq_level, eventq_irq);
847 drive_irq_level(irq_priq, m_last_irq_priq_level, priq_irq);
848 }
849
850 void update_secure_irq_levels()
851 {
852 bool gerror_pending = (static_cast<uint32_t>(S_GERROR) ^ static_cast<uint32_t>(S_GERRORN)) != 0;
853 bool eventq_pending = (static_cast<uint32_t>(S_EVENTQ_PROD) != static_cast<uint32_t>(S_EVENTQ_CONS));
854
855 bool gerror_irq = gerror_pending && static_cast<uint32_t>(S_IRQ_CTRL[S_IRQ_CTRL_GERROR_IRQEN]);
856 bool eventq_irq = eventq_pending && static_cast<uint32_t>(S_IRQ_CTRL[S_IRQ_CTRL_EVENTQ_IRQEN]);
857
858 if (gerror_pending && !m_prev_s_gerror_pending && static_cast<uint32_t>(S_IRQ_CTRL[S_IRQ_CTRL_GERROR_IRQEN])) {
859 emit_msi(static_cast<uint32_t>(S_GERROR_IRQ_CFG0_LO), static_cast<uint32_t>(S_GERROR_IRQ_CFG0_HI),
860 static_cast<uint32_t>(S_GERROR_IRQ_CFG2), GERROR_BIT_MSI_GERROR_ABT);
861 }
862 if (eventq_pending && !m_prev_s_eventq_pending && static_cast<uint32_t>(S_IRQ_CTRL[S_IRQ_CTRL_EVENTQ_IRQEN])) {
863 emit_msi(static_cast<uint32_t>(S_EVENTQ_IRQ_CFG0_LO), static_cast<uint32_t>(S_EVENTQ_IRQ_CFG0_HI),
864 static_cast<uint32_t>(S_EVENTQ_IRQ_CFG2), GERROR_BIT_MSI_EVENTQ_ABT);
865 }
866
867 m_prev_s_gerror_pending = gerror_pending;
868 m_prev_s_eventq_pending = eventq_pending;
869
870 drive_irq_level(irq_s_gerror, m_last_irq_s_gerror_level, gerror_irq);
871 drive_irq_level(irq_s_eventq, m_last_irq_s_eventq_level, eventq_irq);
872 }
873
874public:
875 friend class smmuv3_tbu<BUSWIDTH>;
876
877 void start_of_simulation() override;
878 void before_end_of_elaboration() override;
879
880 void test_inject_pri(uint32_t sid, uint64_t iova, uint32_t flags) { record_pri(sid, iova, flags); }
881 void test_inject_secure_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info)
882 {
883 record_secure_event(type, sid, iova, info);
884 }
885 void test_inject_stall(uint32_t stag)
886 {
887 auto& s = m_stalled_txns[stag];
888 s.stag = stag;
889 s.aborted = false;
890 }
891 bool test_stall_present(uint32_t stag) const { return m_stalled_txns.count(stag) != 0; }
892 bool test_stall_aborted(uint32_t stag) const
893 {
894 auto it = m_stalled_txns.find(stag);
895 return it != m_stalled_txns.end() && it->second.aborted;
896 }
897 size_t test_iotlb_size() const { return m_iotlb.size(); }
898 size_t test_ste_cache_size() const { return m_ste_cache.size(); }
899 size_t test_cd_cache_size() const { return m_cd_cache.size(); }
900};
901
902// TBU = Translation Buffer Unit: per-master front-end. One TBU sits between an upstream device's
903// transaction port and the SMMU core; it forwards each transaction through smmuv3_translate() and
904// then on to the downstream interconnect with the translated address.
905template <unsigned int BUSWIDTH>
906class smmuv3_tbu : public sc_core::sc_module
907{
908 SCP_LOGGER();
909 smmuv3<BUSWIDTH>* m_smmu;
910
911 // Snapshot describing one IOVA-page projection of the downstream memory map (used when serving DMI).
912 struct MemoryView {
913 uint64_t address;
914 uint64_t page_start;
915 uint64_t page_end;
916 uint64_t page_size;
917 };
918
919 static uint32_t extract_substream_id(tlm::tlm_generic_payload& txn)
920 {
921 auto* ssx = txn.get_extension<smmuv3_ss_extension>();
922 return (ssx && ssx->ssv) ? ssx->substream_id : 0;
923 }
924
925 static bool extract_secure(tlm::tlm_generic_payload& txn)
926 {
927 auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
928 return secure && secure->value;
929 }
930
931public:
932 void upstream_invalidate(sc_dt::uint64 start, sc_dt::uint64 end)
933 {
934 if (upstream_socket.size() == 0) return;
935 upstream_socket->invalidate_direct_mem_ptr(start, end);
936 }
937
938protected:
939 void b_transport(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay)
940 {
941 sc_dt::uint64 addr = txn.get_address();
942 tlm::tlm_command cmd = txn.get_command();
943
944 auto* ats = txn.get_extension<smmuv3_ats_extension>();
945 uint32_t substream_id = extract_substream_id(txn);
946 bool is_secure = extract_secure(txn);
947
948 if (ats && ats->is_translated) {
949 downstream_socket->b_transport(txn, delay);
950 return;
951 }
952
953 typename smmuv3<BUSWIDTH>::IOMMUTLBEntry te = m_smmu->smmuv3_translate(
954 txn, p_topology_id, substream_id, is_secure, ats && ats->is_translation_request);
955
956 while (te.stallable_fault) {
957 bool aborted = m_smmu->stall_and_wait(p_topology_id, substream_id, addr, te.fault_type, te.fault_level, 0,
958 txn);
959 if (aborted) {
960 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
961 return;
962 }
963 te = m_smmu->smmuv3_translate(txn, p_topology_id, substream_id, is_secure,
964 ats && ats->is_translation_request);
965 }
966
967 if (ats && ats->is_translation_request) {
968 if (te.perm == IOMMUAccessFlags::NONE) {
969 ats->faulted = true;
970 ats->fault_type = te.fault_type;
971 ats->pa = 0;
972 ats->granule_log2 = 0;
973 ats->prot = 0;
974 if (static_cast<uint32_t>(m_smmu->CR0[m_smmu->CR0_PRIQEN]) &&
975 static_cast<uint32_t>(m_smmu->IDR0[m_smmu->IDR0_PRI])) {
976 uint32_t flags = (cmd == tlm::TLM_WRITE_COMMAND ? 0x2u : 0x0u) | (ats->prg_index << 16);
977 m_smmu->record_pri(p_topology_id, addr, flags);
978 }
979 } else {
980 ats->faulted = false;
981 ats->pa = te.translated_addr | (addr & te.addr_mask);
982 uint64_t m = te.addr_mask + 1;
983 uint8_t g = 0;
984 while (m > 1) {
985 m >>= 1;
986 ++g;
987 }
988 ats->granule_log2 = g;
989 ats->prot = static_cast<uint8_t>(te.perm);
990 }
991 txn.set_response_status(tlm::TLM_OK_RESPONSE);
992 return;
993 }
994
995 if (te.perm == IOMMUAccessFlags::NONE || (cmd == tlm::TLM_WRITE_COMMAND && te.perm == IOMMUAccessFlags::RO) ||
996 (cmd == tlm::TLM_READ_COMMAND && te.perm == IOMMUAccessFlags::WO)) {
997 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
998 } else {
999 txn.set_address(te.translated_addr | (addr & SMMUV3_PAGEMASK));
1000
1001 auto ext = std::make_unique<smmuv3_memory_attrs_extension>();
1002 ext->set_from_descriptor(te.descriptor, te.table_attrs);
1003 txn.set_extension(ext.get());
1004
1005 SCP_INFO(()) << std::hex << "smmuv3 TBU b_transport: translate 0x" << addr << " to 0x"
1006 << (te.translated_addr | (addr & SMMUV3_PAGEMASK)) << " attrs=" << ext->to_string();
1007
1008 downstream_socket->b_transport(txn, delay);
1009 txn.set_address(addr);
1010 txn.clear_extension<smmuv3_memory_attrs_extension>();
1011 }
1012 }
1013
1014 virtual unsigned int transport_dbg(tlm::tlm_generic_payload& txn)
1015 {
1016 sc_dt::uint64 addr = txn.get_address();
1017 uint32_t substream_id = extract_substream_id(txn);
1018 bool is_secure = extract_secure(txn);
1019 typename smmuv3<BUSWIDTH>::IOMMUTLBEntry te = m_smmu->smmuv3_translate(txn, p_topology_id, substream_id,
1020 is_secure, false);
1021 txn.set_address(te.translated_addr | (addr & SMMUV3_PAGEMASK));
1022 int ret = downstream_socket->transport_dbg(txn);
1023 txn.set_address(addr);
1024 return ret;
1025 }
1026
1027 virtual bool get_direct_mem_ptr(tlm::tlm_generic_payload& txn, tlm::tlm_dmi& dmi_data)
1028 {
1029 sc_dt::uint64 iova = txn.get_address();
1030 uint32_t substream_id = extract_substream_id(txn);
1031 bool is_secure = extract_secure(txn);
1032 typename smmuv3<BUSWIDTH>::IOMMUTLBEntry te = m_smmu->smmuv3_translate(txn, p_topology_id, substream_id,
1033 is_secure, false);
1034
1035 if (te.perm == IOMMUAccessFlags::NONE) {
1036 txn.set_address(iova);
1037 dmi_data.allow_none();
1038 dmi_data.set_start_address(iova);
1039 dmi_data.set_end_address(iova);
1040 return false;
1041 }
1042
1043 tlm::tlm_command cmd = txn.get_command();
1044 if ((cmd == tlm::TLM_WRITE_COMMAND && te.perm == IOMMUAccessFlags::RO) ||
1045 (cmd == tlm::TLM_READ_COMMAND && te.perm == IOMMUAccessFlags::WO)) {
1046 txn.set_address(iova);
1047 dmi_data.allow_none();
1048 dmi_data.set_start_address(iova);
1049 dmi_data.set_end_address(iova);
1050 return false;
1051 }
1052
1053 sc_dt::uint64 offset_in_page = iova & te.addr_mask;
1054 sc_dt::uint64 pa = te.translated_addr | offset_in_page;
1055 txn.set_address(pa);
1056 bool ok = downstream_socket->get_direct_mem_ptr(txn, dmi_data);
1057 txn.set_address(iova);
1058 if (!ok) return false;
1059
1060 sc_dt::uint64 pa_page_start = te.translated_addr;
1061 sc_dt::uint64 pa_page_end = te.translated_addr | te.addr_mask;
1062 sc_dt::uint64 dmi_start = dmi_data.get_start_address();
1063 sc_dt::uint64 dmi_end = dmi_data.get_end_address();
1064
1065 sc_dt::uint64 clamp_pa_start = pa_page_start > dmi_start ? pa_page_start : dmi_start;
1066 sc_dt::uint64 clamp_pa_end = pa_page_end < dmi_end ? pa_page_end : dmi_end;
1067 if (clamp_pa_end < clamp_pa_start) return false;
1068
1069 sc_dt::uint64 iova_page_start = iova & ~te.addr_mask;
1070 sc_dt::uint64 new_iova_start = iova_page_start + (clamp_pa_start - pa_page_start);
1071 sc_dt::uint64 new_iova_end = iova_page_start + (clamp_pa_end - pa_page_start);
1072
1073 unsigned char* new_dmi_ptr = dmi_data.get_dmi_ptr() + (clamp_pa_start - dmi_start);
1074
1075 dmi_data.set_dmi_ptr(new_dmi_ptr);
1076 dmi_data.set_start_address(new_iova_start);
1077 dmi_data.set_end_address(new_iova_end);
1078
1079 if (te.perm == IOMMUAccessFlags::RO) {
1080 dmi_data.allow_read();
1081 } else if (te.perm == IOMMUAccessFlags::WO) {
1082 dmi_data.allow_write();
1083 }
1084 return true;
1085 }
1086
1087 virtual void invalidate_direct_mem_ptr(sc_dt::uint64 start, sc_dt::uint64 end)
1088 {
1089 if (!m_smmu) return;
1090 for (const auto& e : m_smmu->m_iotlb) {
1091 const auto& entry = e.second;
1092 sc_dt::uint64 pa_start = entry.translated_addr;
1093 sc_dt::uint64 pa_end = entry.translated_addr | entry.addr_mask;
1094 if (pa_start > end || pa_end < start) continue;
1095 sc_dt::uint64 iova_start = entry.iova;
1096 sc_dt::uint64 iova_end = entry.iova | entry.addr_mask;
1097 upstream_socket->invalidate_direct_mem_ptr(iova_start, iova_end);
1098 }
1099 }
1100
1101public:
1102 // p_topology_id is the StreamID this TBU stamps onto every incoming transaction (i.e. "I am device N").
1103 cci::cci_param<uint32_t> p_topology_id;
1104 tlm_utils::simple_target_socket<smmuv3_tbu> upstream_socket; // accepts transactions from the master
1105 tlm_utils::simple_initiator_socket<smmuv3_tbu>
1106 downstream_socket; // forwards translated transactions to the interconnect
1107
1108 smmuv3_tbu(const sc_core::sc_module_name& name, sc_core::sc_object* o)
1109 : smmuv3_tbu(name, dynamic_cast<smmuv3<BUSWIDTH>*>(o))
1110 {
1111 }
1112
1113 smmuv3_tbu(sc_core::sc_module_name name, smmuv3<BUSWIDTH>* smmu)
1114 : sc_core::sc_module(name)
1115 , m_smmu(smmu)
1116 , p_topology_id("topology_id", 0)
1117 , upstream_socket("upstream_socket")
1118 , downstream_socket("downstream_socket")
1119 {
1120 if (m_smmu) {
1121 m_smmu->tbus.push_back(this);
1122 }
1123
1124 upstream_socket.register_b_transport(this, &smmuv3_tbu::b_transport);
1125 upstream_socket.register_transport_dbg(this, &smmuv3_tbu::transport_dbg);
1126 upstream_socket.register_get_direct_mem_ptr(this, &smmuv3_tbu::get_direct_mem_ptr);
1127 downstream_socket.register_invalidate_direct_mem_ptr(this, &smmuv3_tbu::invalidate_direct_mem_ptr);
1128 }
1129};
1130
1131template <unsigned int BUSWIDTH>
1132smmuv3<BUSWIDTH>::smmuv3(sc_core::sc_module_name name)
1133 : sc_core::sc_module(name)
1134 , smmuv3_gen()
1135 , m_broker(cci::cci_get_broker())
1136 , m_jza(gs::zip_open_from_memory(smmuv3_zip_archive_data(), smmuv3_zip_archive_size()))
1137 , m_loaded_ok(m_jza.json_read_cci(m_broker, std::string(this->name()) + ".smmuv3"))
1138 , M("smmuv3", m_jza)
1139 , p_pamax("pamax", 48)
1140 , p_sidsize("sidsize", 16)
1141 , p_ato("ato", true)
1142 , p_num_tbu("num_tbu", 1)
1143 , p_iotlb_size("iotlb_size", 256)
1144 , p_iidr("iidr", 0x00000000u)
1145 , socket("target_socket")
1146 , dma_socket("dma")
1147 , irq_eventq("irq_eventq")
1148 , irq_priq("irq_priq")
1149 , irq_cmd_sync("irq_cmd_sync")
1150 , irq_gerror("irq_gerror")
1151 , irq_s_gerror("irq_s_gerror")
1152 , irq_s_eventq("irq_s_eventq")
1153 , reset("reset")
1154 , m_next_stag(1)
1155{
1156 SCP_LOGGER();
1157 sc_assert(m_loaded_ok);
1158
1159 socket.bind(M.target_socket);
1160 bind_regs(M);
1161
1162 dma_socket.register_invalidate_direct_mem_ptr(this, &smmuv3<BUSWIDTH>::dma_dmi_invalidate);
1163
1164 reset.register_value_changed_cb([this](bool v) {
1165 if (v) {
1166 m_iotlb.clear();
1167 m_iotlb_lru.clear();
1168 m_ste_cache.clear();
1169 m_ste_cache_lru.clear();
1170 m_cd_cache.clear();
1171 m_cd_cache_lru.clear();
1172 m_stalled_txns.clear();
1173 m_dma_dmi_cache.clear();
1174 } else {
1175 start_of_simulation();
1176 }
1177 });
1178}
1179
1180template <unsigned int BUSWIDTH>
1181void smmuv3<BUSWIDTH>::start_of_simulation()
1182{
1183 IDR0 = 0;
1184 IDR0[IDR0_S2P] = 1;
1185 IDR0[IDR0_S1P] = 1;
1186 IDR0[IDR0_TTF] = IDR0_TTF_AARCH64;
1187 IDR0[IDR0_COHACC] = 1;
1188 IDR0[IDR0_ASID16] = 1;
1189 IDR0[IDR0_VMID16] = 1;
1190 IDR0[IDR0_PRI] = 1;
1191 IDR0[IDR0_ATOS] = static_cast<uint32_t>(p_ato.get_value());
1192 IDR0[IDR0_HTTU] = 3;
1193 IDR0[IDR0_STLEVEL] = IDR0_STLEVEL_2LVL;
1194 IDR0[IDR0_MSI] = 1;
1195 IDR0[IDR0_ATS] = 1;
1196
1197 IDR1 = 0;
1198 IDR1[IDR1_SIDSIZE] = p_sidsize.get_value();
1199 IDR1[IDR1_EVENTQS] = IDR1_DEFAULT_QUEUE_LOG2;
1200 IDR1[IDR1_CMDQS] = IDR1_DEFAULT_QUEUE_LOG2;
1201
1202 IDR5 = 0;
1203 uint32_t oas_val = OAS_48;
1204 switch (p_pamax.get_value()) {
1205 case 32:
1206 oas_val = OAS_32;
1207 break;
1208 case 36:
1209 oas_val = OAS_36;
1210 break;
1211 case 40:
1212 oas_val = OAS_40;
1213 break;
1214 case 42:
1215 oas_val = OAS_42;
1216 break;
1217 case 44:
1218 oas_val = OAS_44;
1219 break;
1220 case 48:
1221 oas_val = OAS_48;
1222 break;
1223 default:
1224 oas_val = OAS_48;
1225 break;
1226 }
1227 IDR5[IDR5_OAS] = oas_val;
1228 IDR5[IDR5_GRAN4K] = 1;
1229 IDR5[IDR5_GRAN16K] = 1;
1230 IDR5[IDR5_GRAN64K] = 1;
1231
1232 for (size_t i = 0; i < PRIMECELL_IDS.size(); ++i) {
1233 *IDREGS[i] = PRIMECELL_IDS[i];
1234 }
1235
1236 IIDR = p_iidr.get_value();
1237
1238 GBPA[GBPA_ABORT] = 1;
1239
1240 // Reset all writable runtime registers to zero. The reset-values .hex only seeds the read-only
1241 // ID/IIDR/GBPA registers, and the gs_register backing is not guaranteed to be zero-initialised
1242 // on every host (observed on macOS-15 debug runners: registers come back as recycled function
1243 // pointers, e.g. S_GERROR=0x10274150 / S_GERRORN=0x00000001, which causes set_secure_gerror's
1244 // "already pending" check to short-circuit and never emit the MSI doorbell).
1245 CR0 = 0;
1246 CR0ACK = 0;
1247 CR1 = 0;
1248 CR2 = 0;
1249 STATUSR = 0;
1250 AGBPA = 0;
1251 IRQ_CTRL = 0;
1252 IRQ_CTRL_ACK = 0;
1253 GERROR = 0;
1254 GERRORN = 0;
1255 GERROR_IRQ_CFG0_LO = 0;
1256 GERROR_IRQ_CFG0_HI = 0;
1257 GERROR_IRQ_CFG1 = 0;
1258 GERROR_IRQ_CFG2 = 0;
1259 STRTAB_BASE_LO = 0;
1260 STRTAB_BASE_HI = 0;
1261 STRTAB_BASE_CFG = 0;
1262 CMDQ_BASE_LO = 0;
1263 CMDQ_BASE_HI = 0;
1264 CMDQ_PROD = 0;
1265 CMDQ_CONS = 0;
1266 EVENTQ_BASE_LO = 0;
1267 EVENTQ_BASE_HI = 0;
1268 EVENTQ_PROD = 0;
1269 EVENTQ_CONS = 0;
1270 EVENTQ_IRQ_CFG0_LO = 0;
1271 EVENTQ_IRQ_CFG0_HI = 0;
1272 EVENTQ_IRQ_CFG1 = 0;
1273 EVENTQ_IRQ_CFG2 = 0;
1274 PRIQ_BASE_LO = 0;
1275 PRIQ_BASE_HI = 0;
1276 PRIQ_PROD = 0;
1277 PRIQ_CONS = 0;
1278 PRIQ_IRQ_CFG0_LO = 0;
1279 PRIQ_IRQ_CFG0_HI = 0;
1280 PRIQ_IRQ_CFG1 = 0;
1281 PRIQ_IRQ_CFG2 = 0;
1282 GATOS_CTRL = 0;
1283 GATOS_SID = 0;
1284 GATOS_ADDR_LO = 0;
1285 GATOS_ADDR_HI = 0;
1286 GATOS_PAR_LO = 0;
1287 GATOS_PAR_HI = 0;
1288
1289 // Secure-world mirrors of the same registers.
1290 S_IDR0 = 0;
1291 S_IDR1 = 0;
1292 S_IDR2 = 0;
1293 S_IDR3 = 0;
1294 S_IDR4 = 0;
1295 S_CR0 = 0;
1296 S_CR0ACK = 0;
1297 S_CR1 = 0;
1298 S_CR2 = 0;
1299 S_INIT = 0;
1300 S_GBPA = 0;
1301 S_AGBPA = 0;
1302 S_IRQ_CTRL = 0;
1303 S_IRQ_CTRL_ACK = 0;
1304 S_GERROR = 0;
1305 S_GERRORN = 0;
1306 S_GERROR_IRQ_CFG0_LO = 0;
1307 S_GERROR_IRQ_CFG0_HI = 0;
1308 S_GERROR_IRQ_CFG1 = 0;
1309 S_GERROR_IRQ_CFG2 = 0;
1310 S_STRTAB_BASE_LO = 0;
1311 S_STRTAB_BASE_HI = 0;
1312 S_STRTAB_BASE_CFG = 0;
1313 S_CMDQ_BASE_LO = 0;
1314 S_CMDQ_BASE_HI = 0;
1315 S_CMDQ_PROD = 0;
1316 S_CMDQ_CONS = 0;
1317 S_EVENTQ_BASE_LO = 0;
1318 S_EVENTQ_BASE_HI = 0;
1319 S_EVENTQ_PROD = 0;
1320 S_EVENTQ_CONS = 0;
1321 S_EVENTQ_IRQ_CFG0_LO = 0;
1322 S_EVENTQ_IRQ_CFG0_HI = 0;
1323 S_EVENTQ_IRQ_CFG1 = 0;
1324 S_EVENTQ_IRQ_CFG2 = 0;
1325
1326 // Page-1 aliases of the queue pointers; pre_read refreshes them later but make the initial
1327 // visible state deterministic too.
1328 CMDQ_PROD_P1 = 0;
1329 CMDQ_CONS_P1 = 0;
1330 EVENTQ_PROD_P1 = 0;
1331 EVENTQ_CONS_P1 = 0;
1332 PRIQ_PROD_P1 = 0;
1333 PRIQ_CONS_P1 = 0;
1334}
1335
1336template <unsigned int BUSWIDTH>
1337void smmuv3<BUSWIDTH>::before_end_of_elaboration()
1338{
1339 SCP_INFO(()) << "SMMUv3: registering post_write callbacks";
1340
1341 CR0.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1342 uint32_t cr0 = static_cast<uint32_t>(CR0);
1343 SCP_INFO(()) << "SMMUv3: CR0 write = 0x" << std::hex << cr0;
1344 CR0ACK = cr0;
1345 });
1346
1347 IRQ_CTRL.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1348 uint32_t val = static_cast<uint32_t>(IRQ_CTRL);
1349 SCP_INFO(()) << "SMMUv3: IRQ_CTRL write = 0x" << std::hex << val;
1350 IRQ_CTRL_ACK = val;
1351 update_irq_levels();
1352 });
1353
1354 GERRORN.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_irq_levels(); });
1355 EVENTQ_CONS.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_irq_levels(); });
1356 PRIQ_CONS.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_irq_levels(); });
1357
1358 S_CR0.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1359 S_CR0ACK = static_cast<uint32_t>(S_CR0);
1360 update_secure_irq_levels();
1361 });
1362 S_IRQ_CTRL.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1363 S_IRQ_CTRL_ACK = static_cast<uint32_t>(S_IRQ_CTRL);
1364 update_secure_irq_levels();
1365 });
1366 S_GERRORN.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_secure_irq_levels(); });
1367 S_EVENTQ_CONS.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_secure_irq_levels(); });
1368
1369 S_CMDQ_PROD.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1370 if (static_cast<uint32_t>(S_CR0[S_CR0_CMDQEN])) {
1371 consume_secure_cmdq();
1372 }
1373 });
1374
1375 S_CMDQ_CONS.pre_read([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1376 if (static_cast<uint32_t>(S_CR0[S_CR0_CMDQEN])) {
1377 consume_secure_cmdq();
1378 }
1379 });
1380
1381 CMDQ_PROD.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1382 uint32_t prod = static_cast<uint32_t>(CMDQ_PROD);
1383 SCP_INFO(()) << "SMMUv3: CMDQ_PROD write = 0x" << std::hex << prod;
1384 if (static_cast<uint32_t>(CR0[CR0_CMDQEN])) {
1385 consume_cmdq();
1386 }
1387 });
1388
1389 CMDQ_CONS.pre_read([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1390 if (static_cast<uint32_t>(CR0[CR0_CMDQEN])) {
1391 consume_cmdq();
1392 }
1393 });
1394
1395 STRTAB_BASE_CFG.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1396 m_ste_cache.clear();
1397 m_ste_cache_lru.clear();
1398 });
1399
1400 GBPA.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) { GBPA[GBPA_UPDATE] = 0u; });
1401 S_GBPA.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) { S_GBPA[S_GBPA_UPDATE] = 0u; });
1402 S_INIT.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1403 if (static_cast<uint32_t>(S_INIT[S_INIT_INV_ALL])) {
1404 m_iotlb.clear();
1405 m_iotlb_lru.clear();
1406 m_ste_cache.clear();
1407 m_ste_cache_lru.clear();
1408 m_cd_cache.clear();
1409 m_cd_cache_lru.clear();
1410 m_ste_cache_secure.clear();
1411 m_cd_cache_secure.clear();
1412 S_INIT[S_INIT_INV_ALL] = 0u;
1413 }
1414 });
1415
1416 GATOS_CTRL.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1417 if (static_cast<uint32_t>(GATOS_CTRL[GATOS_CTRL_RUN])) {
1418 do_gatos();
1419 }
1420 });
1421
1422 PRIQ_CONS.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {});
1423
1424 CMDQ_PROD_P1.post_write([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1425 CMDQ_PROD = static_cast<uint32_t>(CMDQ_PROD_P1);
1426 if (static_cast<uint32_t>(CR0[CR0_CMDQEN])) {
1427 consume_cmdq();
1428 }
1429 });
1430
1431 EVENTQ_CONS_P1.post_write(
1432 [this](tlm::tlm_generic_payload&, sc_core::sc_time&) { EVENTQ_CONS = static_cast<uint32_t>(EVENTQ_CONS_P1); });
1433
1434 PRIQ_CONS_P1.post_write(
1435 [this](tlm::tlm_generic_payload&, sc_core::sc_time&) { PRIQ_CONS = static_cast<uint32_t>(PRIQ_CONS_P1); });
1436
1437 CMDQ_PROD_P1.pre_read(
1438 [this](tlm::tlm_generic_payload&, sc_core::sc_time&) { CMDQ_PROD_P1 = static_cast<uint32_t>(CMDQ_PROD); });
1439 CMDQ_CONS_P1.pre_read([this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1440 if (static_cast<uint32_t>(CR0[CR0_CMDQEN])) consume_cmdq();
1441 CMDQ_CONS_P1 = static_cast<uint32_t>(CMDQ_CONS);
1442 });
1443 EVENTQ_PROD_P1.pre_read(
1444 [this](tlm::tlm_generic_payload&, sc_core::sc_time&) { EVENTQ_PROD_P1 = static_cast<uint32_t>(EVENTQ_PROD); });
1445 EVENTQ_CONS_P1.pre_read(
1446 [this](tlm::tlm_generic_payload&, sc_core::sc_time&) { EVENTQ_CONS_P1 = static_cast<uint32_t>(EVENTQ_CONS); });
1447 PRIQ_PROD_P1.pre_read(
1448 [this](tlm::tlm_generic_payload&, sc_core::sc_time&) { PRIQ_PROD_P1 = static_cast<uint32_t>(PRIQ_PROD); });
1449 PRIQ_CONS_P1.pre_read(
1450 [this](tlm::tlm_generic_payload&, sc_core::sc_time&) { PRIQ_CONS_P1 = static_cast<uint32_t>(PRIQ_CONS); });
1451}
1452
1453template <unsigned int BUSWIDTH>
1454bool smmuv3<BUSWIDTH>::find_ste(uint32_t sid, STE& ste)
1455{
1456 auto it = m_ste_cache.find(sid);
1457 if (it != m_ste_cache.end()) {
1458 ste = it->second;
1459 return ste.valid;
1460 }
1461
1462 constexpr uint64_t L1STD_VALID_BIT = 1;
1463 constexpr uint32_t L1STD_PTR_LOW_BITS = 6;
1464 constexpr size_t L1STD_SIZE = 8;
1465 constexpr uint32_t STRTAB_FMT_LINEAR = 0;
1466
1467 uint64_t strtab_base = ((static_cast<uint64_t>(STRTAB_BASE_HI) << 32) | static_cast<uint64_t>(STRTAB_BASE_LO)) &
1468 STE_BASE_ADDR_MASK;
1469 uint32_t fmt = static_cast<uint32_t>(STRTAB_BASE_CFG[STRTAB_BASE_CFG_FMT]);
1470
1471 uint32_t log2size = static_cast<uint32_t>(STRTAB_BASE_CFG[STRTAB_BASE_CFG_LOG2SIZE]);
1472 if (log2size > 31u) log2size = 31u;
1473
1474 std::array<uint8_t, SMMUV3_STE_SIZE> ste_buf{};
1475 uint64_t ste_addr;
1476
1477 if (fmt == STRTAB_FMT_LINEAR) {
1478 if (sid >= (1u << log2size)) {
1479 return false;
1480 }
1481 ste_addr = strtab_base + (sid * SMMUV3_STE_SIZE);
1482 } else {
1483 uint32_t split = static_cast<uint32_t>(STRTAB_BASE_CFG[STRTAB_BASE_CFG_SPLIT]);
1484 uint32_t l1_idx = sid >> split;
1485 uint32_t l2_idx = sid & ((1u << split) - 1);
1486
1487 if (l1_idx >= (1u << log2size)) {
1488 return false;
1489 }
1490
1491 uint64_t l1std_addr = strtab_base + (l1_idx * L1STD_SIZE);
1492 uint64_t l1std;
1493 if (!dma_read(l1std_addr, &l1std, L1STD_SIZE)) {
1494 set_gerror(GERROR_BIT_SFM_ERR);
1495 return false;
1496 }
1497
1498 if ((l1std & L1STD_VALID_BIT) == 0) {
1499 return false;
1500 }
1501
1502 uint64_t l2ptr = l1std & ~((1ULL << L1STD_PTR_LOW_BITS) - 1);
1503 ste_addr = l2ptr + (l2_idx * SMMUV3_STE_SIZE);
1504 }
1505
1506 if (!dma_read(ste_addr, ste_buf.data(), SMMUV3_STE_SIZE)) {
1507 set_gerror(GERROR_BIT_SFM_ERR);
1508 return false;
1509 }
1510
1511 uint64_t dw0 = load_le<uint64_t>(ste_buf.data() + 0);
1512 uint64_t dw1 = load_le<uint64_t>(ste_buf.data() + 8);
1513 uint64_t dw2 = load_le<uint64_t>(ste_buf.data() + 16);
1514 uint64_t dw3 = load_le<uint64_t>(ste_buf.data() + 24);
1515
1516 ste.valid = dw0 & 1;
1517 ste.config = extract64(dw0, 1, 3);
1518 ste.s1fmt = extract64(dw0, 4, 2);
1519 ste.s1contextptr = dw0 & STE_BASE_ADDR_MASK;
1520 ste.s1cdmax = extract64(dw0, 59, 5);
1521
1522 ste.s1stalld = extract64(dw1, 27, 1);
1523 ste.eats = extract64(dw1, 28, 2);
1524 ste.strw = extract64(dw1, 30,
1525 2); // STRW: 0=NS-EL1, 2=NS-EL2 (E2H further selected by CR2.E2H), other=secure-only/reserved
1526
1527 ste.vmid = extract64(dw2, 0, 16);
1528 ste.s2t0sz = extract64(dw2, 32, 6);
1529 ste.s2sl0 = extract64(dw2, 38, 2);
1530 ste.s2tg = extract64(dw2, 46, 2);
1531 ste.s2ps = extract64(dw2, 48, 3);
1532 ste.s2hd = extract64(dw2, 56, 1);
1533 ste.s2affd = extract64(dw2, 60, 1);
1534
1535 ste.s2ttb = dw3 & S2TTB_ADDR_MASK;
1536
1537 if (m_ste_cache.size() >= STE_CACHE_MAX && !m_ste_cache_lru.empty()) {
1538 uint32_t victim = m_ste_cache_lru.back();
1539 m_ste_cache_lru.pop_back();
1540 m_ste_cache.erase(victim);
1541 }
1542 m_ste_cache[sid] = ste;
1543 m_ste_cache_lru.push_front(sid);
1544 return ste.valid;
1545}
1546
1547template <unsigned int BUSWIDTH>
1548bool smmuv3<BUSWIDTH>::read_cd(uint32_t sid, const STE& ste, uint32_t substream_id, CD& cd)
1549{
1550 uint64_t key = (static_cast<uint64_t>(sid) << 32) | substream_id;
1551 auto it = m_cd_cache.find(key);
1552 if (it != m_cd_cache.end()) {
1553 cd = it->second;
1554 return cd.valid;
1555 }
1556
1557 constexpr uint32_t CD_TSZ_MASK = 0x3F;
1558 constexpr uint32_t CD_TG_MASK = 0x3;
1559 constexpr uint32_t CD_IPS_MASK = 0x7;
1560 constexpr uint32_t CD_ASID_MASK = 0xFFFF;
1561 constexpr uint32_t CD_VALID_SHIFT = 31;
1562 constexpr uint32_t CD_AARCH64_SHIFT = 9;
1563 constexpr uint32_t CD_T0_TG_SHIFT = 6;
1564 constexpr uint32_t CD_EPD0_SHIFT = 14;
1565 constexpr uint32_t CD_T1SZ_SHIFT = 16;
1566 constexpr uint32_t CD_T1_TG_SHIFT = 22;
1567 constexpr uint32_t CD_EPD1_SHIFT = 30;
1568 constexpr uint32_t CD_ASID_SHIFT = 16;
1569 constexpr uint32_t CD_TTB_HI_MASK = 0xFFFFF;
1570 constexpr uint32_t CD_TTB_LO_MASK = 0xFFFFFFF0;
1571 constexpr uint32_t CD_AFFD_BIT_IN_W1 = 3;
1572 constexpr uint32_t CD_HA_BIT_IN_W1 = 11;
1573 constexpr uint32_t CD_HD_BIT_IN_W1 = 10;
1574
1575 uint64_t cd_addr;
1576 constexpr size_t L1CD_SIZE = 8;
1577 constexpr uint64_t L1CD_VALID_BIT = 1ULL;
1578 constexpr uint64_t L1CD_PTR_MASK = ~0xFFFULL;
1579
1580 if (ste.s1fmt == S1FMT_LINEAR || ste.s1cdmax == 0) {
1581 cd_addr = ste.s1contextptr + (substream_id * SMMUV3_CD_SIZE);
1582 } else {
1583 uint32_t split = (ste.s1fmt == S1FMT_4KB_L2) ? S1FMT_4KB_L2_SPLIT : S1FMT_64KB_L2_SPLIT;
1584 uint64_t l1_idx = static_cast<uint64_t>(substream_id) >> split;
1585 uint64_t l2_idx = static_cast<uint64_t>(substream_id) & ((1ULL << split) - 1);
1586 uint64_t l1cd_addr = ste.s1contextptr + (l1_idx * L1CD_SIZE);
1587 uint64_t l1cd = 0;
1588 if (!dma_read(l1cd_addr, &l1cd, L1CD_SIZE)) {
1589 set_gerror(GERROR_BIT_SFM_ERR);
1590 return false;
1591 }
1592 if ((l1cd & L1CD_VALID_BIT) == 0) {
1593 cd.valid = false;
1594 return false;
1595 }
1596 uint64_t l2_ptr = l1cd & L1CD_PTR_MASK;
1597 cd_addr = l2_ptr + (l2_idx * SMMUV3_CD_SIZE);
1598 }
1599
1600 std::array<uint8_t, SMMUV3_CD_SIZE> cd_buf{};
1601
1602 if (!dma_read(cd_addr, cd_buf.data(), SMMUV3_CD_SIZE)) {
1603 set_gerror(GERROR_BIT_SFM_ERR);
1604 return false;
1605 }
1606
1607 std::array<uint32_t, 6> w{};
1608 for (size_t i = 0; i < w.size(); ++i) {
1609 w[i] = load_le<uint32_t>(cd_buf.data() + i * sizeof(uint32_t));
1610 }
1611
1612 cd.valid = (w[0] >> CD_VALID_SHIFT) & 0x1;
1613 cd.aarch64 = (w[1] >> CD_AARCH64_SHIFT) & 0x1;
1614 cd.tsz[0] = w[0] & CD_TSZ_MASK;
1615 cd.tg[0] = (w[0] >> CD_T0_TG_SHIFT) & CD_TG_MASK;
1616 cd.epd[0] = (w[0] >> CD_EPD0_SHIFT) & 0x1;
1617 cd.tsz[1] = (w[0] >> CD_T1SZ_SHIFT) & CD_TSZ_MASK;
1618 cd.tg[1] = (w[0] >> CD_T1_TG_SHIFT) & CD_TG_MASK;
1619 cd.epd[1] = (w[0] >> CD_EPD1_SHIFT) & 0x1;
1620 cd.ips = w[1] & CD_IPS_MASK;
1621 cd.asid = (w[1] >> CD_ASID_SHIFT) & CD_ASID_MASK;
1622 cd.affd = (w[1] >> CD_AFFD_BIT_IN_W1) & 0x1;
1623 cd.ha = (w[1] >> CD_HA_BIT_IN_W1) & 0x1;
1624 cd.hd = (w[1] >> CD_HD_BIT_IN_W1) & 0x1;
1625
1626 cd.ttb[0] = (static_cast<uint64_t>(w[3] & CD_TTB_HI_MASK) << 32) | static_cast<uint64_t>(w[2] & CD_TTB_LO_MASK);
1627 cd.ttb[1] = (static_cast<uint64_t>(w[5] & CD_TTB_HI_MASK) << 32) | static_cast<uint64_t>(w[4] & CD_TTB_LO_MASK);
1628
1629 if (m_cd_cache.size() >= CD_CACHE_MAX && !m_cd_cache_lru.empty()) {
1630 uint64_t victim = m_cd_cache_lru.back();
1631 m_cd_cache_lru.pop_back();
1632 m_cd_cache.erase(victim);
1633 }
1634 m_cd_cache[key] = cd;
1635 m_cd_cache_lru.push_front(key);
1636 return cd.valid;
1637}
1638
1639template <unsigned int BUSWIDTH>
1640bool smmuv3<BUSWIDTH>::smmuv3_ptw64(PtwCtx& ctx, TransReq& req)
1641{
1642 unsigned int tsz;
1643 unsigned int inputsize;
1644 unsigned int outputsize;
1645 unsigned int grainsize = 0;
1646 unsigned int stride;
1647 int level = 0;
1648 unsigned int firstblocklevel = 0;
1649 unsigned int tg;
1650 unsigned int baselowerbound;
1651 bool blocktranslate = false;
1652 bool epd = false;
1653 uint64_t descmask;
1654 uint64_t ttbr;
1655 uint64_t desc;
1656 uint32_t attrs;
1657 uint32_t s2attrs;
1658
1659 req.err = false;
1660 req.tableattrs = 0;
1661 req.fault_level = 0;
1662
1663 constexpr uint32_t VA_BITS_TOTAL = 64;
1664 constexpr uint32_t STRIDE_OFFSET = 3;
1665 constexpr uint32_t LEVEL_LEAF = 3;
1666 constexpr uint32_t FIRSTBLOCK_LARGE_GRAIN = 2;
1667 constexpr uint32_t FIRSTBLOCK_4K = 1;
1668 constexpr uint32_t TG1_INVALID = 0xFF;
1669
1670 if (ctx.stage == 1) {
1671 if ((ctx.iova & (1ULL << VA_SIGN_BIT_SHIFT)) == 0) {
1672 ttbr = ctx.ttb[0];
1673 tsz = ctx.tsz[0];
1674 tg = ctx.tg[0];
1675 epd = ctx.epd[0];
1676 } else {
1677 ttbr = ctx.ttb[1];
1678 tsz = ctx.tsz[1];
1679 epd = ctx.epd[1];
1680 switch (ctx.tg[1]) {
1681 case 1:
1682 tg = CD_TG_16K;
1683 break;
1684 case 2:
1685 tg = CD_TG_4K;
1686 break;
1687 case 3:
1688 tg = CD_TG_64K;
1689 break;
1690 default:
1691 tg = TG1_INVALID;
1692 break;
1693 }
1694 }
1695 } else {
1696 ttbr = ctx.ttb[0];
1697 tsz = ctx.tsz[0];
1698 tg = ctx.tg[0];
1699 epd = ctx.epd[0];
1700 }
1701
1702 if (epd) {
1703 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1704 req.err = true;
1705 return false;
1706 }
1707
1708 inputsize = VA_BITS_TOTAL - tsz;
1709
1710 switch (tg) {
1711 case CD_TG_64K:
1712 grainsize = GRAINSIZE_64K;
1713 level = LEVEL_LEAF;
1714 firstblocklevel = FIRSTBLOCK_LARGE_GRAIN;
1715 break;
1716 case CD_TG_16K:
1717 grainsize = GRAINSIZE_16K;
1718 level = LEVEL_LEAF;
1719 firstblocklevel = FIRSTBLOCK_LARGE_GRAIN;
1720 break;
1721 case CD_TG_4K:
1722 grainsize = GRAINSIZE_4K;
1723 level = 2;
1724 firstblocklevel = FIRSTBLOCK_4K;
1725 break;
1726 default:
1727 SCP_ERR(()) << "Invalid TG value: " << tg;
1728 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1729 req.err = true;
1730 return false;
1731 }
1732
1733 outputsize = OUTPUT_SIZE_MAP[ctx.ips];
1734 if (outputsize > p_pamax) outputsize = p_pamax;
1735
1736 stride = grainsize - STRIDE_OFFSET;
1737
1738 if (ctx.stage == 1) {
1739 if (grainsize < GRAINSIZE_64K && (inputsize > (grainsize + 3 * stride)))
1740 level = 0;
1741 else if (inputsize > (grainsize + 2 * stride))
1742 level = 1;
1743 else if (inputsize > (grainsize + stride))
1744 level = 2;
1745
1746 if (inputsize < INPUT_SIZE_MIN || inputsize > INPUT_SIZE_MAX ||
1747 extract64(ctx.iova, inputsize, VA_BITS_TOTAL - inputsize)) {
1748 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1749 req.err = true;
1750 return false;
1751 }
1752 } else {
1753 unsigned int startlevel = ctx.sl0;
1754 level = 3 - startlevel;
1755 if (grainsize == GRAINSIZE_4K) level = 2 - startlevel;
1756 if (!check_s2_startlevel(outputsize, level, inputsize, stride)) {
1757 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1758 req.err = true;
1759 return false;
1760 }
1761 }
1762
1763 constexpr uint64_t INDEX_ALIGN_8B = 7ULL;
1764
1765 unsigned int level_coverage = grainsize + (LEVEL_LEAF - level + 1) * stride;
1766 unsigned int concat_bits = 0;
1767 if (ctx.stage == 2 && inputsize > level_coverage) {
1768 concat_bits = inputsize - level_coverage;
1769 if (concat_bits > 4) {
1770 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1771 req.err = true;
1772 return false;
1773 }
1774 }
1775
1776 baselowerbound = STRIDE_OFFSET + inputsize - ((LEVEL_LEAF - level) * stride + grainsize);
1777 ttbr = extract64(ttbr, 0, PA_BITS);
1778 ttbr &= ~((1ULL << baselowerbound) - 1);
1779
1780 if (concat_bits > 0) {
1781 uint64_t concat_idx = (ctx.iova >> level_coverage) & ((1ULL << concat_bits) - 1);
1782 uint64_t table_bytes = 1ULL << (grainsize + stride);
1783 ttbr |= concat_idx * table_bytes;
1784 }
1785
1786 if (!check_out_addr(ttbr, outputsize)) {
1787 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1788 req.err = true;
1789 return false;
1790 }
1791
1792 descmask = (1ULL << grainsize) - 1;
1793
1794 do {
1795 unsigned int addrselectbottom = (LEVEL_LEAF - level) * stride + grainsize;
1796 uint64_t index = (ctx.iova >> (addrselectbottom - STRIDE_OFFSET)) & descmask;
1797 index &= ~INDEX_ALIGN_8B;
1798 uint64_t descaddr = ttbr | index;
1799
1800 if (ctx.stage == 1 && ctx.s2_enabled) {
1801 PtwCtx s2ctx;
1802 s2ctx.ttb[0] = ctx.ste->s2ttb;
1803 s2ctx.tsz[0] = ctx.ste->s2t0sz;
1804 s2ctx.tg[0] = ctx.ste->s2tg;
1805 s2ctx.ips = ctx.ste->s2ps;
1806 s2ctx.sl0 = ctx.ste->s2sl0;
1807 s2ctx.epd[0] = false;
1808 s2ctx.stage = 2;
1809 s2ctx.iova = descaddr;
1810 s2ctx.access = IOMMUAccessFlags::RO;
1811 s2ctx.s2_enabled = false;
1812 s2ctx.ste = ctx.ste;
1813 s2ctx.affd = ctx.ste->s2affd;
1814 s2ctx.ha = false;
1815 s2ctx.hd = ctx.ste->s2hd;
1816
1817 TransReq s2req;
1818 s2req.iova = descaddr;
1819 s2req.access = IOMMUAccessFlags::RO;
1820 s2req.pa = descaddr;
1821
1822 if (!smmuv3_ptw64(s2ctx, s2req) || s2req.err) {
1823 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1824 req.fault_level = level;
1825 req.err = true;
1826 return false;
1827 }
1828 descaddr = s2req.pa;
1829 }
1830
1831 if (!dma_read(descaddr, &desc, sizeof(desc))) {
1832 req.event_type = SMMUV3_EVT_F_WALK_EABT;
1833 req.fault_level = level;
1834 req.err = true;
1835 return false;
1836 }
1837
1838 req.leaf_desc_addr = descaddr;
1839
1840 constexpr uint32_t DESC_TYPE_MASK = 0x3;
1841 constexpr uint32_t TABLE_ATTRS_SHIFT = 59;
1842 constexpr uint32_t TABLE_ATTRS_WIDTH = 5;
1843
1844 unsigned int type = desc & DESC_TYPE_MASK;
1845 SCP_INFO(()) << "S" << ctx.stage << " L" << level << " iova=0x" << std::hex << ctx.iova << " descaddr=0x"
1846 << descaddr << " desc=0x" << desc << " type=" << type;
1847
1848 ttbr = extract64(desc, 0, PA_BITS);
1849 ttbr &= ~descmask;
1850
1851 if (level == LEVEL_LEAF) {
1852 if (type != DESC_TYPE_TABLE_OR_PAGE) {
1853 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1854 req.fault_level = level;
1855 req.err = true;
1856 return false;
1857 }
1858 break;
1859 }
1860
1861 switch (type) {
1862 case DESC_TYPE_RESERVED:
1863 case DESC_TYPE_INVALID:
1864 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1865 req.fault_level = level;
1866 req.err = true;
1867 return false;
1868
1869 case DESC_TYPE_BLOCK:
1870 blocktranslate = true;
1871 if (level < (int)firstblocklevel) {
1872 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1873 req.fault_level = level;
1874 req.err = true;
1875 return false;
1876 }
1877 break;
1878
1879 case DESC_TYPE_TABLE_OR_PAGE:
1880 req.tableattrs |= extract64(desc, TABLE_ATTRS_SHIFT, TABLE_ATTRS_WIDTH);
1881 if (!check_out_addr(ttbr, outputsize)) {
1882 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1883 req.fault_level = level;
1884 req.err = true;
1885 return false;
1886 }
1887 level++;
1888 break;
1889
1890 default:
1891 SCP_ERR(()) << "SMMUv3: unreachable descriptor type " << type;
1892 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1893 req.fault_level = level;
1894 req.err = true;
1895 return false;
1896 }
1897 } while (!blocktranslate);
1898
1899 if (!check_out_addr(ttbr, outputsize)) {
1900 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1901 req.fault_level = level;
1902 req.err = true;
1903 return false;
1904 }
1905
1906 constexpr uint32_t LEVEL_COUNT = 4;
1907 constexpr uint32_t DESC_ATTRS_LOW_SHIFT = 2;
1908 constexpr uint32_t DESC_ATTRS_LOW_WIDTH = 10;
1909 constexpr uint32_t DESC_ATTRS_HI_SHIFT = 52;
1910 constexpr uint32_t DESC_ATTRS_HI_WIDTH = 12;
1911 constexpr uint32_t TABLE_AP_SHIFT_IN_ATTRS = 11;
1912 constexpr uint32_t TABLE_PXN_BIT_IN_TABLEATTRS = 3;
1913
1914 unsigned long page_size = (1ULL << ((stride * (LEVEL_COUNT - level)) + STRIDE_OFFSET));
1915 ttbr |= (ctx.iova & (page_size - 1));
1916 req.page_size = ((stride * (LEVEL_COUNT - level)) + STRIDE_OFFSET);
1917 req.pa = ttbr;
1918
1919 req.descriptor = desc;
1920
1921 s2attrs = attrs = extract64(desc, DESC_ATTRS_LOW_SHIFT, DESC_ATTRS_LOW_WIDTH) |
1922 (extract64(desc, DESC_ATTRS_HI_SHIFT, DESC_ATTRS_HI_WIDTH) << DESC_ATTRS_LOW_WIDTH);
1923
1924 if (ctx.stage == 1) {
1925 attrs |= extract32(req.tableattrs, 0, 2) << TABLE_AP_SHIFT_IN_ATTRS;
1926 attrs |= extract32(req.tableattrs, TABLE_PXN_BIT_IN_TABLEATTRS, 1) << DESC_ATTR_AP2_BIT;
1927 }
1928
1929 req.prot = IOMMUAccessFlags::RW;
1930
1931 bool dbm_set = (desc >> DESC_BIT_DBM) & 1ULL;
1932 bool stage1_httu_af = (ctx.stage == 1) ? ctx.ha : false;
1933 bool stage1_httu_dbm = (ctx.stage == 1) ? ctx.hd : false;
1934 bool stage2_httu_af = (ctx.stage == 2) && ctx.ste && ctx.ste->s2hd;
1935 (void)stage2_httu_af;
1936
1937 bool af_set = attrs & (1u << DESC_ATTR_AF_BIT);
1938 if (!af_set) {
1939 bool affd = (ctx.stage == 1) ? ctx.affd : (ctx.ste ? ctx.ste->s2affd : false);
1940 bool httu_af = stage1_httu_af || stage2_httu_af;
1941 if (httu_af) {
1942 uint64_t new_desc = desc | (1ULL << DESC_BIT_AF);
1943 if (dma_write(req.leaf_desc_addr, &new_desc, sizeof(new_desc))) {
1944 desc = new_desc;
1945 attrs |= (1u << DESC_ATTR_AF_BIT);
1946 af_set = true;
1947 }
1948 }
1949 if (!af_set && !affd) {
1950 req.event_type = SMMUV3_EVT_F_ACCESS;
1951 req.fault_level = level;
1952 req.err = true;
1953 return false;
1954 }
1955 }
1956
1957 if (ctx.stage == 1) {
1958 if (!(attrs & (1u << DESC_ATTR_AP1_BIT))) {
1959 req.event_type = SMMUV3_EVT_F_PERMISSION;
1960 req.fault_level = level;
1961 req.err = true;
1962 return false;
1963 }
1964 if (attrs & (1u << DESC_ATTR_AP2_BIT)) {
1965 if (ctx.access == IOMMUAccessFlags::WO) {
1966 if (stage1_httu_dbm && dbm_set) {
1967 uint64_t new_desc = desc & ~(1ULL << (DESC_ATTRS_LOW_SHIFT + DESC_ATTR_AP2_BIT));
1968 if (dma_write(req.leaf_desc_addr, &new_desc, sizeof(new_desc))) {
1969 desc = new_desc;
1970 attrs &= ~(1u << DESC_ATTR_AP2_BIT);
1971 }
1972 }
1973 if (attrs & (1u << DESC_ATTR_AP2_BIT)) {
1974 req.event_type = SMMUV3_EVT_F_PERMISSION;
1975 req.fault_level = level;
1976 req.err = true;
1977 return false;
1978 }
1979 } else {
1980 req.prot &= ~IOMMUAccessFlags::WO;
1981 }
1982 }
1983 } else {
1984 switch ((s2attrs >> DESC_ATTR_AP1_BIT) & 0x3) {
1985 case S2AP_NONE:
1986 req.event_type = SMMUV3_EVT_F_PERMISSION;
1987 req.fault_level = level;
1988 req.err = true;
1989 return false;
1990 case S2AP_RO:
1991 if (ctx.access == IOMMUAccessFlags::WO) {
1992 req.event_type = SMMUV3_EVT_F_PERMISSION;
1993 req.fault_level = level;
1994 req.err = true;
1995 return false;
1996 }
1997 req.prot &= ~IOMMUAccessFlags::WO;
1998 break;
1999 case S2AP_WO:
2000 if (ctx.access == IOMMUAccessFlags::RO) {
2001 req.event_type = SMMUV3_EVT_F_PERMISSION;
2002 req.fault_level = level;
2003 req.err = true;
2004 return false;
2005 }
2006 req.prot &= ~IOMMUAccessFlags::RO;
2007 break;
2008 case S2AP_RW:
2009 break;
2010 default:
2011 SCP_ERR(()) << "SMMUv3: unreachable S2AP value";
2012 req.event_type = SMMUV3_EVT_F_PERMISSION;
2013 req.fault_level = level;
2014 req.err = true;
2015 return false;
2016 }
2017 }
2018
2019 SCP_INFO(()) << "PTW success: 0x" << std::hex << ctx.iova << " -> 0x" << req.pa
2020 << " prot=" << static_cast<uint32_t>(req.prot) << " page_size=" << std::dec << req.page_size;
2021 return true;
2022}
2023
2024template <unsigned int BUSWIDTH>
2025typename smmuv3<BUSWIDTH>::IOMMUTLBEntry smmuv3<BUSWIDTH>::smmuv3_translate(tlm::tlm_generic_payload& txn, uint32_t sid,
2026 uint32_t substream_id, bool is_secure,
2027 bool is_ats_tr)
2028{
2029 (void)is_secure;
2030 (void)is_ats_tr;
2031
2032 uint64_t iova = txn.get_address();
2033 IOMMUTLBEntry ret = {
2034 .iova = iova,
2035 .translated_addr = iova,
2036 .addr_mask = (1ULL << PAGE_SHIFT_4K) - 1,
2037 .perm = IOMMUAccessFlags::RW,
2038 .descriptor = 0,
2039 .table_attrs = 0,
2040 .fault_type = 0,
2041 .fault_level = 0,
2042 .stallable_fault = false,
2043 };
2044
2045 if (static_cast<uint32_t>(CR0[CR0_SMMUEN]) == 0) {
2046 ret.addr_mask = -1ULL;
2047 return ret;
2048 }
2049
2050 TransReq req;
2051 req.iova = iova;
2052 req.sid = sid;
2053 req.substream_id = substream_id;
2054 req.access = (txn.get_command() == tlm::TLM_WRITE_COMMAND) ? IOMMUAccessFlags::WO : IOMMUAccessFlags::RO;
2055
2056 if (!find_ste(sid, req.ste) || !req.ste.valid) {
2057 if (static_cast<uint32_t>(GBPA[GBPA_ABORT])) {
2058 ret.perm = IOMMUAccessFlags::NONE;
2059 ret.fault_type = SMMUV3_EVT_F_STE_FETCH;
2060 record_event_full(SMMUV3_EVT_F_STE_FETCH, sid, substream_id, iova, txn, 0, 0, 0, false);
2061 return ret;
2062 } else {
2063 ret.addr_mask = -1ULL;
2064 return ret;
2065 }
2066 }
2067
2068 if (req.ste.s1cdmax > 0 && substream_id >= (1u << req.ste.s1cdmax)) {
2069 ret.perm = IOMMUAccessFlags::NONE;
2070 ret.fault_type = SMMUV3_EVT_C_BAD_SUBSTREAMID;
2071 record_event_full(SMMUV3_EVT_C_BAD_SUBSTREAMID, sid, substream_id, iova, txn, 0, 0, 0, false);
2072 return ret;
2073 }
2074
2075 // Tag every IOTLB key for this transaction with the resolved CPU translation regime so cached
2076 // entries cannot alias across regimes (NS-EL1 / NS-EL2 / NS-EL2-E2H / Secure).
2077 const bool e2h_enabled = static_cast<uint32_t>(CR2[CR2_E2H]) != 0;
2078 const uint8_t iotlb_regime = compute_iotlb_regime(req.ste.strw, e2h_enabled, is_secure);
2079
2080 constexpr uint8_t IOTLB_LEAF_LEVEL = 3;
2081
2082 switch (req.ste.config) {
2083 case STE_CONFIG_ABORT:
2084 ret.perm = IOMMUAccessFlags::NONE;
2085 ret.fault_type = SMMUV3_EVT_C_BAD_STE;
2086 record_event_full(SMMUV3_EVT_C_BAD_STE, sid, substream_id, iova, txn, 0, 0, 0, false);
2087 return ret;
2088 case STE_CONFIG_BYPASS:
2089 ret.addr_mask = -1ULL;
2090 return ret;
2091 case STE_CONFIG_S1:
2092 if (!read_cd(sid, req.ste, substream_id, req.cd) || !req.cd.valid) {
2093 ret.perm = IOMMUAccessFlags::NONE;
2094 ret.fault_type = SMMUV3_EVT_F_CD_FETCH;
2095 record_event_full(SMMUV3_EVT_F_CD_FETCH, sid, substream_id, iova, txn, SMMUV3_CLASS_CD, 0, 0, false);
2096 return ret;
2097 }
2098 {
2099 IOTLBKey lookup{ req.ste.vmid,
2100 req.cd.asid,
2101 iova & ~((1ULL << PAGE_SHIFT_4K) - 1),
2102 static_cast<uint8_t>(req.cd.tg[0]),
2103 IOTLB_LEAF_LEVEL,
2104 is_secure,
2105 iotlb_regime };
2106 if (iotlb_lookup(lookup, ret)) {
2107 return ret;
2108 }
2109 }
2110 {
2111 PtwCtx ctx;
2112 ctx.ttb[0] = req.cd.ttb[0];
2113 ctx.ttb[1] = req.cd.ttb[1];
2114 ctx.tsz[0] = req.cd.tsz[0];
2115 ctx.tsz[1] = req.cd.tsz[1];
2116 ctx.tg[0] = req.cd.tg[0];
2117 ctx.tg[1] = req.cd.tg[1];
2118 ctx.ips = req.cd.ips;
2119 ctx.epd[0] = req.cd.epd[0];
2120 ctx.epd[1] = req.cd.epd[1];
2121 ctx.stage = 1;
2122 ctx.iova = iova;
2123 ctx.access = req.access;
2124 ctx.s2_enabled = false;
2125 ctx.ste = nullptr;
2126 ctx.affd = req.cd.affd;
2127 ctx.ha = req.cd.ha;
2128 ctx.hd = req.cd.hd;
2129 smmuv3_ptw64(ctx, req);
2130 }
2131 break;
2132 case STE_CONFIG_S2: {
2133 IOTLBKey lookup{ req.ste.vmid,
2134 0,
2135 iova & ~((1ULL << PAGE_SHIFT_4K) - 1),
2136 static_cast<uint8_t>(req.ste.s2tg),
2137 IOTLB_LEAF_LEVEL,
2138 is_secure,
2139 iotlb_regime };
2140 if (iotlb_lookup(lookup, ret)) {
2141 return ret;
2142 }
2143 PtwCtx ctx;
2144 ctx.ttb[0] = req.ste.s2ttb;
2145 ctx.tsz[0] = req.ste.s2t0sz;
2146 ctx.tg[0] = req.ste.s2tg;
2147 ctx.ips = req.ste.s2ps;
2148 ctx.sl0 = req.ste.s2sl0;
2149 ctx.epd[0] = false;
2150 ctx.stage = 2;
2151 ctx.iova = iova;
2152 ctx.access = req.access;
2153 ctx.s2_enabled = false;
2154 ctx.ste = &req.ste;
2155 ctx.affd = req.ste.s2affd;
2156 ctx.ha = false;
2157 ctx.hd = req.ste.s2hd;
2158 smmuv3_ptw64(ctx, req);
2159 } break;
2160 case STE_CONFIG_NESTED:
2161 if (!read_cd(sid, req.ste, substream_id, req.cd) || !req.cd.valid) {
2162 ret.perm = IOMMUAccessFlags::NONE;
2163 ret.fault_type = SMMUV3_EVT_F_CD_FETCH;
2164 record_event_full(SMMUV3_EVT_F_CD_FETCH, sid, substream_id, iova, txn, SMMUV3_CLASS_CD, 0, 0, false);
2165 return ret;
2166 }
2167 {
2168 IOTLBKey lookup{ req.ste.vmid,
2169 req.cd.asid,
2170 iova & ~((1ULL << PAGE_SHIFT_4K) - 1),
2171 static_cast<uint8_t>(req.cd.tg[0]),
2172 IOTLB_LEAF_LEVEL,
2173 is_secure,
2174 iotlb_regime };
2175 if (iotlb_lookup(lookup, ret)) {
2176 return ret;
2177 }
2178 }
2179 {
2180 PtwCtx ctx;
2181 ctx.ttb[0] = req.cd.ttb[0];
2182 ctx.ttb[1] = req.cd.ttb[1];
2183 ctx.tsz[0] = req.cd.tsz[0];
2184 ctx.tsz[1] = req.cd.tsz[1];
2185 ctx.tg[0] = req.cd.tg[0];
2186 ctx.tg[1] = req.cd.tg[1];
2187 ctx.ips = req.cd.ips;
2188 ctx.epd[0] = req.cd.epd[0];
2189 ctx.epd[1] = req.cd.epd[1];
2190 ctx.stage = 1;
2191 ctx.iova = iova;
2192 ctx.access = req.access;
2193 ctx.s2_enabled = true;
2194 ctx.ste = &req.ste;
2195 ctx.affd = req.cd.affd;
2196 ctx.ha = req.cd.ha;
2197 ctx.hd = req.cd.hd;
2198 smmuv3_ptw64(ctx, req);
2199
2200 if (!req.err) {
2201 uint64_t s1_pa = req.pa;
2202 PtwCtx s2ctx;
2203 s2ctx.ttb[0] = req.ste.s2ttb;
2204 s2ctx.tsz[0] = req.ste.s2t0sz;
2205 s2ctx.tg[0] = req.ste.s2tg;
2206 s2ctx.ips = req.ste.s2ps;
2207 s2ctx.sl0 = req.ste.s2sl0;
2208 s2ctx.epd[0] = false;
2209 s2ctx.stage = 2;
2210 s2ctx.iova = s1_pa;
2211 s2ctx.access = req.access;
2212 s2ctx.s2_enabled = false;
2213 s2ctx.ste = &req.ste;
2214 s2ctx.affd = req.ste.s2affd;
2215 s2ctx.ha = false;
2216 s2ctx.hd = req.ste.s2hd;
2217 smmuv3_ptw64(s2ctx, req);
2218 }
2219 }
2220 break;
2221 default:
2222 ret.perm = IOMMUAccessFlags::NONE;
2223 ret.fault_type = SMMUV3_EVT_C_BAD_STE;
2224 record_event_full(SMMUV3_EVT_C_BAD_STE, sid, substream_id, iova, txn, 0, 0, 0, false);
2225 return ret;
2226 }
2227
2228 if (req.err) {
2229 ret.perm = IOMMUAccessFlags::NONE;
2230 ret.fault_type = req.event_type;
2231 ret.fault_level = req.fault_level;
2232 bool stallable = (req.event_type == SMMUV3_EVT_F_TRANSLATION || req.event_type == SMMUV3_EVT_F_PERMISSION ||
2233 req.event_type == SMMUV3_EVT_F_ACCESS || req.event_type == SMMUV3_EVT_F_ADDR_SIZE);
2234 ret.stallable_fault = stallable && (req.ste.config == STE_CONFIG_S1 || req.ste.config == STE_CONFIG_NESTED) &&
2235 !req.ste.s1stalld;
2236 uint32_t class_ = (req.ste.config == STE_CONFIG_S2) ? SMMUV3_CLASS_IN : SMMUV3_CLASS_TT;
2237 if (!ret.stallable_fault) {
2238 record_event_full(req.event_type, sid, substream_id, iova, txn, class_, req.fault_level, 0, false);
2239 }
2240 return ret;
2241 }
2242
2243 ret.translated_addr = req.pa;
2244 ret.perm = req.prot;
2245 ret.addr_mask = (1ULL << req.page_size) - 1;
2246 ret.descriptor = req.descriptor;
2247 ret.table_attrs = req.tableattrs;
2248
2249 IOTLBKey key;
2250 key.vmid = req.ste.vmid;
2251 key.iova = iova & ~ret.addr_mask;
2252 key.level = IOTLB_LEAF_LEVEL;
2253 key.secure = is_secure;
2254 key.regime = iotlb_regime;
2255
2256 if (req.ste.config == STE_CONFIG_S1 || req.ste.config == STE_CONFIG_NESTED) {
2257 key.asid = req.cd.asid;
2258 key.tg = static_cast<uint8_t>(req.cd.tg[0]);
2259 } else {
2260 key.asid = 0;
2261 key.tg = static_cast<uint8_t>(req.ste.s2tg);
2262 }
2263 iotlb_insert(key, ret);
2264
2265 return ret;
2266}
2267
2268template <unsigned int BUSWIDTH>
2269void smmuv3<BUSWIDTH>::iotlb_insert(const IOTLBKey& key, const IOMMUTLBEntry& entry)
2270{
2271 if (m_iotlb.find(key) != m_iotlb.end()) {
2272 m_iotlb_lru.remove(key);
2273 } else if (m_iotlb.size() >= p_iotlb_size.get_value()) {
2274 auto lru_key = m_iotlb_lru.back();
2275 m_iotlb_lru.pop_back();
2276 m_iotlb.erase(lru_key);
2277 }
2278 m_iotlb[key] = entry;
2279 m_iotlb_lru.push_front(key);
2280}
2281
2282template <unsigned int BUSWIDTH>
2283bool smmuv3<BUSWIDTH>::iotlb_lookup(const IOTLBKey& key, IOMMUTLBEntry& entry)
2284{
2285 auto it = m_iotlb.find(key);
2286 if (it != m_iotlb.end()) {
2287 entry = it->second;
2288 return true;
2289 }
2290 return false;
2291}
2292
2293template <unsigned int BUSWIDTH>
2294void smmuv3<BUSWIDTH>::iotlb_inv_all()
2295{
2296 m_iotlb.clear();
2297 m_iotlb_lru.clear();
2298}
2299
2300template <unsigned int BUSWIDTH>
2301void smmuv3<BUSWIDTH>::iotlb_inv_asid(uint16_t asid)
2302{
2303 for (auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2304 if (it->first.asid == asid) {
2305 m_iotlb_lru.remove(it->first);
2306 it = m_iotlb.erase(it);
2307 } else {
2308 ++it;
2309 }
2310 }
2311}
2312
2313template <unsigned int BUSWIDTH>
2314void smmuv3<BUSWIDTH>::iotlb_inv_vmid(uint16_t vmid)
2315{
2316 for (auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2317 if (it->first.vmid == vmid) {
2318 m_iotlb_lru.remove(it->first);
2319 it = m_iotlb.erase(it);
2320 } else {
2321 ++it;
2322 }
2323 }
2324}
2325
2326template <unsigned int BUSWIDTH>
2327void smmuv3<BUSWIDTH>::iotlb_inv_iova(uint16_t vmid, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t addr_mask)
2328{
2329 uint64_t iova_masked = iova & ~addr_mask;
2330 for (auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2331 uint64_t entry_masked = it->first.iova & ~addr_mask;
2332 if (it->first.vmid == vmid && it->first.asid == asid && entry_masked == iova_masked && it->first.tg == tg) {
2333 m_iotlb_lru.remove(it->first);
2334 it = m_iotlb.erase(it);
2335 } else {
2336 ++it;
2337 }
2338 }
2339}
2340
2341template <unsigned int BUSWIDTH>
2342void smmuv3<BUSWIDTH>::consume_cmdq()
2343{
2344 constexpr uint64_t CMDQ_BASE_LOG2_BITS_MASK = 0x1FULL;
2345
2346 uint64_t hi = static_cast<uint64_t>(CMDQ_BASE_HI) & CMDQ_BASE_HI_MASK;
2347 uint64_t lo = static_cast<uint64_t>(CMDQ_BASE_LO) & ~CMDQ_BASE_LOG2_BITS_MASK;
2348 uint64_t base = (hi << 32) | lo;
2349 uint32_t log2size = clamp_queue_log2size(static_cast<uint32_t>(CMDQ_BASE_LO[CMDQ_BASE_LOG2SIZE]));
2350 if (log2size == 0) return;
2351
2352 uint32_t prod = static_cast<uint32_t>(CMDQ_PROD);
2353 uint32_t cons = static_cast<uint32_t>(CMDQ_CONS);
2354 uint32_t wrap_mask = (1u << (log2size + 1)) - 1;
2355 uint32_t idx_mask = (1u << log2size) - 1;
2356
2357 SCP_INFO(()) << "SMMUv3: consume_cmdq base=0x" << std::hex << base << " log2size=" << std::dec << log2size
2358 << " prod=0x" << std::hex << prod << " cons=0x" << cons;
2359
2360 while ((prod & wrap_mask) != (cons & wrap_mask)) {
2361 uint32_t idx = cons & idx_mask;
2362 uint64_t cmd_addr = base + (idx * SMMUV3_CMD_SIZE);
2363 std::array<uint8_t, SMMUV3_CMD_SIZE> cmd{};
2364
2365 if (!dma_read(cmd_addr, cmd.data(), SMMUV3_CMD_SIZE)) {
2366 SCP_WARN(()) << "SMMUv3: dma_read failed at 0x" << std::hex << cmd_addr;
2367 CMDQ_CONS = cons;
2368 set_cmdq_err(SMMUV3_CERROR_ABT);
2369 return;
2370 }
2371
2372 uint8_t opcode = cmd[0];
2373 SCP_INFO(()) << "SMMUv3: cmd opcode=0x" << std::hex << (int)opcode << " at idx=" << std::dec << idx;
2374
2375 switch (opcode) {
2376 case CMD_OP_PREFETCH_CONFIG:
2377 handle_cmd_prefetch_config(cmd.data());
2378 break;
2379 case CMD_OP_PREFETCH_ADDR:
2380 handle_cmd_prefetch_addr(cmd.data());
2381 break;
2382 case CMD_OP_CFGI_STE:
2383 handle_cmd_cfgi_ste(cmd.data());
2384 break;
2385 case CMD_OP_CFGI_STE_RANGE:
2386 handle_cmd_cfgi_ste_range(cmd.data());
2387 break;
2388 case CMD_OP_CFGI_CD:
2389 handle_cmd_cfgi_cd(cmd.data());
2390 break;
2391 case CMD_OP_CFGI_CD_ALL:
2392 handle_cmd_cfgi_cd_all(cmd.data());
2393 break;
2394 case CMD_OP_CFGI_ALL:
2395 handle_cmd_cfgi_all(cmd.data());
2396 break;
2397 case CMD_OP_TLBI_NH_ALL:
2398 handle_cmd_tlbi_nh_all(cmd.data());
2399 break;
2400 case CMD_OP_TLBI_NH_ASID:
2401 handle_cmd_tlbi_nh_asid(cmd.data());
2402 break;
2403 case CMD_OP_TLBI_NH_VA:
2404 handle_cmd_tlbi_nh_va(cmd.data());
2405 break;
2406 case CMD_OP_TLBI_NH_VAA:
2407 handle_cmd_tlbi_nh_vaa(cmd.data());
2408 break;
2409 case CMD_OP_TLBI_EL3_ALL:
2410 handle_cmd_tlbi_nh_all(cmd.data());
2411 break;
2412 case CMD_OP_TLBI_EL3_VA:
2413 handle_cmd_tlbi_nh_va(cmd.data());
2414 break;
2415 case CMD_OP_TLBI_EL2_ALL:
2416 handle_cmd_tlbi_nh_all(cmd.data());
2417 break;
2418 case CMD_OP_TLBI_EL2_ASID:
2419 handle_cmd_tlbi_nh_asid(cmd.data());
2420 break;
2421 case CMD_OP_TLBI_EL2_VA:
2422 handle_cmd_tlbi_nh_va(cmd.data());
2423 break;
2424 case CMD_OP_TLBI_EL2_VAA:
2425 handle_cmd_tlbi_nh_vaa(cmd.data());
2426 break;
2427 case CMD_OP_TLBI_S12_VMALL:
2428 handle_cmd_tlbi_s12_vmall(cmd.data());
2429 break;
2430 case CMD_OP_TLBI_S2_IPA:
2431 handle_cmd_tlbi_s2_ipa(cmd.data());
2432 break;
2433 case CMD_OP_TLBI_NSNH_ALL:
2434 handle_cmd_tlbi_nsnh_all(cmd.data());
2435 break;
2436 case CMD_OP_ATC_INV:
2437 handle_cmd_atc_inv(cmd.data());
2438 break;
2439 case CMD_OP_PRI_RESP:
2440 handle_cmd_pri_resp(cmd.data());
2441 break;
2442 case CMD_OP_RESUME:
2443 handle_cmd_resume(cmd.data());
2444 break;
2445 case CMD_OP_STALL_TERM:
2446 break;
2447 case CMD_OP_SYNC:
2448 handle_cmd_sync(cmd.data());
2449 break;
2450 default:
2451 SCP_WARN(()) << "SMMUv3: unknown opcode 0x" << std::hex << (int)opcode;
2452 CMDQ_CONS = cons;
2453 set_cmdq_err(SMMUV3_CERROR_ILL);
2454 return;
2455 }
2456
2457 cons = (cons + 1) & wrap_mask;
2458 }
2459
2460 CMDQ_CONS = cons;
2461 SCP_INFO(()) << "SMMUv3: consume_cmdq done, CMDQ_CONS=0x" << std::hex << cons;
2462}
2463
2464template <unsigned int BUSWIDTH>
2465void smmuv3<BUSWIDTH>::consume_secure_cmdq()
2466{
2467 constexpr uint64_t CMDQ_BASE_LOG2_BITS_MASK = 0x1FULL;
2468
2469 uint64_t hi = static_cast<uint64_t>(S_CMDQ_BASE_HI) & CMDQ_BASE_HI_MASK;
2470 uint64_t lo = static_cast<uint64_t>(S_CMDQ_BASE_LO) & ~CMDQ_BASE_LOG2_BITS_MASK;
2471 uint64_t base = (hi << 32) | lo;
2472 uint32_t log2size = clamp_queue_log2size(static_cast<uint32_t>(S_CMDQ_BASE_LO[S_CMDQ_BASE_LOG2SIZE]));
2473 if (log2size == 0) return;
2474
2475 uint32_t prod = static_cast<uint32_t>(S_CMDQ_PROD);
2476 uint32_t cons = static_cast<uint32_t>(S_CMDQ_CONS);
2477 uint32_t wrap_mask = (1u << (log2size + 1)) - 1;
2478 uint32_t idx_mask = (1u << log2size) - 1;
2479
2480 while ((prod & wrap_mask) != (cons & wrap_mask)) {
2481 uint32_t idx = cons & idx_mask;
2482 uint64_t cmd_addr = base + (idx * SMMUV3_CMD_SIZE);
2483 std::array<uint8_t, SMMUV3_CMD_SIZE> cmd{};
2484
2485 if (!dma_read(cmd_addr, cmd.data(), SMMUV3_CMD_SIZE)) {
2486 S_CMDQ_CONS = cons;
2487 set_secure_cmdq_err(SMMUV3_CERROR_ABT);
2488 return;
2489 }
2490
2491 constexpr size_t CMD_SID_OFFSET = 4;
2492 constexpr size_t CMD_WORD1_OFFSET = 8;
2493 constexpr uint32_t CMD_SYNC_CS_SHIFT_LOCAL = 12;
2494 constexpr uint32_t CMD_SYNC_CS_MASK_LOCAL = 0x3;
2495
2496 uint8_t opcode = cmd[0];
2497 switch (opcode) {
2498 case CMD_OP_PREFETCH_CONFIG:
2499 case CMD_OP_PREFETCH_ADDR:
2500 break;
2501 case CMD_OP_CFGI_STE: {
2502 uint32_t sid = load_le<uint32_t>(cmd.data() + CMD_SID_OFFSET);
2503 m_ste_cache_secure.erase(sid);
2504 break;
2505 }
2506 case CMD_OP_CFGI_STE_RANGE:
2507 m_ste_cache_secure.clear();
2508 break;
2509 case CMD_OP_CFGI_CD:
2510 case CMD_OP_CFGI_CD_ALL:
2511 m_cd_cache_secure.clear();
2512 break;
2513 case CMD_OP_CFGI_ALL:
2514 m_ste_cache_secure.clear();
2515 m_cd_cache_secure.clear();
2516 break;
2517 case CMD_OP_TLBI_NSNH_ALL:
2518 case CMD_OP_TLBI_NH_ALL:
2519 case CMD_OP_TLBI_NH_ASID:
2520 case CMD_OP_TLBI_NH_VA:
2521 case CMD_OP_TLBI_NH_VAA:
2522 case CMD_OP_TLBI_S12_VMALL:
2523 case CMD_OP_TLBI_S2_IPA:
2524 iotlb_inv_all();
2525 break;
2526 case CMD_OP_SYNC: {
2527 uint64_t dw0 = load_le<uint64_t>(cmd.data());
2528 uint64_t dw1 = load_le<uint64_t>(cmd.data() + CMD_WORD1_OFFSET);
2529 uint32_t cs = (static_cast<uint32_t>(dw0) >> CMD_SYNC_CS_SHIFT_LOCAL) & CMD_SYNC_CS_MASK_LOCAL;
2530 if (cs == SYNC_CS_IRQ) {
2531 constexpr uint64_t CMD_SYNC_MSIADDR_MASK = 0x000FFFFFFFFFFFFCULL;
2532 constexpr uint32_t CMD_SYNC_MSIDATA_SHIFT = 32;
2533 uint64_t msiaddr = dw1 & CMD_SYNC_MSIADDR_MASK;
2534 uint32_t msidata = static_cast<uint32_t>(dw0 >> CMD_SYNC_MSIDATA_SHIFT);
2535 if (msiaddr != 0) {
2536 if (!dma_write(msiaddr, &msidata, sizeof(msidata))) {
2537 set_secure_gerror(GERROR_BIT_MSI_CMDQ_ABT);
2538 }
2539 }
2540 }
2541 break;
2542 }
2543 default:
2544 S_CMDQ_CONS = cons;
2545 set_secure_cmdq_err(SMMUV3_CERROR_ILL);
2546 return;
2547 }
2548
2549 cons = (cons + 1) & wrap_mask;
2550 }
2551
2552 S_CMDQ_CONS = cons;
2553}
2554
2555template <unsigned int BUSWIDTH>
2556void smmuv3<BUSWIDTH>::record_secure_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info)
2557{
2558 if (static_cast<uint32_t>(S_CR0[S_CR0_EVENTQEN]) == 0) return;
2559
2560 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2561 constexpr size_t EVT_SID_OFFSET = 4;
2562 constexpr size_t EVT_IOVA_OFFSET = 8;
2563 constexpr size_t EVT_INFO_OFFSET = 16;
2564
2565 uint64_t base = (static_cast<uint64_t>(S_EVENTQ_BASE_HI) << 32) |
2566 (static_cast<uint64_t>(S_EVENTQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2567 uint32_t log2size = clamp_queue_log2size(static_cast<uint32_t>(S_EVENTQ_BASE_LO[S_EVENTQ_BASE_LOG2SIZE]));
2568 uint32_t prod = static_cast<uint32_t>(S_EVENTQ_PROD);
2569 uint32_t cons = static_cast<uint32_t>(S_EVENTQ_CONS);
2570
2571 if (queue_full(prod, cons, log2size)) {
2572 set_secure_gerror(GERROR_BIT_EVENTQ_ABT);
2573 return;
2574 }
2575
2576 uint32_t idx = prod & ((1u << log2size) - 1);
2577 uint64_t evt_addr = base + (idx * SMMUV3_EVENT_SIZE);
2578 std::array<uint8_t, SMMUV3_EVENT_SIZE> evt{};
2579
2580 evt[0] = type;
2581 store_le<uint32_t>(evt.data() + EVT_SID_OFFSET, sid);
2582 store_le<uint64_t>(evt.data() + EVT_IOVA_OFFSET, iova);
2583 store_le<uint32_t>(evt.data() + EVT_INFO_OFFSET, info);
2584
2585 if (!dma_write(evt_addr, evt.data(), SMMUV3_EVENT_SIZE)) {
2586 set_secure_gerror(GERROR_BIT_EVENTQ_ABT);
2587 return;
2588 }
2589
2590 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2591 S_EVENTQ_PROD = prod;
2592 update_secure_irq_levels();
2593}
2594
2595template <unsigned int BUSWIDTH>
2596bool smmuv3<BUSWIDTH>::queue_full(uint32_t prod, uint32_t cons, uint32_t size)
2597{
2598 size = clamp_queue_log2size(size);
2599 uint32_t mask = (1u << size) - 1;
2600 uint32_t wrap = 1u << size;
2601 return ((prod ^ cons) == wrap) && ((prod & mask) == (cons & mask));
2602}
2603
2604template <unsigned int BUSWIDTH>
2605void smmuv3<BUSWIDTH>::record_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info)
2606{
2607 if (static_cast<uint32_t>(CR0[CR0_EVENTQEN]) == 0) {
2608 return;
2609 }
2610
2611 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2612 constexpr size_t EVT_SID_OFFSET = 4;
2613 constexpr size_t EVT_IOVA_OFFSET = 8;
2614 constexpr size_t EVT_INFO_OFFSET = 16;
2615
2616 uint64_t base = (static_cast<uint64_t>(EVENTQ_BASE_HI) << 32) |
2617 (static_cast<uint64_t>(EVENTQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2618 uint32_t log2size = clamp_queue_log2size(static_cast<uint32_t>(EVENTQ_BASE_LO[EVENTQ_BASE_LOG2SIZE]));
2619 uint32_t prod = static_cast<uint32_t>(EVENTQ_PROD);
2620 uint32_t cons = static_cast<uint32_t>(EVENTQ_CONS);
2621
2622 if (queue_full(prod, cons, log2size)) {
2623 set_gerror(GERROR_BIT_EVENTQ_ABT);
2624 return;
2625 }
2626
2627 uint32_t idx = prod & ((1u << log2size) - 1);
2628 uint64_t evt_addr = base + (idx * SMMUV3_EVENT_SIZE);
2629 std::array<uint8_t, SMMUV3_EVENT_SIZE> evt{};
2630
2631 evt[0] = type;
2632 store_le<uint32_t>(evt.data() + EVT_SID_OFFSET, sid);
2633 store_le<uint64_t>(evt.data() + EVT_IOVA_OFFSET, iova);
2634 store_le<uint32_t>(evt.data() + EVT_INFO_OFFSET, info);
2635
2636 if (!dma_write(evt_addr, evt.data(), SMMUV3_EVENT_SIZE)) {
2637 set_gerror(GERROR_BIT_EVENTQ_ABT);
2638 return;
2639 }
2640
2641 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2642 EVENTQ_PROD = prod;
2643 update_irq_levels();
2644}
2645
2646template <unsigned int BUSWIDTH>
2647bool smmuv3<BUSWIDTH>::stall_and_wait(uint32_t sid, uint32_t ssid, uint64_t iova, uint32_t fault_type,
2648 uint32_t fault_level, uint32_t class_, tlm::tlm_generic_payload& txn)
2649{
2650 uint32_t stag = m_next_stag++;
2651 auto& stalled = m_stalled_txns[stag];
2652 stalled.stag = stag;
2653 stalled.sid = sid;
2654 stalled.substream_id = ssid;
2655 stalled.aborted = false;
2656
2657 record_event_full(fault_type, sid, ssid, iova, txn, class_, fault_level, stag, true);
2658
2659 sc_core::wait(stalled.resume_event);
2660
2661 bool aborted = stalled.aborted;
2662 m_stalled_txns.erase(stag);
2663 return aborted;
2664}
2665
2666template <unsigned int BUSWIDTH>
2667void smmuv3<BUSWIDTH>::record_event_full(uint32_t type, uint32_t sid, uint32_t ssid, uint64_t iova,
2668 tlm::tlm_generic_payload& txn, uint32_t class_, uint32_t fault_level,
2669 uint32_t stag, bool stall)
2670{
2671 if (static_cast<uint32_t>(CR0[CR0_EVENTQEN]) == 0) return;
2672
2673 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2674 uint64_t base = (static_cast<uint64_t>(EVENTQ_BASE_HI) << 32) |
2675 (static_cast<uint64_t>(EVENTQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2676 uint32_t log2size = clamp_queue_log2size(static_cast<uint32_t>(EVENTQ_BASE_LO[EVENTQ_BASE_LOG2SIZE]));
2677 uint32_t prod = static_cast<uint32_t>(EVENTQ_PROD);
2678 uint32_t cons = static_cast<uint32_t>(EVENTQ_CONS);
2679
2680 if (queue_full(prod, cons, log2size)) {
2681 set_gerror(GERROR_BIT_EVENTQ_ABT);
2682 return;
2683 }
2684
2685 uint32_t idx = prod & ((1u << log2size) - 1);
2686 uint64_t evt_addr = base + (idx * SMMUV3_EVENT_SIZE);
2687 std::array<uint64_t, 4> evt{};
2688
2689 bool rnw = (txn.get_command() == tlm::TLM_READ_COMMAND);
2690 bool ssv = (ssid != 0);
2691
2692 evt[0] = (static_cast<uint64_t>(type) & EVT_W0_TYPE_MASK) | (ssv ? (1ULL << EVT_W0_SSV_BIT) : 0ULL) |
2693 ((static_cast<uint64_t>(ssid) & EVT_W0_SSID_MASK) << EVT_W0_SSID_SHIFT) |
2694 (static_cast<uint64_t>(sid) << EVT_W0_STREAMID_SHIFT);
2695
2696 evt[1] = (static_cast<uint64_t>(stag) & EVT_W1_STAG_MASK) | (rnw ? (1ULL << EVT_W1_RNW_BIT) : 0ULL) |
2697 ((static_cast<uint64_t>(class_) & EVT_W1_CLASS_MASK) << EVT_W1_CLASS_SHIFT) |
2698 (stall ? (1ULL << EVT_W1_STALL_BIT) : 0ULL) |
2699 ((static_cast<uint64_t>(fault_level) & EVT_W1_REASON_MASK) << EVT_W1_REASON_SHIFT);
2700
2701 evt[2] = iova;
2702 evt[3] = 0;
2703
2704 if (!dma_write(evt_addr, evt.data(), SMMUV3_EVENT_SIZE)) {
2705 set_gerror(GERROR_BIT_EVENTQ_ABT);
2706 return;
2707 }
2708
2709 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2710 EVENTQ_PROD = prod;
2711 update_irq_levels();
2712}
2713
2714template <unsigned int BUSWIDTH>
2715void smmuv3<BUSWIDTH>::record_pri(uint32_t sid, uint64_t iova, uint32_t flags)
2716{
2717 if (static_cast<uint32_t>(CR0[CR0_PRIQEN]) == 0) {
2718 return;
2719 }
2720
2721 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2722 constexpr size_t PRI_SID_OFFSET = 0;
2723 constexpr size_t PRI_FLAGS_OFFSET = 4;
2724 constexpr size_t PRI_IOVA_OFFSET = 8;
2725
2726 uint64_t base = (static_cast<uint64_t>(PRIQ_BASE_HI) << 32) |
2727 (static_cast<uint64_t>(PRIQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2728 uint32_t log2size = clamp_queue_log2size(static_cast<uint32_t>(PRIQ_BASE_LO[PRIQ_BASE_LOG2SIZE]));
2729 uint32_t prod = static_cast<uint32_t>(PRIQ_PROD);
2730 uint32_t cons = static_cast<uint32_t>(PRIQ_CONS);
2731
2732 if (queue_full(prod, cons, log2size)) {
2733 set_gerror(GERROR_BIT_PRIQ_ABT);
2734 return;
2735 }
2736
2737 uint32_t idx = prod & ((1u << log2size) - 1);
2738 uint64_t pri_addr = base + (idx * SMMUV3_PRI_SIZE);
2739 std::array<uint8_t, SMMUV3_PRI_SIZE> pri{};
2740
2741 store_le<uint32_t>(pri.data() + PRI_SID_OFFSET, sid);
2742 store_le<uint32_t>(pri.data() + PRI_FLAGS_OFFSET, flags);
2743 store_le<uint64_t>(pri.data() + PRI_IOVA_OFFSET, iova);
2744
2745 if (!dma_write(pri_addr, pri.data(), SMMUV3_PRI_SIZE)) {
2746 set_gerror(GERROR_BIT_PRIQ_ABT);
2747 return;
2748 }
2749
2750 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2751 PRIQ_PROD = prod;
2752 update_irq_levels();
2753}
2754
2755template <unsigned int BUSWIDTH>
2756void smmuv3<BUSWIDTH>::handle_cmd_prefetch_config(const uint8_t*)
2757{
2758}
2759
2760template <unsigned int BUSWIDTH>
2761void smmuv3<BUSWIDTH>::handle_cmd_prefetch_addr(const uint8_t*)
2762{
2763}
2764
2765namespace detail_smmuv3 {
2766constexpr size_t CMD_SID_OFFSET = 4;
2767constexpr size_t CMD_ASID_OFFSET = 6;
2768constexpr size_t CMD_WORD1_OFFSET = 8;
2769constexpr size_t CMD_STAG_OFFSET = 4;
2770constexpr size_t CMD_RESUME_RESP_OFFSET = 11;
2771constexpr uint32_t CMD_WORD0_TG_SHIFT = 10;
2772constexpr uint32_t CMD_WORD0_TG_MASK = 0x3;
2773constexpr uint32_t CMD_WORD0_SCALE_SHIFT = 20;
2774constexpr uint32_t CMD_WORD0_SCALE_MASK = 0xF;
2775constexpr uint32_t CMD_WORD0_VMID_SHIFT = 32;
2776constexpr uint32_t CMD_WORD0_VMID_MASK = 0xFFFF;
2777constexpr uint32_t CMD_WORD0_ASID_SHIFT = 48;
2778constexpr uint32_t CMD_WORD0_ASID_MASK = 0xFFFF;
2779constexpr uint32_t CMD_SYNC_CS_SHIFT = 12;
2780constexpr uint32_t CMD_SYNC_CS_MASK = 0x3;
2781constexpr uint64_t CMD_VA_LOW_MASK = 0xFFFULL;
2782
2784 uint32_t page_shift;
2785 uint8_t key_tg;
2786};
2787
2788inline TlbiTgDecode decode_tlbi_tg(uint8_t tlbi_tg)
2789{
2790 switch (tlbi_tg) {
2791 case TLBI_TG_4K:
2792 return { PAGE_SHIFT_4K, CD_TG_4K };
2793 case TLBI_TG_16K:
2794 return { PAGE_SHIFT_16K, CD_TG_16K };
2795 case TLBI_TG_64K:
2796 return { PAGE_SHIFT_64K, CD_TG_64K };
2797 default:
2798 return { PAGE_SHIFT_4K, CD_TG_4K };
2799 }
2800}
2801} // namespace detail_smmuv3
2802
2803template <unsigned int BUSWIDTH>
2804void smmuv3<BUSWIDTH>::handle_cmd_cfgi_ste(const uint8_t* cmd)
2805{
2806 uint32_t sid = load_le<uint32_t>(cmd + detail_smmuv3::CMD_SID_OFFSET);
2807 m_ste_cache.erase(sid);
2808 m_ste_cache_lru.remove(sid);
2809}
2810
2811template <unsigned int BUSWIDTH>
2812void smmuv3<BUSWIDTH>::handle_cmd_cfgi_ste_range(const uint8_t*)
2813{
2814 m_ste_cache.clear();
2815 m_ste_cache_lru.clear();
2816}
2817
2818template <unsigned int BUSWIDTH>
2819void smmuv3<BUSWIDTH>::handle_cmd_cfgi_cd(const uint8_t*)
2820{
2821 m_cd_cache.clear();
2822 m_cd_cache_lru.clear();
2823}
2824
2825template <unsigned int BUSWIDTH>
2826void smmuv3<BUSWIDTH>::handle_cmd_cfgi_cd_all(const uint8_t*)
2827{
2828 m_cd_cache.clear();
2829 m_cd_cache_lru.clear();
2830}
2831
2832template <unsigned int BUSWIDTH>
2833void smmuv3<BUSWIDTH>::handle_cmd_cfgi_all(const uint8_t*)
2834{
2835 m_ste_cache.clear();
2836 m_ste_cache_lru.clear();
2837 m_cd_cache.clear();
2838 m_cd_cache_lru.clear();
2839}
2840
2841template <unsigned int BUSWIDTH>
2842void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_all(const uint8_t*)
2843{
2844 iotlb_inv_all();
2845}
2846
2847template <unsigned int BUSWIDTH>
2848void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_asid(const uint8_t* cmd)
2849{
2850 uint16_t asid = load_le<uint16_t>(cmd + detail_smmuv3::CMD_ASID_OFFSET);
2851 iotlb_inv_asid(asid);
2852}
2853
2854template <unsigned int BUSWIDTH>
2855void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_va(const uint8_t* cmd)
2856{
2857 uint64_t word0 = load_le<uint64_t>(cmd);
2858 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2859 uint16_t asid = (word0 >> detail_smmuv3::CMD_WORD0_ASID_SHIFT) & detail_smmuv3::CMD_WORD0_ASID_MASK;
2860 uint8_t tlbi_tg = (word0 >> detail_smmuv3::CMD_WORD0_TG_SHIFT) & detail_smmuv3::CMD_WORD0_TG_MASK;
2861 uint8_t scale = (word0 >> detail_smmuv3::CMD_WORD0_SCALE_SHIFT) & detail_smmuv3::CMD_WORD0_SCALE_MASK;
2862 uint64_t va = word1 & ~detail_smmuv3::CMD_VA_LOW_MASK;
2863
2864 detail_smmuv3::TlbiTgDecode tg = detail_smmuv3::decode_tlbi_tg(tlbi_tg);
2865 uint64_t mask = safe_shl1_u64(tg.page_shift + scale) - 1;
2866 iotlb_inv_iova(0, asid, va, tg.key_tg, mask);
2867}
2868
2869template <unsigned int BUSWIDTH>
2870void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_vaa(const uint8_t* cmd)
2871{
2872 uint64_t word0 = load_le<uint64_t>(cmd);
2873 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2874 uint8_t tlbi_tg = (word0 >> detail_smmuv3::CMD_WORD0_TG_SHIFT) & detail_smmuv3::CMD_WORD0_TG_MASK;
2875 uint8_t scale = (word0 >> detail_smmuv3::CMD_WORD0_SCALE_SHIFT) & detail_smmuv3::CMD_WORD0_SCALE_MASK;
2876 uint64_t va = word1 & ~detail_smmuv3::CMD_VA_LOW_MASK;
2877
2878 detail_smmuv3::TlbiTgDecode tg = detail_smmuv3::decode_tlbi_tg(tlbi_tg);
2879 uint64_t mask = safe_shl1_u64(tg.page_shift + scale) - 1;
2880 uint64_t va_masked = va & ~mask;
2881 uint8_t key_tg = tg.key_tg;
2882
2883 for (auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2884 uint64_t entry_masked = it->first.iova & ~mask;
2885 if (entry_masked == va_masked && it->first.tg == key_tg) {
2886 m_iotlb_lru.remove(it->first);
2887 it = m_iotlb.erase(it);
2888 } else {
2889 ++it;
2890 }
2891 }
2892}
2893
2894template <unsigned int BUSWIDTH>
2895void smmuv3<BUSWIDTH>::handle_cmd_atc_inv(const uint8_t* cmd)
2896{
2897 constexpr uint32_t CMD_WORD0_SID_SHIFT = 32;
2898 constexpr uint64_t CMD_ATC_SIZE_MASK = 0x3FULL;
2899 constexpr uint64_t CMD_ATC_ADDR_MASK = ~((1ULL << PAGE_SHIFT_4K) - 1ULL);
2900
2901 uint64_t word0 = load_le<uint64_t>(cmd);
2902 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2903 uint32_t sid = static_cast<uint32_t>(word0 >> CMD_WORD0_SID_SHIFT);
2904 uint8_t size = word1 & CMD_ATC_SIZE_MASK;
2905 uint64_t addr_lo = word1 & CMD_ATC_ADDR_MASK;
2906
2907 uint64_t range = safe_shl1_u64(size + PAGE_SHIFT_4K);
2908 uint64_t end = addr_lo + range - 1;
2909
2910 SCP_INFO(()) << "SMMUv3: ATC_INV sid=0x" << std::hex << sid << " addr=0x" << addr_lo << " range=0x" << range;
2911
2912 for (auto* tbu : tbus) {
2913 if (!tbu) continue;
2914 tbu->upstream_invalidate(addr_lo, end);
2915 }
2916}
2917
2918template <unsigned int BUSWIDTH>
2919void smmuv3<BUSWIDTH>::handle_cmd_tlbi_s12_vmall(const uint8_t* cmd)
2920{
2921 uint64_t word0 = load_le<uint64_t>(cmd);
2922 uint16_t vmid = (word0 >> detail_smmuv3::CMD_WORD0_VMID_SHIFT) & detail_smmuv3::CMD_WORD0_VMID_MASK;
2923 iotlb_inv_vmid(vmid);
2924}
2925
2926template <unsigned int BUSWIDTH>
2927void smmuv3<BUSWIDTH>::handle_cmd_tlbi_s2_ipa(const uint8_t* cmd)
2928{
2929 uint64_t word0 = load_le<uint64_t>(cmd);
2930 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2931 uint16_t vmid = (word0 >> detail_smmuv3::CMD_WORD0_VMID_SHIFT) & detail_smmuv3::CMD_WORD0_VMID_MASK;
2932 uint8_t tlbi_tg = (word0 >> detail_smmuv3::CMD_WORD0_TG_SHIFT) & detail_smmuv3::CMD_WORD0_TG_MASK;
2933 uint8_t scale = (word0 >> detail_smmuv3::CMD_WORD0_SCALE_SHIFT) & detail_smmuv3::CMD_WORD0_SCALE_MASK;
2934 uint64_t ipa = word1 & IPA_ADDR_MASK;
2935
2936 detail_smmuv3::TlbiTgDecode tg = detail_smmuv3::decode_tlbi_tg(tlbi_tg);
2937 uint64_t mask = safe_shl1_u64(tg.page_shift + scale) - 1;
2938 uint64_t ipa_masked = ipa & ~mask;
2939 uint8_t key_tg = tg.key_tg;
2940
2941 for (auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2942 uint64_t entry_masked = it->first.iova & ~mask;
2943 if (it->first.vmid == vmid && entry_masked == ipa_masked && it->first.tg == key_tg) {
2944 m_iotlb_lru.remove(it->first);
2945 it = m_iotlb.erase(it);
2946 } else {
2947 ++it;
2948 }
2949 }
2950}
2951
2952template <unsigned int BUSWIDTH>
2953void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nsnh_all(const uint8_t*)
2954{
2955 iotlb_inv_all();
2956}
2957
2958template <unsigned int BUSWIDTH>
2959void smmuv3<BUSWIDTH>::handle_cmd_sync(const uint8_t* cmd)
2960{
2961 uint64_t dw0 = load_le<uint64_t>(cmd);
2962 uint64_t dw1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2963
2964 uint32_t cs = (static_cast<uint32_t>(dw0) >> detail_smmuv3::CMD_SYNC_CS_SHIFT) & detail_smmuv3::CMD_SYNC_CS_MASK;
2965
2966 if (cs == SYNC_CS_IRQ) {
2967 constexpr uint64_t CMD_SYNC_MSIADDR_MASK = 0x000FFFFFFFFFFFFCULL;
2968 constexpr uint32_t CMD_SYNC_MSIDATA_SHIFT = 32;
2969 uint64_t msiaddr = dw1 & CMD_SYNC_MSIADDR_MASK;
2970 uint32_t msidata = static_cast<uint32_t>(dw0 >> CMD_SYNC_MSIDATA_SHIFT);
2971 if (msiaddr != 0) {
2972 if (!dma_write(msiaddr, &msidata, sizeof(msidata))) {
2973 set_gerror(GERROR_BIT_MSI_CMDQ_ABT);
2974 }
2975 }
2976 trigger_irq(irq_cmd_sync);
2977 }
2978}
2979
2980template <unsigned int BUSWIDTH>
2981void smmuv3<BUSWIDTH>::handle_cmd_resume(const uint8_t* cmd)
2982{
2983 uint32_t stag = load_le<uint32_t>(cmd + detail_smmuv3::CMD_STAG_OFFSET);
2984 uint8_t resp = cmd[detail_smmuv3::CMD_RESUME_RESP_OFFSET];
2985
2986 auto it = m_stalled_txns.find(stag);
2987 if (it != m_stalled_txns.end()) {
2988 if (resp != 0) {
2989 it->second.aborted = true;
2990 }
2991 it->second.resume_event.notify();
2992 }
2993}
2994
2995template <unsigned int BUSWIDTH>
2996void smmuv3<BUSWIDTH>::handle_cmd_pri_resp(const uint8_t*)
2997{
2998}
2999
3000template <unsigned int BUSWIDTH>
3001void smmuv3<BUSWIDTH>::do_gatos()
3002{
3003 GATOS_CTRL[GATOS_CTRL_INPRG] = 1;
3004
3005 uint32_t sid = static_cast<uint32_t>(GATOS_SID);
3006 uint64_t iova = (static_cast<uint64_t>(GATOS_ADDR_HI) << 32) | static_cast<uint64_t>(GATOS_ADDR_LO);
3007 bool wr = static_cast<uint32_t>(GATOS_CTRL[GATOS_CTRL_READ_NWRITE]) == 0;
3008
3009 tlm::tlm_generic_payload dummy_txn;
3010 dummy_txn.set_command(wr ? tlm::TLM_WRITE_COMMAND : tlm::TLM_READ_COMMAND);
3011 dummy_txn.set_address(iova);
3012
3013 IOMMUTLBEntry te = smmuv3_translate(dummy_txn, sid, 0);
3014
3015 if (te.perm == IOMMUAccessFlags::NONE) {
3016 GATOS_PAR_LO[GATOS_PAR_F] = 1;
3017 GATOS_PAR_LO[GATOS_PAR_FST] = GATOS_PAR_FAULT_TRANSLATION;
3018 } else {
3019 GATOS_PAR_LO = static_cast<uint32_t>(te.translated_addr);
3020 GATOS_PAR_HI = static_cast<uint32_t>(te.translated_addr >> 32);
3021 GATOS_PAR_LO[GATOS_PAR_F] = 0;
3022 }
3023
3024 GATOS_CTRL[GATOS_CTRL_RUN] = 0;
3025 GATOS_CTRL[GATOS_CTRL_INPRG] = 0;
3026}
3027
3028} // namespace gs
3029
3030#endif
Definition target-signal-socket.h:163
Definition reg_model_maker.h:457
Definition reg_model_maker.h:35
Definition smmuv3_extensions.h:35
Definition smmuv3_gen.h:21
Definition smmuv3_memory_attrs.h:19
Definition smmuv3_extensions.h:18
Definition smmuv3.h:907
Definition smmuv3.h:336
Tool which reads a Lua configuration file and sets parameters.
Definition biflow.cc:10
Definition smmuv3.h:2783
Definition smmuv3.h:346