10const void* smmuv3_zip_archive_data() noexcept;
11unsigned int smmuv3_zip_archive_size() noexcept;
15#include <cci_configuration>
17#include <scp/report.h>
19#include <module_factory_registery.h>
20#include <ports/initiator-signal-socket.h>
21#include <ports/target-signal-socket.h>
22#include <tlm_utils/simple_initiator_socket.h>
23#include <tlm_utils/simple_target_socket.h>
24#include <tlm-extensions/underlying-dmi.h>
25#include <tlm-extensions/qemu-memtx-attrs.h>
27#include <reg_model_maker/reg_model_maker.h>
28#include <zip_loader.h>
37#include <unordered_map>
40#include "smmuv3_gen.h"
41#include "smmuv3_memory_attrs.h"
42#include "smmuv3_extensions.h"
46constexpr uint64_t SMMUV3_PAGESIZE = 4096;
47constexpr uint64_t SMMUV3_PAGEMASK = SMMUV3_PAGESIZE - 1;
52constexpr size_t SMMUV3_STE_SIZE = 64;
55constexpr size_t SMMUV3_CMD_SIZE = 16;
56constexpr size_t SMMUV3_EVENT_SIZE = 32;
57constexpr size_t SMMUV3_PRI_SIZE = 16;
62constexpr uint32_t SMMUV3_EVT_F_UUT = 0x01;
63constexpr uint32_t SMMUV3_EVT_C_BAD_STREAMID = 0x02;
64constexpr uint32_t SMMUV3_EVT_F_STE_FETCH = 0x03;
65constexpr uint32_t SMMUV3_EVT_C_BAD_STE = 0x04;
66constexpr uint32_t SMMUV3_EVT_C_BAD_SUBSTREAMID = 0x08;
67constexpr uint32_t SMMUV3_EVT_F_CD_FETCH = 0x09;
68constexpr uint32_t SMMUV3_EVT_C_BAD_CD = 0x0A;
69constexpr uint32_t SMMUV3_EVT_F_WALK_EABT = 0x0B;
70constexpr uint32_t SMMUV3_EVT_F_TRANSLATION = 0x10;
71constexpr uint32_t SMMUV3_EVT_F_ADDR_SIZE = 0x11;
72constexpr uint32_t SMMUV3_EVT_F_ACCESS = 0x12;
73constexpr uint32_t SMMUV3_EVT_F_PERMISSION = 0x13;
74constexpr uint32_t SMMUV3_EVT_F_STALL = 0x29;
77constexpr uint8_t SMMUV3_CERROR_NONE = 0;
78constexpr uint8_t SMMUV3_CERROR_ILL = 1;
79constexpr uint8_t SMMUV3_CERROR_ABT = 2;
80constexpr uint8_t SMMUV3_CERROR_ATC_INV_SYNC = 3;
83constexpr uint32_t SMMUV3_CLASS_TT = 0;
84constexpr uint32_t SMMUV3_CLASS_IN = 1;
85constexpr uint32_t SMMUV3_CLASS_CD = 2;
88constexpr uint32_t EVT_W0_TYPE_MASK = 0xFFu;
89constexpr uint32_t EVT_W0_SSV_BIT = 12;
90constexpr uint32_t EVT_W0_SSID_SHIFT = 13;
91constexpr uint32_t EVT_W0_SSID_MASK = 0xFFFFFu;
92constexpr uint32_t EVT_W0_STREAMID_SHIFT = 32;
94 EVT_W1_STAG_MASK = 0xFFFFu;
95constexpr uint32_t EVT_W1_RNW_BIT = 17;
96constexpr uint32_t EVT_W1_CLASS_SHIFT = 20;
97constexpr uint32_t EVT_W1_CLASS_MASK = 0x3u;
98constexpr uint32_t EVT_W1_STALL_BIT = 23;
99constexpr uint32_t EVT_W1_REASON_SHIFT = 24;
100constexpr uint32_t EVT_W1_REASON_MASK = 0x7u;
104constexpr uint32_t GERROR_BIT_MSI_CMDQ_ABT = 4;
105constexpr uint32_t GERROR_BIT_MSI_EVENTQ_ABT = 5;
106constexpr uint32_t GERROR_BIT_MSI_PRIQ_ABT = 6;
107constexpr uint32_t GERROR_BIT_MSI_GERROR_ABT = 7;
110constexpr uint32_t S1FMT_LINEAR = 0;
111constexpr uint32_t S1FMT_4KB_L2 = 1;
112constexpr uint32_t S1FMT_64KB_L2 = 2;
113constexpr uint32_t S1FMT_4KB_L2_SPLIT = 4;
114constexpr uint32_t S1FMT_64KB_L2_SPLIT = 8;
117constexpr uint32_t DESC_BIT_AF = 10;
124static inline T load_le(
const uint8_t* buf)
127 std::memcpy(&v, buf,
sizeof(T));
132static inline void store_le(uint8_t* buf, T v)
134 std::memcpy(buf, &v,
sizeof(T));
137static inline unsigned clamp_shift_u64(
unsigned shift)
139 constexpr unsigned MAX_SHIFT = 63;
140 return shift > MAX_SHIFT ? MAX_SHIFT : shift;
143static inline uint64_t safe_shl1_u64(
unsigned shift) {
return 1ULL << clamp_shift_u64(shift); }
145static inline uint32_t clamp_queue_log2size(uint32_t log2size)
147 constexpr uint32_t QUEUE_LOG2_MAX = 19;
148 return log2size > QUEUE_LOG2_MAX ? QUEUE_LOG2_MAX : log2size;
152constexpr uint32_t STE_CONFIG_ABORT = 0;
153constexpr uint32_t STE_CONFIG_BYPASS = 4;
154constexpr uint32_t STE_CONFIG_S1 = 5;
155constexpr uint32_t STE_CONFIG_S2 = 6;
156constexpr uint32_t STE_CONFIG_NESTED = 7;
159constexpr uint8_t CD_TG_4K = 0;
160constexpr uint8_t CD_TG_64K = 1;
161constexpr uint8_t CD_TG_16K = 2;
165constexpr uint8_t TLBI_TG_DC = 0;
166constexpr uint8_t TLBI_TG_4K = 1;
167constexpr uint8_t TLBI_TG_16K = 2;
168constexpr uint8_t TLBI_TG_64K = 3;
171constexpr uint32_t PAGE_SHIFT_4K = 12;
172constexpr uint32_t PAGE_SHIFT_16K = 14;
173constexpr uint32_t PAGE_SHIFT_64K = 16;
176constexpr uint32_t GRAINSIZE_4K = 12;
177constexpr uint32_t GRAINSIZE_16K = 14;
178constexpr uint32_t GRAINSIZE_64K = 16;
181constexpr uint32_t STRIDE_4K = 9;
182constexpr uint32_t STRIDE_16K = 11;
183constexpr uint32_t STRIDE_64K = 13;
186constexpr uint32_t GERROR_BIT_CMDQ_ERR = 0;
187constexpr uint32_t GERROR_BIT_EVENTQ_ABT = 2;
188constexpr uint32_t GERROR_BIT_PRIQ_ABT = 3;
190constexpr uint32_t GERROR_BIT_SFM_ERR = 8;
193constexpr uint32_t DESC_TYPE_INVALID = 0;
194constexpr uint32_t DESC_TYPE_BLOCK = 1;
195constexpr uint32_t DESC_TYPE_RESERVED = 2;
196constexpr uint32_t DESC_TYPE_TABLE_OR_PAGE = 3;
199constexpr uint32_t DESC_ATTR_AP1_BIT = 4;
200constexpr uint32_t DESC_ATTR_AP2_BIT = 5;
201constexpr uint32_t DESC_ATTR_AF_BIT = 8;
204constexpr uint32_t S2AP_NONE = 0;
205constexpr uint32_t S2AP_RO = 1;
206constexpr uint32_t S2AP_WO = 2;
207constexpr uint32_t S2AP_RW = 3;
210constexpr uint64_t STE_BASE_ADDR_MASK = 0x000FFFFFFFFFFFC0ULL;
211constexpr uint64_t S2TTB_ADDR_MASK = 0x000FFFFFFFFFFFF0ULL;
213 IPA_ADDR_MASK = 0x000FFFFFFFFFF000ULL;
214constexpr uint64_t CMDQ_BASE_HI_MASK = 0x000FFFFFULL;
230constexpr uint8_t CMD_OP_PREFETCH_CONFIG = 0x01;
231constexpr uint8_t CMD_OP_PREFETCH_ADDR = 0x02;
232constexpr uint8_t CMD_OP_CFGI_STE = 0x03;
233constexpr uint8_t CMD_OP_CFGI_STE_RANGE = 0x04;
234constexpr uint8_t CMD_OP_CFGI_CD = 0x05;
235constexpr uint8_t CMD_OP_CFGI_CD_ALL = 0x06;
236constexpr uint8_t CMD_OP_CFGI_ALL = 0x07;
237constexpr uint8_t CMD_OP_TLBI_NH_ALL = 0x10;
238constexpr uint8_t CMD_OP_TLBI_NH_ASID = 0x11;
239constexpr uint8_t CMD_OP_TLBI_NH_VA = 0x12;
240constexpr uint8_t CMD_OP_TLBI_NH_VAA = 0x13;
241constexpr uint8_t CMD_OP_TLBI_EL3_ALL = 0x18;
242constexpr uint8_t CMD_OP_TLBI_EL3_VA = 0x1A;
243constexpr uint8_t CMD_OP_TLBI_EL2_ALL = 0x20;
244constexpr uint8_t CMD_OP_TLBI_EL2_ASID = 0x21;
245constexpr uint8_t CMD_OP_TLBI_EL2_VA = 0x22;
246constexpr uint8_t CMD_OP_TLBI_EL2_VAA = 0x23;
247constexpr uint8_t CMD_OP_TLBI_S12_VMALL = 0x28;
248constexpr uint8_t CMD_OP_TLBI_S2_IPA = 0x2A;
249constexpr uint8_t CMD_OP_TLBI_NSNH_ALL = 0x30;
250constexpr uint8_t CMD_OP_ATC_INV = 0x40;
251constexpr uint8_t CMD_OP_PRI_RESP = 0x41;
252constexpr uint8_t CMD_OP_RESUME = 0x44;
253constexpr uint8_t CMD_OP_STALL_TERM = 0x45;
254constexpr uint8_t CMD_OP_SYNC = 0x46;
257constexpr uint32_t SYNC_CS_NONE = 0;
258constexpr uint32_t SYNC_CS_IRQ = 1;
259constexpr uint32_t SYNC_CS_SEV = 2;
261constexpr uint32_t GATOS_PAR_FAULT_TRANSLATION = 0x10;
263constexpr uint32_t IDR0_TTF_AARCH64 = 0x2;
264constexpr uint32_t IDR0_STLEVEL_2LVL = 0x2;
265constexpr uint32_t IDR1_DEFAULT_QUEUE_LOG2 = 19;
268constexpr uint32_t OAS_32 = 0;
269constexpr uint32_t OAS_36 = 1;
270constexpr uint32_t OAS_40 = 2;
271constexpr uint32_t OAS_42 = 3;
272constexpr uint32_t OAS_44 = 4;
273constexpr uint32_t OAS_48 = 5;
275constexpr uint32_t PA_BITS = 48;
277 VA_SIGN_BIT_SHIFT = 63;
278constexpr uint32_t IPA_PAGE_SHIFT = 12;
280constexpr uint32_t INPUT_SIZE_MIN = 25;
281constexpr uint32_t INPUT_SIZE_MAX = 48;
284constexpr std::array<uint32_t, 8> OUTPUT_SIZE_MAP = { 32, 36, 40, 42, 44, 48, 48, 48 };
288constexpr std::array<uint32_t, 12> PRIMECELL_IDS = {
289 0x04, 0x00, 0x00, 0x00, 0x34, 0xB0, 0x0B, 0x00, 0x0D, 0xF0, 0x05, 0xB1,
294enum class IOMMUAccessFlags : uint8_t {
301constexpr IOMMUAccessFlags operator&(IOMMUAccessFlags a, IOMMUAccessFlags b)
303 return static_cast<IOMMUAccessFlags
>(
static_cast<uint8_t
>(a) &
static_cast<uint8_t
>(b));
306constexpr IOMMUAccessFlags operator|(IOMMUAccessFlags a, IOMMUAccessFlags b)
308 return static_cast<IOMMUAccessFlags
>(
static_cast<uint8_t
>(a) |
static_cast<uint8_t
>(b));
311constexpr IOMMUAccessFlags operator~(IOMMUAccessFlags a)
313 return static_cast<IOMMUAccessFlags
>(~static_cast<uint8_t>(a) & 0x3u);
316inline IOMMUAccessFlags& operator&=(IOMMUAccessFlags& a, IOMMUAccessFlags b)
322inline IOMMUAccessFlags& operator|=(IOMMUAccessFlags& a, IOMMUAccessFlags b)
328template <
unsigned int BUSWIDTH = 32>
334template <
unsigned int BUSWIDTH = 32>
335class smmuv3 :
public sc_core::sc_module,
public smmuv3_gen
338 cci::cci_broker_handle m_broker;
348 uint64_t translated_addr;
350 IOMMUAccessFlags perm;
352 uint32_t table_attrs;
354 uint32_t fault_level;
355 bool stallable_fault;
359 cci::cci_param<uint32_t> p_pamax;
360 cci::cci_param<uint16_t> p_sidsize;
361 cci::cci_param<bool> p_ato;
362 cci::cci_param<uint8_t> p_num_tbu;
363 cci::cci_param<uint32_t> p_iotlb_size;
364 cci::cci_param<uint32_t> p_iidr;
366 tlm_utils::multi_passthrough_target_socket<smmuv3, BUSWIDTH>
368 tlm_utils::simple_initiator_socket<smmuv3>
370 InitiatorSignalSocket<bool> irq_eventq;
371 InitiatorSignalSocket<bool> irq_priq;
372 InitiatorSignalSocket<bool> irq_cmd_sync;
373 InitiatorSignalSocket<bool> irq_gerror;
374 InitiatorSignalSocket<bool> irq_s_gerror;
375 InitiatorSignalSocket<bool> irq_s_eventq;
378 std::vector<smmuv3_tbu<BUSWIDTH>*> tbus;
380 smmuv3(sc_core::sc_module_name name);
388 uint64_t s1contextptr = 0;
395 bool s1stalld =
false;
396 uint32_t s1cdmax = 0;
409 bool aarch64 =
false;
410 std::array<uint64_t, 2> ttb{};
411 std::array<uint32_t, 2> tsz{};
412 std::array<uint32_t, 2> tg{};
415 std::array<bool, 2> epd{};
423 std::array<uint64_t, 2> ttb{};
424 std::array<uint32_t, 2> tsz{};
425 std::array<uint32_t, 2> tg{};
428 std::array<bool, 2> epd{};
431 IOMMUAccessFlags access = IOMMUAccessFlags::NONE;
432 bool s2_enabled =
false;
443 IOMMUAccessFlags access = IOMMUAccessFlags::NONE;
445 uint32_t substream_id = 0;
449 IOMMUAccessFlags prot = IOMMUAccessFlags::NONE;
450 uint64_t page_size = 0;
452 uint32_t event_type = 0;
453 uint32_t tableattrs = 0;
455 uint64_t descriptor = 0;
456 uint64_t leaf_desc_addr = 0;
472 bool operator==(
const IOTLBKey& o)
const
474 return vmid == o.vmid && asid == o.asid && iova == o.iova && tg == o.tg && level == o.level &&
475 secure == o.secure && regime == o.regime;
479 struct IOTLBKeyHash {
480 static constexpr uint32_t VMID_HASH_SHIFT = 48;
481 static constexpr uint32_t ASID_HASH_SHIFT = 32;
483 static constexpr uint64_t SECURE_HASH_SALT = 0x9E3779B97F4A7C15ULL;
484 static constexpr uint64_t REGIME_HASH_MULT = 0xBF58476D1CE4E5B9ULL;
485 std::size_t operator()(
const IOTLBKey& k)
const
487 uint64_t base = ((uint64_t)k.vmid << VMID_HASH_SHIFT) | ((uint64_t)k.asid << ASID_HASH_SHIFT) | k.iova;
488 if (k.secure) base ^= SECURE_HASH_SALT;
489 base ^=
static_cast<uint64_t
>(k.regime) * REGIME_HASH_MULT;
490 return std::hash<uint64_t>()(base);
494 std::unordered_map<IOTLBKey, IOMMUTLBEntry, IOTLBKeyHash> m_iotlb;
495 std::list<IOTLBKey> m_iotlb_lru;
500 tlm::tlm_generic_payload txn;
501 sc_core::sc_time delay;
504 uint32_t substream_id;
505 sc_core::sc_event resume_event;
508 std::unordered_map<uint32_t, StalledTxn> m_stalled_txns;
509 uint32_t m_next_stag;
513 static constexpr size_t STE_CACHE_MAX = 1024;
514 static constexpr size_t CD_CACHE_MAX = 1024;
515 static constexpr size_t DMA_DMI_CACHE_MAX = 64;
517 std::unordered_map<uint32_t, STE> m_ste_cache;
518 std::list<uint32_t> m_ste_cache_lru;
519 std::unordered_map<uint64_t, CD> m_cd_cache;
520 std::list<uint64_t> m_cd_cache_lru;
522 std::unordered_map<uint32_t, STE> m_ste_cache_secure;
523 std::unordered_map<uint64_t, CD> m_cd_cache_secure;
525 bool m_prev_gerror_pending =
false;
526 bool m_prev_eventq_pending =
false;
527 bool m_prev_priq_pending =
false;
528 bool m_prev_s_gerror_pending =
false;
529 bool m_prev_s_eventq_pending =
false;
530 bool m_last_irq_gerror_level =
false;
531 bool m_last_irq_eventq_level =
false;
532 bool m_last_irq_priq_level =
false;
533 bool m_last_irq_s_gerror_level =
false;
534 bool m_last_irq_s_eventq_level =
false;
536 static uint32_t extract32(uint32_t val,
int start,
int length) {
return (val >> start) & ((1u << length) - 1); }
538 static uint64_t extract64(uint64_t val,
int start,
int length) {
return (val >> start) & ((1ULL << length) - 1); }
546 static constexpr uint8_t IOTLB_REGIME_NS_EL1 = 0;
547 static constexpr uint8_t IOTLB_REGIME_NS_EL2 = 1;
548 static constexpr uint8_t IOTLB_REGIME_NS_EL2_E2H = 2;
549 static constexpr uint8_t IOTLB_REGIME_SECURE = 3;
550 static uint8_t compute_iotlb_regime(uint8_t strw,
bool e2h,
bool is_secure)
552 if (is_secure)
return IOTLB_REGIME_SECURE;
554 if ((strw & 0x3) == 0x2)
return e2h ? IOTLB_REGIME_NS_EL2_E2H : IOTLB_REGIME_NS_EL2;
555 return IOTLB_REGIME_NS_EL1;
558 bool check_s2_startlevel(
unsigned int pamax_val,
int level,
int inputsize,
int stride)
560 if (level < 0)
return false;
563 if (level == 0 || (level == 1 && pamax_val <= 42))
return false;
566 if (level == 0 || (level == 1 && pamax_val <= 40))
return false;
569 if (level == 0 && pamax_val <= 42)
return false;
577 bool check_out_addr(uint64_t addr,
unsigned int outputsize)
579 if (outputsize >= PA_BITS)
return true;
580 return extract64(addr, outputsize, PA_BITS - outputsize) == 0;
586 struct DmaDmiRegion {
593 std::vector<DmaDmiRegion> m_dma_dmi_cache;
595 const DmaDmiRegion* find_dma_dmi(uint64_t addr,
size_t len,
bool write)
const
597 if (len == 0)
return nullptr;
598 sc_dt::uint64 last = addr + len - 1;
599 for (
const auto& r : m_dma_dmi_cache) {
600 if (addr >= r.start && last <= r.end) {
601 if ((write && r.write_allowed) || (!write && r.read_allowed)) {
609 bool grant_dma_dmi(uint64_t addr, tlm::tlm_command cmd)
611 tlm::tlm_generic_payload probe;
612 probe.set_command(cmd);
613 probe.set_address(addr);
615 probe.set_data_ptr(&scratch);
616 probe.set_data_length(1);
617 probe.set_streaming_width(1);
618 probe.set_byte_enable_length(0);
619 probe.set_dmi_allowed(
true);
620 probe.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
622 if (!dma_socket->get_direct_mem_ptr(probe, dmi)) {
625 if (!dmi.is_read_allowed() && !dmi.is_write_allowed()) {
629 r.start = dmi.get_start_address();
630 r.end = dmi.get_end_address();
631 r.ptr = dmi.get_dmi_ptr();
632 r.read_allowed = dmi.is_read_allowed();
633 r.write_allowed = dmi.is_write_allowed();
635 if (m_dma_dmi_cache.size() >= DMA_DMI_CACHE_MAX) {
636 m_dma_dmi_cache.erase(m_dma_dmi_cache.begin());
638 m_dma_dmi_cache.push_back(r);
642 void dma_dmi_invalidate(sc_dt::uint64 start, sc_dt::uint64 end)
644 m_dma_dmi_cache.erase(
645 std::remove_if(m_dma_dmi_cache.begin(), m_dma_dmi_cache.end(),
646 [start, end](
const DmaDmiRegion& r) { return !(r.end < start || r.start > end); }),
647 m_dma_dmi_cache.end());
650 bool dma_read(uint64_t addr,
void* data,
size_t len)
652 const DmaDmiRegion* r = find_dma_dmi(addr, len,
false);
653 if (!r && grant_dma_dmi(addr, tlm::TLM_READ_COMMAND)) {
654 r = find_dma_dmi(addr, len,
false);
657 std::memcpy(data, r->ptr + (addr - r->start), len);
660 tlm::tlm_generic_payload txn;
661 txn.set_command(tlm::TLM_READ_COMMAND);
662 txn.set_address(addr);
663 txn.set_data_ptr(
reinterpret_cast<unsigned char*
>(data));
664 txn.set_data_length(len);
665 txn.set_streaming_width(len);
666 txn.set_byte_enable_length(0);
667 txn.set_dmi_allowed(
false);
668 txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
669 sc_core::sc_time delay = sc_core::SC_ZERO_TIME;
670 dma_socket->b_transport(txn, delay);
671 return txn.get_response_status() == tlm::TLM_OK_RESPONSE;
674 bool dma_write(uint64_t addr,
const void* data,
size_t len)
676 const DmaDmiRegion* r = find_dma_dmi(addr, len,
true);
677 if (!r && grant_dma_dmi(addr, tlm::TLM_WRITE_COMMAND)) {
678 r = find_dma_dmi(addr, len,
true);
681 std::memcpy(r->ptr + (addr - r->start), data, len);
684 tlm::tlm_generic_payload txn;
685 txn.set_command(tlm::TLM_WRITE_COMMAND);
686 txn.set_address(addr);
687 txn.set_data_ptr(
const_cast<unsigned char*
>(
reinterpret_cast<const unsigned char*
>(data)));
688 txn.set_data_length(len);
689 txn.set_streaming_width(len);
690 txn.set_byte_enable_length(0);
691 txn.set_dmi_allowed(
false);
692 txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
693 sc_core::sc_time delay = sc_core::SC_ZERO_TIME;
694 dma_socket->b_transport(txn, delay);
695 return txn.get_response_status() == tlm::TLM_OK_RESPONSE;
698 bool find_ste(uint32_t sid, STE& ste);
699 bool read_cd(uint32_t sid,
const STE& ste, uint32_t substream_id,
701 bool smmuv3_ptw64(PtwCtx& ctx, TransReq& req);
702 IOMMUTLBEntry smmuv3_translate(tlm::tlm_generic_payload& txn, uint32_t sid, uint32_t substream_id,
703 bool is_secure =
false,
704 bool is_ats_tr =
false);
707 void iotlb_insert(
const IOTLBKey& key,
const IOMMUTLBEntry& entry);
708 bool iotlb_lookup(
const IOTLBKey& key,
IOMMUTLBEntry& entry);
709 void iotlb_inv_all();
710 void iotlb_inv_asid(uint16_t asid);
711 void iotlb_inv_vmid(uint16_t vmid);
712 void iotlb_inv_iova(uint16_t vmid, uint16_t asid, uint64_t iova, uint8_t tg,
716 void consume_secure_cmdq();
717 void record_event(uint32_t type, uint32_t sid, uint64_t iova,
719 void record_event_full(uint32_t type, uint32_t sid, uint32_t ssid, uint64_t iova, tlm::tlm_generic_payload& txn,
720 uint32_t class_, uint32_t fault_level, uint32_t stag,
bool stall);
721 void record_secure_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info);
722 void record_pri(uint32_t sid, uint64_t iova, uint32_t flags);
723 bool queue_full(uint32_t prod, uint32_t cons, uint32_t size);
726 void handle_cmd_prefetch_config(
const uint8_t* cmd);
727 void handle_cmd_prefetch_addr(
const uint8_t* cmd);
728 void handle_cmd_cfgi_ste(
const uint8_t* cmd);
729 void handle_cmd_cfgi_ste_range(
const uint8_t* cmd);
730 void handle_cmd_cfgi_cd(
const uint8_t* cmd);
731 void handle_cmd_cfgi_cd_all(
const uint8_t* cmd);
732 void handle_cmd_cfgi_all(
const uint8_t* cmd);
733 void handle_cmd_tlbi_nh_all(
const uint8_t* cmd);
734 void handle_cmd_tlbi_nh_asid(
const uint8_t* cmd);
735 void handle_cmd_tlbi_nh_va(
const uint8_t* cmd);
736 void handle_cmd_tlbi_nh_vaa(
const uint8_t* cmd);
737 void handle_cmd_atc_inv(
const uint8_t* cmd);
738 void handle_cmd_tlbi_s12_vmall(
const uint8_t* cmd);
739 void handle_cmd_tlbi_s2_ipa(
const uint8_t* cmd);
740 void handle_cmd_tlbi_nsnh_all(
const uint8_t* cmd);
741 void handle_cmd_sync(
const uint8_t* cmd);
742 void handle_cmd_resume(
const uint8_t* cmd);
743 void handle_cmd_pri_resp(
const uint8_t* cmd);
747 bool stall_and_wait(uint32_t sid, uint32_t ssid, uint64_t iova, uint32_t fault_type, uint32_t fault_level,
748 uint32_t class_, tlm::tlm_generic_payload& txn);
753 void trigger_irq(InitiatorSignalSocket<bool>& irq)
755 if (irq.size() > 0) {
763 void set_gerror(uint32_t bit)
765 uint32_t err =
static_cast<uint32_t
>(GERROR);
766 uint32_t ack =
static_cast<uint32_t
>(GERRORN);
767 if (((err ^ ack) >> bit) & 1u) {
770 GERROR = err ^ (1u << bit);
774 void set_secure_gerror(uint32_t bit)
776 uint32_t err =
static_cast<uint32_t
>(S_GERROR);
777 uint32_t ack =
static_cast<uint32_t
>(S_GERRORN);
778 if (((err ^ ack) >> bit) & 1u) {
781 S_GERROR = err ^ (1u << bit);
782 update_secure_irq_levels();
785 void set_cmdq_err(uint8_t cerror)
787 CMDQ_CONS[CMDQ_CONS_ERR] = cerror;
788 set_gerror(GERROR_BIT_CMDQ_ERR);
791 void set_secure_cmdq_err(uint8_t cerror)
793 S_CMDQ_CONS[S_CMDQ_CONS_ERR] = cerror;
794 set_secure_gerror(GERROR_BIT_CMDQ_ERR);
797 void drive_irq_level(InitiatorSignalSocket<bool>& irq,
bool& last_level,
bool new_level)
799 if (last_level == new_level)
return;
800 last_level = new_level;
801 if (irq.size() > 0) {
802 irq->write(new_level);
808 void emit_msi(uint32_t cfg0_lo, uint32_t cfg0_hi, uint32_t data, uint32_t abort_bit_on_fail)
810 uint64_t addr = (
static_cast<uint64_t
>(cfg0_hi) << 32) |
static_cast<uint64_t
>(cfg0_lo);
811 if (addr == 0)
return;
812 uint32_t payload = data;
813 if (!dma_write(addr, &payload,
sizeof(payload))) {
814 set_gerror(abort_bit_on_fail);
818 void update_irq_levels()
820 bool gerror_pending = (
static_cast<uint32_t
>(GERROR) ^
static_cast<uint32_t
>(GERRORN)) != 0;
821 bool eventq_pending = (
static_cast<uint32_t
>(EVENTQ_PROD) !=
static_cast<uint32_t
>(EVENTQ_CONS));
822 bool priq_pending = (
static_cast<uint32_t
>(PRIQ_PROD) !=
static_cast<uint32_t
>(PRIQ_CONS));
824 bool gerror_irq = gerror_pending &&
static_cast<uint32_t
>(IRQ_CTRL[IRQ_CTRL_GERROR_IRQEN]);
825 bool eventq_irq = eventq_pending &&
static_cast<uint32_t
>(IRQ_CTRL[IRQ_CTRL_EVENTQ_IRQEN]);
826 bool priq_irq = priq_pending &&
static_cast<uint32_t
>(IRQ_CTRL[IRQ_CTRL_PRI_IRQEN]);
828 if (gerror_pending && !m_prev_gerror_pending &&
static_cast<uint32_t
>(IRQ_CTRL[IRQ_CTRL_GERROR_IRQEN])) {
829 emit_msi(
static_cast<uint32_t
>(GERROR_IRQ_CFG0_LO),
static_cast<uint32_t
>(GERROR_IRQ_CFG0_HI),
830 static_cast<uint32_t
>(GERROR_IRQ_CFG2), GERROR_BIT_MSI_GERROR_ABT);
832 if (eventq_pending && !m_prev_eventq_pending &&
static_cast<uint32_t
>(IRQ_CTRL[IRQ_CTRL_EVENTQ_IRQEN])) {
833 emit_msi(
static_cast<uint32_t
>(EVENTQ_IRQ_CFG0_LO),
static_cast<uint32_t
>(EVENTQ_IRQ_CFG0_HI),
834 static_cast<uint32_t
>(EVENTQ_IRQ_CFG2), GERROR_BIT_MSI_EVENTQ_ABT);
836 if (priq_pending && !m_prev_priq_pending &&
static_cast<uint32_t
>(IRQ_CTRL[IRQ_CTRL_PRI_IRQEN])) {
837 emit_msi(
static_cast<uint32_t
>(PRIQ_IRQ_CFG0_LO),
static_cast<uint32_t
>(PRIQ_IRQ_CFG0_HI),
838 static_cast<uint32_t
>(PRIQ_IRQ_CFG2), GERROR_BIT_MSI_PRIQ_ABT);
841 m_prev_gerror_pending = gerror_pending;
842 m_prev_eventq_pending = eventq_pending;
843 m_prev_priq_pending = priq_pending;
845 drive_irq_level(irq_gerror, m_last_irq_gerror_level, gerror_irq);
846 drive_irq_level(irq_eventq, m_last_irq_eventq_level, eventq_irq);
847 drive_irq_level(irq_priq, m_last_irq_priq_level, priq_irq);
850 void update_secure_irq_levels()
852 bool gerror_pending = (
static_cast<uint32_t
>(S_GERROR) ^
static_cast<uint32_t
>(S_GERRORN)) != 0;
853 bool eventq_pending = (
static_cast<uint32_t
>(S_EVENTQ_PROD) !=
static_cast<uint32_t
>(S_EVENTQ_CONS));
855 bool gerror_irq = gerror_pending &&
static_cast<uint32_t
>(S_IRQ_CTRL[S_IRQ_CTRL_GERROR_IRQEN]);
856 bool eventq_irq = eventq_pending &&
static_cast<uint32_t
>(S_IRQ_CTRL[S_IRQ_CTRL_EVENTQ_IRQEN]);
858 if (gerror_pending && !m_prev_s_gerror_pending &&
static_cast<uint32_t
>(S_IRQ_CTRL[S_IRQ_CTRL_GERROR_IRQEN])) {
859 emit_msi(
static_cast<uint32_t
>(S_GERROR_IRQ_CFG0_LO),
static_cast<uint32_t
>(S_GERROR_IRQ_CFG0_HI),
860 static_cast<uint32_t
>(S_GERROR_IRQ_CFG2), GERROR_BIT_MSI_GERROR_ABT);
862 if (eventq_pending && !m_prev_s_eventq_pending &&
static_cast<uint32_t
>(S_IRQ_CTRL[S_IRQ_CTRL_EVENTQ_IRQEN])) {
863 emit_msi(
static_cast<uint32_t
>(S_EVENTQ_IRQ_CFG0_LO),
static_cast<uint32_t
>(S_EVENTQ_IRQ_CFG0_HI),
864 static_cast<uint32_t
>(S_EVENTQ_IRQ_CFG2), GERROR_BIT_MSI_EVENTQ_ABT);
867 m_prev_s_gerror_pending = gerror_pending;
868 m_prev_s_eventq_pending = eventq_pending;
870 drive_irq_level(irq_s_gerror, m_last_irq_s_gerror_level, gerror_irq);
871 drive_irq_level(irq_s_eventq, m_last_irq_s_eventq_level, eventq_irq);
875 friend class smmuv3_tbu<BUSWIDTH>;
877 void start_of_simulation()
override;
878 void before_end_of_elaboration()
override;
880 void test_inject_pri(uint32_t sid, uint64_t iova, uint32_t flags) { record_pri(sid, iova, flags); }
881 void test_inject_secure_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info)
883 record_secure_event(type, sid, iova, info);
885 void test_inject_stall(uint32_t stag)
887 auto& s = m_stalled_txns[stag];
891 bool test_stall_present(uint32_t stag)
const {
return m_stalled_txns.count(stag) != 0; }
892 bool test_stall_aborted(uint32_t stag)
const
894 auto it = m_stalled_txns.find(stag);
895 return it != m_stalled_txns.end() && it->second.aborted;
897 size_t test_iotlb_size()
const {
return m_iotlb.size(); }
898 size_t test_ste_cache_size()
const {
return m_ste_cache.size(); }
899 size_t test_cd_cache_size()
const {
return m_cd_cache.size(); }
905template <
unsigned int BUSWIDTH>
906class smmuv3_tbu :
public sc_core::sc_module
919 static uint32_t extract_substream_id(tlm::tlm_generic_payload& txn)
922 return (ssx && ssx->ssv) ? ssx->substream_id : 0;
925 static bool extract_secure(tlm::tlm_generic_payload& txn)
927 auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
928 return secure && secure->value;
932 void upstream_invalidate(sc_dt::uint64 start, sc_dt::uint64 end)
934 if (upstream_socket.size() == 0)
return;
935 upstream_socket->invalidate_direct_mem_ptr(start, end);
939 void b_transport(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay)
941 sc_dt::uint64 addr = txn.get_address();
942 tlm::tlm_command cmd = txn.get_command();
945 uint32_t substream_id = extract_substream_id(txn);
946 bool is_secure = extract_secure(txn);
948 if (ats && ats->is_translated) {
949 downstream_socket->b_transport(txn, delay);
954 txn, p_topology_id, substream_id, is_secure, ats && ats->is_translation_request);
956 while (te.stallable_fault) {
957 bool aborted = m_smmu->stall_and_wait(p_topology_id, substream_id, addr, te.fault_type, te.fault_level, 0,
960 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
963 te = m_smmu->smmuv3_translate(txn, p_topology_id, substream_id, is_secure,
964 ats && ats->is_translation_request);
967 if (ats && ats->is_translation_request) {
968 if (te.perm == IOMMUAccessFlags::NONE) {
970 ats->fault_type = te.fault_type;
972 ats->granule_log2 = 0;
974 if (
static_cast<uint32_t
>(m_smmu->CR0[m_smmu->CR0_PRIQEN]) &&
975 static_cast<uint32_t
>(m_smmu->IDR0[m_smmu->IDR0_PRI])) {
976 uint32_t flags = (cmd == tlm::TLM_WRITE_COMMAND ? 0x2u : 0x0u) | (ats->prg_index << 16);
977 m_smmu->record_pri(p_topology_id, addr, flags);
980 ats->faulted =
false;
981 ats->pa = te.translated_addr | (addr & te.addr_mask);
982 uint64_t m = te.addr_mask + 1;
988 ats->granule_log2 = g;
989 ats->prot =
static_cast<uint8_t
>(te.perm);
991 txn.set_response_status(tlm::TLM_OK_RESPONSE);
995 if (te.perm == IOMMUAccessFlags::NONE || (cmd == tlm::TLM_WRITE_COMMAND && te.perm == IOMMUAccessFlags::RO) ||
996 (cmd == tlm::TLM_READ_COMMAND && te.perm == IOMMUAccessFlags::WO)) {
997 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
999 txn.set_address(te.translated_addr | (addr & SMMUV3_PAGEMASK));
1001 auto ext = std::make_unique<smmuv3_memory_attrs_extension>();
1002 ext->set_from_descriptor(te.descriptor, te.table_attrs);
1003 txn.set_extension(ext.get());
1005 SCP_INFO(()) << std::hex <<
"smmuv3 TBU b_transport: translate 0x" << addr <<
" to 0x"
1006 << (te.translated_addr | (addr & SMMUV3_PAGEMASK)) <<
" attrs=" << ext->to_string();
1008 downstream_socket->b_transport(txn, delay);
1009 txn.set_address(addr);
1014 virtual unsigned int transport_dbg(tlm::tlm_generic_payload& txn)
1016 sc_dt::uint64 addr = txn.get_address();
1017 uint32_t substream_id = extract_substream_id(txn);
1018 bool is_secure = extract_secure(txn);
1021 txn.set_address(te.translated_addr | (addr & SMMUV3_PAGEMASK));
1022 int ret = downstream_socket->transport_dbg(txn);
1023 txn.set_address(addr);
1027 virtual bool get_direct_mem_ptr(tlm::tlm_generic_payload& txn, tlm::tlm_dmi& dmi_data)
1029 sc_dt::uint64 iova = txn.get_address();
1030 uint32_t substream_id = extract_substream_id(txn);
1031 bool is_secure = extract_secure(txn);
1035 if (te.perm == IOMMUAccessFlags::NONE) {
1036 txn.set_address(iova);
1037 dmi_data.allow_none();
1038 dmi_data.set_start_address(iova);
1039 dmi_data.set_end_address(iova);
1043 tlm::tlm_command cmd = txn.get_command();
1044 if ((cmd == tlm::TLM_WRITE_COMMAND && te.perm == IOMMUAccessFlags::RO) ||
1045 (cmd == tlm::TLM_READ_COMMAND && te.perm == IOMMUAccessFlags::WO)) {
1046 txn.set_address(iova);
1047 dmi_data.allow_none();
1048 dmi_data.set_start_address(iova);
1049 dmi_data.set_end_address(iova);
1053 sc_dt::uint64 offset_in_page = iova & te.addr_mask;
1054 sc_dt::uint64 pa = te.translated_addr | offset_in_page;
1055 txn.set_address(pa);
1056 bool ok = downstream_socket->get_direct_mem_ptr(txn, dmi_data);
1057 txn.set_address(iova);
1058 if (!ok)
return false;
1060 sc_dt::uint64 pa_page_start = te.translated_addr;
1061 sc_dt::uint64 pa_page_end = te.translated_addr | te.addr_mask;
1062 sc_dt::uint64 dmi_start = dmi_data.get_start_address();
1063 sc_dt::uint64 dmi_end = dmi_data.get_end_address();
1065 sc_dt::uint64 clamp_pa_start = pa_page_start > dmi_start ? pa_page_start : dmi_start;
1066 sc_dt::uint64 clamp_pa_end = pa_page_end < dmi_end ? pa_page_end : dmi_end;
1067 if (clamp_pa_end < clamp_pa_start)
return false;
1069 sc_dt::uint64 iova_page_start = iova & ~te.addr_mask;
1070 sc_dt::uint64 new_iova_start = iova_page_start + (clamp_pa_start - pa_page_start);
1071 sc_dt::uint64 new_iova_end = iova_page_start + (clamp_pa_end - pa_page_start);
1073 unsigned char* new_dmi_ptr = dmi_data.get_dmi_ptr() + (clamp_pa_start - dmi_start);
1075 dmi_data.set_dmi_ptr(new_dmi_ptr);
1076 dmi_data.set_start_address(new_iova_start);
1077 dmi_data.set_end_address(new_iova_end);
1079 if (te.perm == IOMMUAccessFlags::RO) {
1080 dmi_data.allow_read();
1081 }
else if (te.perm == IOMMUAccessFlags::WO) {
1082 dmi_data.allow_write();
1087 virtual void invalidate_direct_mem_ptr(sc_dt::uint64 start, sc_dt::uint64 end)
1089 if (!m_smmu)
return;
1090 for (
const auto& e : m_smmu->m_iotlb) {
1091 const auto& entry = e.second;
1092 sc_dt::uint64 pa_start = entry.translated_addr;
1093 sc_dt::uint64 pa_end = entry.translated_addr | entry.addr_mask;
1094 if (pa_start > end || pa_end < start)
continue;
1095 sc_dt::uint64 iova_start = entry.iova;
1096 sc_dt::uint64 iova_end = entry.iova | entry.addr_mask;
1097 upstream_socket->invalidate_direct_mem_ptr(iova_start, iova_end);
1103 cci::cci_param<uint32_t> p_topology_id;
1104 tlm_utils::simple_target_socket<smmuv3_tbu> upstream_socket;
1105 tlm_utils::simple_initiator_socket<smmuv3_tbu>
1108 smmuv3_tbu(
const sc_core::sc_module_name& name, sc_core::sc_object* o)
1114 : sc_core::sc_module(name)
1116 , p_topology_id(
"topology_id", 0)
1117 , upstream_socket(
"upstream_socket")
1118 , downstream_socket(
"downstream_socket")
1121 m_smmu->tbus.push_back(
this);
1124 upstream_socket.register_b_transport(
this, &smmuv3_tbu::b_transport);
1125 upstream_socket.register_transport_dbg(
this, &smmuv3_tbu::transport_dbg);
1126 upstream_socket.register_get_direct_mem_ptr(
this, &smmuv3_tbu::get_direct_mem_ptr);
1127 downstream_socket.register_invalidate_direct_mem_ptr(
this, &smmuv3_tbu::invalidate_direct_mem_ptr);
1131template <
unsigned int BUSWIDTH>
1132smmuv3<BUSWIDTH>::smmuv3(sc_core::sc_module_name name)
1133 : sc_core::sc_module(name)
1135 , m_broker(cci::cci_get_broker())
1136 , m_jza(
gs::zip_open_from_memory(smmuv3_zip_archive_data(), smmuv3_zip_archive_size()))
1137 , m_loaded_ok(m_jza.json_read_cci(m_broker, std::string(this->name()) +
".smmuv3"))
1138 , M(
"smmuv3", m_jza)
1139 , p_pamax(
"pamax", 48)
1140 , p_sidsize(
"sidsize", 16)
1141 , p_ato(
"ato", true)
1142 , p_num_tbu(
"num_tbu", 1)
1143 , p_iotlb_size(
"iotlb_size", 256)
1144 , p_iidr(
"iidr", 0x00000000u)
1145 , socket(
"target_socket")
1147 , irq_eventq(
"irq_eventq")
1148 , irq_priq(
"irq_priq")
1149 , irq_cmd_sync(
"irq_cmd_sync")
1150 , irq_gerror(
"irq_gerror")
1151 , irq_s_gerror(
"irq_s_gerror")
1152 , irq_s_eventq(
"irq_s_eventq")
1157 sc_assert(m_loaded_ok);
1159 socket.bind(M.target_socket);
1162 dma_socket.register_invalidate_direct_mem_ptr(
this, &smmuv3<BUSWIDTH>::dma_dmi_invalidate);
1164 reset.register_value_changed_cb([
this](
bool v) {
1167 m_iotlb_lru.clear();
1168 m_ste_cache.clear();
1169 m_ste_cache_lru.clear();
1171 m_cd_cache_lru.clear();
1172 m_stalled_txns.clear();
1173 m_dma_dmi_cache.clear();
1175 start_of_simulation();
1180template <
unsigned int BUSWIDTH>
1181void smmuv3<BUSWIDTH>::start_of_simulation()
1186 IDR0[IDR0_TTF] = IDR0_TTF_AARCH64;
1187 IDR0[IDR0_COHACC] = 1;
1188 IDR0[IDR0_ASID16] = 1;
1189 IDR0[IDR0_VMID16] = 1;
1191 IDR0[IDR0_ATOS] =
static_cast<uint32_t
>(p_ato.get_value());
1192 IDR0[IDR0_HTTU] = 3;
1193 IDR0[IDR0_STLEVEL] = IDR0_STLEVEL_2LVL;
1198 IDR1[IDR1_SIDSIZE] = p_sidsize.get_value();
1199 IDR1[IDR1_EVENTQS] = IDR1_DEFAULT_QUEUE_LOG2;
1200 IDR1[IDR1_CMDQS] = IDR1_DEFAULT_QUEUE_LOG2;
1203 uint32_t oas_val = OAS_48;
1204 switch (p_pamax.get_value()) {
1227 IDR5[IDR5_OAS] = oas_val;
1228 IDR5[IDR5_GRAN4K] = 1;
1229 IDR5[IDR5_GRAN16K] = 1;
1230 IDR5[IDR5_GRAN64K] = 1;
1232 for (
size_t i = 0; i < PRIMECELL_IDS.size(); ++i) {
1233 *IDREGS[i] = PRIMECELL_IDS[i];
1236 IIDR = p_iidr.get_value();
1238 GBPA[GBPA_ABORT] = 1;
1255 GERROR_IRQ_CFG0_LO = 0;
1256 GERROR_IRQ_CFG0_HI = 0;
1257 GERROR_IRQ_CFG1 = 0;
1258 GERROR_IRQ_CFG2 = 0;
1261 STRTAB_BASE_CFG = 0;
1270 EVENTQ_IRQ_CFG0_LO = 0;
1271 EVENTQ_IRQ_CFG0_HI = 0;
1272 EVENTQ_IRQ_CFG1 = 0;
1273 EVENTQ_IRQ_CFG2 = 0;
1278 PRIQ_IRQ_CFG0_LO = 0;
1279 PRIQ_IRQ_CFG0_HI = 0;
1306 S_GERROR_IRQ_CFG0_LO = 0;
1307 S_GERROR_IRQ_CFG0_HI = 0;
1308 S_GERROR_IRQ_CFG1 = 0;
1309 S_GERROR_IRQ_CFG2 = 0;
1310 S_STRTAB_BASE_LO = 0;
1311 S_STRTAB_BASE_HI = 0;
1312 S_STRTAB_BASE_CFG = 0;
1317 S_EVENTQ_BASE_LO = 0;
1318 S_EVENTQ_BASE_HI = 0;
1321 S_EVENTQ_IRQ_CFG0_LO = 0;
1322 S_EVENTQ_IRQ_CFG0_HI = 0;
1323 S_EVENTQ_IRQ_CFG1 = 0;
1324 S_EVENTQ_IRQ_CFG2 = 0;
1336template <
unsigned int BUSWIDTH>
1337void smmuv3<BUSWIDTH>::before_end_of_elaboration()
1339 SCP_INFO(()) <<
"SMMUv3: registering post_write callbacks";
1341 CR0.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1342 uint32_t cr0 =
static_cast<uint32_t
>(CR0);
1343 SCP_INFO(()) <<
"SMMUv3: CR0 write = 0x" << std::hex << cr0;
1347 IRQ_CTRL.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1348 uint32_t val =
static_cast<uint32_t
>(IRQ_CTRL);
1349 SCP_INFO(()) <<
"SMMUv3: IRQ_CTRL write = 0x" << std::hex << val;
1351 update_irq_levels();
1354 GERRORN.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_irq_levels(); });
1355 EVENTQ_CONS.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_irq_levels(); });
1356 PRIQ_CONS.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_irq_levels(); });
1358 S_CR0.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1359 S_CR0ACK =
static_cast<uint32_t
>(S_CR0);
1360 update_secure_irq_levels();
1362 S_IRQ_CTRL.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1363 S_IRQ_CTRL_ACK =
static_cast<uint32_t
>(S_IRQ_CTRL);
1364 update_secure_irq_levels();
1366 S_GERRORN.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_secure_irq_levels(); });
1367 S_EVENTQ_CONS.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { update_secure_irq_levels(); });
1369 S_CMDQ_PROD.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1370 if (
static_cast<uint32_t
>(S_CR0[S_CR0_CMDQEN])) {
1371 consume_secure_cmdq();
1375 S_CMDQ_CONS.pre_read([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1376 if (
static_cast<uint32_t
>(S_CR0[S_CR0_CMDQEN])) {
1377 consume_secure_cmdq();
1381 CMDQ_PROD.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1382 uint32_t prod =
static_cast<uint32_t
>(CMDQ_PROD);
1383 SCP_INFO(()) <<
"SMMUv3: CMDQ_PROD write = 0x" << std::hex << prod;
1384 if (
static_cast<uint32_t
>(CR0[CR0_CMDQEN])) {
1389 CMDQ_CONS.pre_read([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1390 if (
static_cast<uint32_t
>(CR0[CR0_CMDQEN])) {
1395 STRTAB_BASE_CFG.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1396 m_ste_cache.clear();
1397 m_ste_cache_lru.clear();
1400 GBPA.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { GBPA[GBPA_UPDATE] = 0u; });
1401 S_GBPA.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { S_GBPA[S_GBPA_UPDATE] = 0u; });
1402 S_INIT.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1403 if (
static_cast<uint32_t
>(S_INIT[S_INIT_INV_ALL])) {
1405 m_iotlb_lru.clear();
1406 m_ste_cache.clear();
1407 m_ste_cache_lru.clear();
1409 m_cd_cache_lru.clear();
1410 m_ste_cache_secure.clear();
1411 m_cd_cache_secure.clear();
1412 S_INIT[S_INIT_INV_ALL] = 0u;
1416 GATOS_CTRL.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1417 if (
static_cast<uint32_t
>(GATOS_CTRL[GATOS_CTRL_RUN])) {
1422 PRIQ_CONS.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {});
1424 CMDQ_PROD_P1.post_write([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1425 CMDQ_PROD =
static_cast<uint32_t
>(CMDQ_PROD_P1);
1426 if (
static_cast<uint32_t
>(CR0[CR0_CMDQEN])) {
1431 EVENTQ_CONS_P1.post_write(
1432 [
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { EVENTQ_CONS =
static_cast<uint32_t
>(EVENTQ_CONS_P1); });
1434 PRIQ_CONS_P1.post_write(
1435 [
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { PRIQ_CONS =
static_cast<uint32_t
>(PRIQ_CONS_P1); });
1437 CMDQ_PROD_P1.pre_read(
1438 [
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { CMDQ_PROD_P1 =
static_cast<uint32_t
>(CMDQ_PROD); });
1439 CMDQ_CONS_P1.pre_read([
this](tlm::tlm_generic_payload&, sc_core::sc_time&) {
1440 if (
static_cast<uint32_t
>(CR0[CR0_CMDQEN])) consume_cmdq();
1441 CMDQ_CONS_P1 =
static_cast<uint32_t
>(CMDQ_CONS);
1443 EVENTQ_PROD_P1.pre_read(
1444 [
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { EVENTQ_PROD_P1 =
static_cast<uint32_t
>(EVENTQ_PROD); });
1445 EVENTQ_CONS_P1.pre_read(
1446 [
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { EVENTQ_CONS_P1 =
static_cast<uint32_t
>(EVENTQ_CONS); });
1447 PRIQ_PROD_P1.pre_read(
1448 [
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { PRIQ_PROD_P1 =
static_cast<uint32_t
>(PRIQ_PROD); });
1449 PRIQ_CONS_P1.pre_read(
1450 [
this](tlm::tlm_generic_payload&, sc_core::sc_time&) { PRIQ_CONS_P1 =
static_cast<uint32_t
>(PRIQ_CONS); });
1453template <
unsigned int BUSWIDTH>
1454bool smmuv3<BUSWIDTH>::find_ste(uint32_t sid, STE& ste)
1456 auto it = m_ste_cache.find(sid);
1457 if (it != m_ste_cache.end()) {
1462 constexpr uint64_t L1STD_VALID_BIT = 1;
1463 constexpr uint32_t L1STD_PTR_LOW_BITS = 6;
1464 constexpr size_t L1STD_SIZE = 8;
1465 constexpr uint32_t STRTAB_FMT_LINEAR = 0;
1467 uint64_t strtab_base = ((
static_cast<uint64_t
>(STRTAB_BASE_HI) << 32) |
static_cast<uint64_t
>(STRTAB_BASE_LO)) &
1469 uint32_t fmt =
static_cast<uint32_t
>(STRTAB_BASE_CFG[STRTAB_BASE_CFG_FMT]);
1471 uint32_t log2size =
static_cast<uint32_t
>(STRTAB_BASE_CFG[STRTAB_BASE_CFG_LOG2SIZE]);
1472 if (log2size > 31u) log2size = 31u;
1474 std::array<uint8_t, SMMUV3_STE_SIZE> ste_buf{};
1477 if (fmt == STRTAB_FMT_LINEAR) {
1478 if (sid >= (1u << log2size)) {
1481 ste_addr = strtab_base + (sid * SMMUV3_STE_SIZE);
1483 uint32_t split =
static_cast<uint32_t
>(STRTAB_BASE_CFG[STRTAB_BASE_CFG_SPLIT]);
1484 uint32_t l1_idx = sid >> split;
1485 uint32_t l2_idx = sid & ((1u << split) - 1);
1487 if (l1_idx >= (1u << log2size)) {
1491 uint64_t l1std_addr = strtab_base + (l1_idx * L1STD_SIZE);
1493 if (!dma_read(l1std_addr, &l1std, L1STD_SIZE)) {
1494 set_gerror(GERROR_BIT_SFM_ERR);
1498 if ((l1std & L1STD_VALID_BIT) == 0) {
1502 uint64_t l2ptr = l1std & ~((1ULL << L1STD_PTR_LOW_BITS) - 1);
1503 ste_addr = l2ptr + (l2_idx * SMMUV3_STE_SIZE);
1506 if (!dma_read(ste_addr, ste_buf.data(), SMMUV3_STE_SIZE)) {
1507 set_gerror(GERROR_BIT_SFM_ERR);
1511 uint64_t dw0 = load_le<uint64_t>(ste_buf.data() + 0);
1512 uint64_t dw1 = load_le<uint64_t>(ste_buf.data() + 8);
1513 uint64_t dw2 = load_le<uint64_t>(ste_buf.data() + 16);
1514 uint64_t dw3 = load_le<uint64_t>(ste_buf.data() + 24);
1516 ste.valid = dw0 & 1;
1517 ste.config = extract64(dw0, 1, 3);
1518 ste.s1fmt = extract64(dw0, 4, 2);
1519 ste.s1contextptr = dw0 & STE_BASE_ADDR_MASK;
1520 ste.s1cdmax = extract64(dw0, 59, 5);
1522 ste.s1stalld = extract64(dw1, 27, 1);
1523 ste.eats = extract64(dw1, 28, 2);
1524 ste.strw = extract64(dw1, 30,
1527 ste.vmid = extract64(dw2, 0, 16);
1528 ste.s2t0sz = extract64(dw2, 32, 6);
1529 ste.s2sl0 = extract64(dw2, 38, 2);
1530 ste.s2tg = extract64(dw2, 46, 2);
1531 ste.s2ps = extract64(dw2, 48, 3);
1532 ste.s2hd = extract64(dw2, 56, 1);
1533 ste.s2affd = extract64(dw2, 60, 1);
1535 ste.s2ttb = dw3 & S2TTB_ADDR_MASK;
1537 if (m_ste_cache.size() >= STE_CACHE_MAX && !m_ste_cache_lru.empty()) {
1538 uint32_t victim = m_ste_cache_lru.back();
1539 m_ste_cache_lru.pop_back();
1540 m_ste_cache.erase(victim);
1542 m_ste_cache[sid] = ste;
1543 m_ste_cache_lru.push_front(sid);
1547template <
unsigned int BUSWIDTH>
1548bool smmuv3<BUSWIDTH>::read_cd(uint32_t sid,
const STE& ste, uint32_t substream_id, CD& cd)
1550 uint64_t key = (
static_cast<uint64_t
>(sid) << 32) | substream_id;
1551 auto it = m_cd_cache.find(key);
1552 if (it != m_cd_cache.end()) {
1557 constexpr uint32_t CD_TSZ_MASK = 0x3F;
1558 constexpr uint32_t CD_TG_MASK = 0x3;
1559 constexpr uint32_t CD_IPS_MASK = 0x7;
1560 constexpr uint32_t CD_ASID_MASK = 0xFFFF;
1561 constexpr uint32_t CD_VALID_SHIFT = 31;
1562 constexpr uint32_t CD_AARCH64_SHIFT = 9;
1563 constexpr uint32_t CD_T0_TG_SHIFT = 6;
1564 constexpr uint32_t CD_EPD0_SHIFT = 14;
1565 constexpr uint32_t CD_T1SZ_SHIFT = 16;
1566 constexpr uint32_t CD_T1_TG_SHIFT = 22;
1567 constexpr uint32_t CD_EPD1_SHIFT = 30;
1568 constexpr uint32_t CD_ASID_SHIFT = 16;
1569 constexpr uint32_t CD_TTB_HI_MASK = 0xFFFFF;
1570 constexpr uint32_t CD_TTB_LO_MASK = 0xFFFFFFF0;
1571 constexpr uint32_t CD_AFFD_BIT_IN_W1 = 3;
1572 constexpr uint32_t CD_HA_BIT_IN_W1 = 11;
1573 constexpr uint32_t CD_HD_BIT_IN_W1 = 10;
1576 constexpr size_t L1CD_SIZE = 8;
1577 constexpr uint64_t L1CD_VALID_BIT = 1ULL;
1578 constexpr uint64_t L1CD_PTR_MASK = ~0xFFFULL;
1580 if (ste.s1fmt == S1FMT_LINEAR || ste.s1cdmax == 0) {
1581 cd_addr = ste.s1contextptr + (substream_id * SMMUV3_CD_SIZE);
1583 uint32_t split = (ste.s1fmt == S1FMT_4KB_L2) ? S1FMT_4KB_L2_SPLIT : S1FMT_64KB_L2_SPLIT;
1584 uint64_t l1_idx =
static_cast<uint64_t
>(substream_id) >> split;
1585 uint64_t l2_idx =
static_cast<uint64_t
>(substream_id) & ((1ULL << split) - 1);
1586 uint64_t l1cd_addr = ste.s1contextptr + (l1_idx * L1CD_SIZE);
1588 if (!dma_read(l1cd_addr, &l1cd, L1CD_SIZE)) {
1589 set_gerror(GERROR_BIT_SFM_ERR);
1592 if ((l1cd & L1CD_VALID_BIT) == 0) {
1596 uint64_t l2_ptr = l1cd & L1CD_PTR_MASK;
1597 cd_addr = l2_ptr + (l2_idx * SMMUV3_CD_SIZE);
1600 std::array<uint8_t, SMMUV3_CD_SIZE> cd_buf{};
1602 if (!dma_read(cd_addr, cd_buf.data(), SMMUV3_CD_SIZE)) {
1603 set_gerror(GERROR_BIT_SFM_ERR);
1607 std::array<uint32_t, 6> w{};
1608 for (
size_t i = 0; i < w.size(); ++i) {
1609 w[i] = load_le<uint32_t>(cd_buf.data() + i *
sizeof(uint32_t));
1612 cd.valid = (w[0] >> CD_VALID_SHIFT) & 0x1;
1613 cd.aarch64 = (w[1] >> CD_AARCH64_SHIFT) & 0x1;
1614 cd.tsz[0] = w[0] & CD_TSZ_MASK;
1615 cd.tg[0] = (w[0] >> CD_T0_TG_SHIFT) & CD_TG_MASK;
1616 cd.epd[0] = (w[0] >> CD_EPD0_SHIFT) & 0x1;
1617 cd.tsz[1] = (w[0] >> CD_T1SZ_SHIFT) & CD_TSZ_MASK;
1618 cd.tg[1] = (w[0] >> CD_T1_TG_SHIFT) & CD_TG_MASK;
1619 cd.epd[1] = (w[0] >> CD_EPD1_SHIFT) & 0x1;
1620 cd.ips = w[1] & CD_IPS_MASK;
1621 cd.asid = (w[1] >> CD_ASID_SHIFT) & CD_ASID_MASK;
1622 cd.affd = (w[1] >> CD_AFFD_BIT_IN_W1) & 0x1;
1623 cd.ha = (w[1] >> CD_HA_BIT_IN_W1) & 0x1;
1624 cd.hd = (w[1] >> CD_HD_BIT_IN_W1) & 0x1;
1626 cd.ttb[0] = (
static_cast<uint64_t
>(w[3] & CD_TTB_HI_MASK) << 32) |
static_cast<uint64_t
>(w[2] & CD_TTB_LO_MASK);
1627 cd.ttb[1] = (
static_cast<uint64_t
>(w[5] & CD_TTB_HI_MASK) << 32) |
static_cast<uint64_t
>(w[4] & CD_TTB_LO_MASK);
1629 if (m_cd_cache.size() >= CD_CACHE_MAX && !m_cd_cache_lru.empty()) {
1630 uint64_t victim = m_cd_cache_lru.back();
1631 m_cd_cache_lru.pop_back();
1632 m_cd_cache.erase(victim);
1634 m_cd_cache[key] = cd;
1635 m_cd_cache_lru.push_front(key);
1639template <
unsigned int BUSWIDTH>
1640bool smmuv3<BUSWIDTH>::smmuv3_ptw64(PtwCtx& ctx, TransReq& req)
1643 unsigned int inputsize;
1644 unsigned int outputsize;
1645 unsigned int grainsize = 0;
1646 unsigned int stride;
1648 unsigned int firstblocklevel = 0;
1650 unsigned int baselowerbound;
1651 bool blocktranslate =
false;
1661 req.fault_level = 0;
1663 constexpr uint32_t VA_BITS_TOTAL = 64;
1664 constexpr uint32_t STRIDE_OFFSET = 3;
1665 constexpr uint32_t LEVEL_LEAF = 3;
1666 constexpr uint32_t FIRSTBLOCK_LARGE_GRAIN = 2;
1667 constexpr uint32_t FIRSTBLOCK_4K = 1;
1668 constexpr uint32_t TG1_INVALID = 0xFF;
1670 if (ctx.stage == 1) {
1671 if ((ctx.iova & (1ULL << VA_SIGN_BIT_SHIFT)) == 0) {
1680 switch (ctx.tg[1]) {
1703 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1708 inputsize = VA_BITS_TOTAL - tsz;
1712 grainsize = GRAINSIZE_64K;
1714 firstblocklevel = FIRSTBLOCK_LARGE_GRAIN;
1717 grainsize = GRAINSIZE_16K;
1719 firstblocklevel = FIRSTBLOCK_LARGE_GRAIN;
1722 grainsize = GRAINSIZE_4K;
1724 firstblocklevel = FIRSTBLOCK_4K;
1727 SCP_ERR(()) <<
"Invalid TG value: " << tg;
1728 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1733 outputsize = OUTPUT_SIZE_MAP[ctx.ips];
1734 if (outputsize > p_pamax) outputsize = p_pamax;
1736 stride = grainsize - STRIDE_OFFSET;
1738 if (ctx.stage == 1) {
1739 if (grainsize < GRAINSIZE_64K && (inputsize > (grainsize + 3 * stride)))
1741 else if (inputsize > (grainsize + 2 * stride))
1743 else if (inputsize > (grainsize + stride))
1746 if (inputsize < INPUT_SIZE_MIN || inputsize > INPUT_SIZE_MAX ||
1747 extract64(ctx.iova, inputsize, VA_BITS_TOTAL - inputsize)) {
1748 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1753 unsigned int startlevel = ctx.sl0;
1754 level = 3 - startlevel;
1755 if (grainsize == GRAINSIZE_4K) level = 2 - startlevel;
1756 if (!check_s2_startlevel(outputsize, level, inputsize, stride)) {
1757 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1763 constexpr uint64_t INDEX_ALIGN_8B = 7ULL;
1765 unsigned int level_coverage = grainsize + (LEVEL_LEAF - level + 1) * stride;
1766 unsigned int concat_bits = 0;
1767 if (ctx.stage == 2 && inputsize > level_coverage) {
1768 concat_bits = inputsize - level_coverage;
1769 if (concat_bits > 4) {
1770 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1776 baselowerbound = STRIDE_OFFSET + inputsize - ((LEVEL_LEAF - level) * stride + grainsize);
1777 ttbr = extract64(ttbr, 0, PA_BITS);
1778 ttbr &= ~((1ULL << baselowerbound) - 1);
1780 if (concat_bits > 0) {
1781 uint64_t concat_idx = (ctx.iova >> level_coverage) & ((1ULL << concat_bits) - 1);
1782 uint64_t table_bytes = 1ULL << (grainsize + stride);
1783 ttbr |= concat_idx * table_bytes;
1786 if (!check_out_addr(ttbr, outputsize)) {
1787 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1792 descmask = (1ULL << grainsize) - 1;
1795 unsigned int addrselectbottom = (LEVEL_LEAF - level) * stride + grainsize;
1796 uint64_t index = (ctx.iova >> (addrselectbottom - STRIDE_OFFSET)) & descmask;
1797 index &= ~INDEX_ALIGN_8B;
1798 uint64_t descaddr = ttbr | index;
1800 if (ctx.stage == 1 && ctx.s2_enabled) {
1802 s2ctx.ttb[0] = ctx.ste->s2ttb;
1803 s2ctx.tsz[0] = ctx.ste->s2t0sz;
1804 s2ctx.tg[0] = ctx.ste->s2tg;
1805 s2ctx.ips = ctx.ste->s2ps;
1806 s2ctx.sl0 = ctx.ste->s2sl0;
1807 s2ctx.epd[0] =
false;
1809 s2ctx.iova = descaddr;
1810 s2ctx.access = IOMMUAccessFlags::RO;
1811 s2ctx.s2_enabled =
false;
1812 s2ctx.ste = ctx.ste;
1813 s2ctx.affd = ctx.ste->s2affd;
1815 s2ctx.hd = ctx.ste->s2hd;
1818 s2req.iova = descaddr;
1819 s2req.access = IOMMUAccessFlags::RO;
1820 s2req.pa = descaddr;
1822 if (!smmuv3_ptw64(s2ctx, s2req) || s2req.err) {
1823 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1824 req.fault_level = level;
1828 descaddr = s2req.pa;
1831 if (!dma_read(descaddr, &desc,
sizeof(desc))) {
1832 req.event_type = SMMUV3_EVT_F_WALK_EABT;
1833 req.fault_level = level;
1838 req.leaf_desc_addr = descaddr;
1840 constexpr uint32_t DESC_TYPE_MASK = 0x3;
1841 constexpr uint32_t TABLE_ATTRS_SHIFT = 59;
1842 constexpr uint32_t TABLE_ATTRS_WIDTH = 5;
1844 unsigned int type = desc & DESC_TYPE_MASK;
1845 SCP_INFO(()) <<
"S" << ctx.stage <<
" L" << level <<
" iova=0x" << std::hex << ctx.iova <<
" descaddr=0x"
1846 << descaddr <<
" desc=0x" << desc <<
" type=" << type;
1848 ttbr = extract64(desc, 0, PA_BITS);
1851 if (level == LEVEL_LEAF) {
1852 if (type != DESC_TYPE_TABLE_OR_PAGE) {
1853 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1854 req.fault_level = level;
1862 case DESC_TYPE_RESERVED:
1863 case DESC_TYPE_INVALID:
1864 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1865 req.fault_level = level;
1869 case DESC_TYPE_BLOCK:
1870 blocktranslate =
true;
1871 if (level < (
int)firstblocklevel) {
1872 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1873 req.fault_level = level;
1879 case DESC_TYPE_TABLE_OR_PAGE:
1880 req.tableattrs |= extract64(desc, TABLE_ATTRS_SHIFT, TABLE_ATTRS_WIDTH);
1881 if (!check_out_addr(ttbr, outputsize)) {
1882 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1883 req.fault_level = level;
1891 SCP_ERR(()) <<
"SMMUv3: unreachable descriptor type " << type;
1892 req.event_type = SMMUV3_EVT_F_TRANSLATION;
1893 req.fault_level = level;
1897 }
while (!blocktranslate);
1899 if (!check_out_addr(ttbr, outputsize)) {
1900 req.event_type = SMMUV3_EVT_F_ADDR_SIZE;
1901 req.fault_level = level;
1906 constexpr uint32_t LEVEL_COUNT = 4;
1907 constexpr uint32_t DESC_ATTRS_LOW_SHIFT = 2;
1908 constexpr uint32_t DESC_ATTRS_LOW_WIDTH = 10;
1909 constexpr uint32_t DESC_ATTRS_HI_SHIFT = 52;
1910 constexpr uint32_t DESC_ATTRS_HI_WIDTH = 12;
1911 constexpr uint32_t TABLE_AP_SHIFT_IN_ATTRS = 11;
1912 constexpr uint32_t TABLE_PXN_BIT_IN_TABLEATTRS = 3;
1914 unsigned long page_size = (1ULL << ((stride * (LEVEL_COUNT - level)) + STRIDE_OFFSET));
1915 ttbr |= (ctx.iova & (page_size - 1));
1916 req.page_size = ((stride * (LEVEL_COUNT - level)) + STRIDE_OFFSET);
1919 req.descriptor = desc;
1921 s2attrs = attrs = extract64(desc, DESC_ATTRS_LOW_SHIFT, DESC_ATTRS_LOW_WIDTH) |
1922 (extract64(desc, DESC_ATTRS_HI_SHIFT, DESC_ATTRS_HI_WIDTH) << DESC_ATTRS_LOW_WIDTH);
1924 if (ctx.stage == 1) {
1925 attrs |= extract32(req.tableattrs, 0, 2) << TABLE_AP_SHIFT_IN_ATTRS;
1926 attrs |= extract32(req.tableattrs, TABLE_PXN_BIT_IN_TABLEATTRS, 1) << DESC_ATTR_AP2_BIT;
1929 req.prot = IOMMUAccessFlags::RW;
1931 bool dbm_set = (desc >> DESC_BIT_DBM) & 1ULL;
1932 bool stage1_httu_af = (ctx.stage == 1) ? ctx.ha : false;
1933 bool stage1_httu_dbm = (ctx.stage == 1) ? ctx.hd : false;
1934 bool stage2_httu_af = (ctx.stage == 2) && ctx.ste && ctx.ste->s2hd;
1935 (void)stage2_httu_af;
1937 bool af_set = attrs & (1u << DESC_ATTR_AF_BIT);
1939 bool affd = (ctx.stage == 1) ? ctx.affd : (ctx.ste ? ctx.ste->s2affd : false);
1940 bool httu_af = stage1_httu_af || stage2_httu_af;
1942 uint64_t new_desc = desc | (1ULL << DESC_BIT_AF);
1943 if (dma_write(req.leaf_desc_addr, &new_desc,
sizeof(new_desc))) {
1945 attrs |= (1u << DESC_ATTR_AF_BIT);
1949 if (!af_set && !affd) {
1950 req.event_type = SMMUV3_EVT_F_ACCESS;
1951 req.fault_level = level;
1957 if (ctx.stage == 1) {
1958 if (!(attrs & (1u << DESC_ATTR_AP1_BIT))) {
1959 req.event_type = SMMUV3_EVT_F_PERMISSION;
1960 req.fault_level = level;
1964 if (attrs & (1u << DESC_ATTR_AP2_BIT)) {
1965 if (ctx.access == IOMMUAccessFlags::WO) {
1966 if (stage1_httu_dbm && dbm_set) {
1967 uint64_t new_desc = desc & ~(1ULL << (DESC_ATTRS_LOW_SHIFT + DESC_ATTR_AP2_BIT));
1968 if (dma_write(req.leaf_desc_addr, &new_desc,
sizeof(new_desc))) {
1970 attrs &= ~(1u << DESC_ATTR_AP2_BIT);
1973 if (attrs & (1u << DESC_ATTR_AP2_BIT)) {
1974 req.event_type = SMMUV3_EVT_F_PERMISSION;
1975 req.fault_level = level;
1980 req.prot &= ~IOMMUAccessFlags::WO;
1984 switch ((s2attrs >> DESC_ATTR_AP1_BIT) & 0x3) {
1986 req.event_type = SMMUV3_EVT_F_PERMISSION;
1987 req.fault_level = level;
1991 if (ctx.access == IOMMUAccessFlags::WO) {
1992 req.event_type = SMMUV3_EVT_F_PERMISSION;
1993 req.fault_level = level;
1997 req.prot &= ~IOMMUAccessFlags::WO;
2000 if (ctx.access == IOMMUAccessFlags::RO) {
2001 req.event_type = SMMUV3_EVT_F_PERMISSION;
2002 req.fault_level = level;
2006 req.prot &= ~IOMMUAccessFlags::RO;
2011 SCP_ERR(()) <<
"SMMUv3: unreachable S2AP value";
2012 req.event_type = SMMUV3_EVT_F_PERMISSION;
2013 req.fault_level = level;
2019 SCP_INFO(()) <<
"PTW success: 0x" << std::hex << ctx.iova <<
" -> 0x" << req.pa
2020 <<
" prot=" <<
static_cast<uint32_t
>(req.prot) <<
" page_size=" << std::dec << req.page_size;
2024template <
unsigned int BUSWIDTH>
2026 uint32_t substream_id,
bool is_secure,
2032 uint64_t iova = txn.get_address();
2033 IOMMUTLBEntry ret = {
2035 .translated_addr = iova,
2036 .addr_mask = (1ULL << PAGE_SHIFT_4K) - 1,
2037 .perm = IOMMUAccessFlags::RW,
2042 .stallable_fault =
false,
2045 if (
static_cast<uint32_t
>(CR0[CR0_SMMUEN]) == 0) {
2046 ret.addr_mask = -1ULL;
2053 req.substream_id = substream_id;
2054 req.access = (txn.get_command() == tlm::TLM_WRITE_COMMAND) ? IOMMUAccessFlags::WO : IOMMUAccessFlags::RO;
2056 if (!find_ste(sid, req.ste) || !req.ste.valid) {
2057 if (
static_cast<uint32_t
>(GBPA[GBPA_ABORT])) {
2058 ret.perm = IOMMUAccessFlags::NONE;
2059 ret.fault_type = SMMUV3_EVT_F_STE_FETCH;
2060 record_event_full(SMMUV3_EVT_F_STE_FETCH, sid, substream_id, iova, txn, 0, 0, 0,
false);
2063 ret.addr_mask = -1ULL;
2068 if (req.ste.s1cdmax > 0 && substream_id >= (1u << req.ste.s1cdmax)) {
2069 ret.perm = IOMMUAccessFlags::NONE;
2070 ret.fault_type = SMMUV3_EVT_C_BAD_SUBSTREAMID;
2071 record_event_full(SMMUV3_EVT_C_BAD_SUBSTREAMID, sid, substream_id, iova, txn, 0, 0, 0,
false);
2077 const bool e2h_enabled =
static_cast<uint32_t
>(CR2[CR2_E2H]) != 0;
2078 const uint8_t iotlb_regime = compute_iotlb_regime(req.ste.strw, e2h_enabled, is_secure);
2080 constexpr uint8_t IOTLB_LEAF_LEVEL = 3;
2082 switch (req.ste.config) {
2083 case STE_CONFIG_ABORT:
2084 ret.perm = IOMMUAccessFlags::NONE;
2085 ret.fault_type = SMMUV3_EVT_C_BAD_STE;
2086 record_event_full(SMMUV3_EVT_C_BAD_STE, sid, substream_id, iova, txn, 0, 0, 0,
false);
2088 case STE_CONFIG_BYPASS:
2089 ret.addr_mask = -1ULL;
2092 if (!read_cd(sid, req.ste, substream_id, req.cd) || !req.cd.valid) {
2093 ret.perm = IOMMUAccessFlags::NONE;
2094 ret.fault_type = SMMUV3_EVT_F_CD_FETCH;
2095 record_event_full(SMMUV3_EVT_F_CD_FETCH, sid, substream_id, iova, txn, SMMUV3_CLASS_CD, 0, 0,
false);
2099 IOTLBKey lookup{ req.ste.vmid,
2101 iova & ~((1ULL << PAGE_SHIFT_4K) - 1),
2102 static_cast<uint8_t
>(req.cd.tg[0]),
2106 if (iotlb_lookup(lookup, ret)) {
2112 ctx.ttb[0] = req.cd.ttb[0];
2113 ctx.ttb[1] = req.cd.ttb[1];
2114 ctx.tsz[0] = req.cd.tsz[0];
2115 ctx.tsz[1] = req.cd.tsz[1];
2116 ctx.tg[0] = req.cd.tg[0];
2117 ctx.tg[1] = req.cd.tg[1];
2118 ctx.ips = req.cd.ips;
2119 ctx.epd[0] = req.cd.epd[0];
2120 ctx.epd[1] = req.cd.epd[1];
2123 ctx.access = req.access;
2124 ctx.s2_enabled =
false;
2126 ctx.affd = req.cd.affd;
2129 smmuv3_ptw64(ctx, req);
2132 case STE_CONFIG_S2: {
2133 IOTLBKey lookup{ req.ste.vmid,
2135 iova & ~((1ULL << PAGE_SHIFT_4K) - 1),
2136 static_cast<uint8_t
>(req.ste.s2tg),
2140 if (iotlb_lookup(lookup, ret)) {
2144 ctx.ttb[0] = req.ste.s2ttb;
2145 ctx.tsz[0] = req.ste.s2t0sz;
2146 ctx.tg[0] = req.ste.s2tg;
2147 ctx.ips = req.ste.s2ps;
2148 ctx.sl0 = req.ste.s2sl0;
2152 ctx.access = req.access;
2153 ctx.s2_enabled =
false;
2155 ctx.affd = req.ste.s2affd;
2157 ctx.hd = req.ste.s2hd;
2158 smmuv3_ptw64(ctx, req);
2160 case STE_CONFIG_NESTED:
2161 if (!read_cd(sid, req.ste, substream_id, req.cd) || !req.cd.valid) {
2162 ret.perm = IOMMUAccessFlags::NONE;
2163 ret.fault_type = SMMUV3_EVT_F_CD_FETCH;
2164 record_event_full(SMMUV3_EVT_F_CD_FETCH, sid, substream_id, iova, txn, SMMUV3_CLASS_CD, 0, 0,
false);
2168 IOTLBKey lookup{ req.ste.vmid,
2170 iova & ~((1ULL << PAGE_SHIFT_4K) - 1),
2171 static_cast<uint8_t
>(req.cd.tg[0]),
2175 if (iotlb_lookup(lookup, ret)) {
2181 ctx.ttb[0] = req.cd.ttb[0];
2182 ctx.ttb[1] = req.cd.ttb[1];
2183 ctx.tsz[0] = req.cd.tsz[0];
2184 ctx.tsz[1] = req.cd.tsz[1];
2185 ctx.tg[0] = req.cd.tg[0];
2186 ctx.tg[1] = req.cd.tg[1];
2187 ctx.ips = req.cd.ips;
2188 ctx.epd[0] = req.cd.epd[0];
2189 ctx.epd[1] = req.cd.epd[1];
2192 ctx.access = req.access;
2193 ctx.s2_enabled =
true;
2195 ctx.affd = req.cd.affd;
2198 smmuv3_ptw64(ctx, req);
2201 uint64_t s1_pa = req.pa;
2203 s2ctx.ttb[0] = req.ste.s2ttb;
2204 s2ctx.tsz[0] = req.ste.s2t0sz;
2205 s2ctx.tg[0] = req.ste.s2tg;
2206 s2ctx.ips = req.ste.s2ps;
2207 s2ctx.sl0 = req.ste.s2sl0;
2208 s2ctx.epd[0] =
false;
2211 s2ctx.access = req.access;
2212 s2ctx.s2_enabled =
false;
2213 s2ctx.ste = &req.ste;
2214 s2ctx.affd = req.ste.s2affd;
2216 s2ctx.hd = req.ste.s2hd;
2217 smmuv3_ptw64(s2ctx, req);
2222 ret.perm = IOMMUAccessFlags::NONE;
2223 ret.fault_type = SMMUV3_EVT_C_BAD_STE;
2224 record_event_full(SMMUV3_EVT_C_BAD_STE, sid, substream_id, iova, txn, 0, 0, 0,
false);
2229 ret.perm = IOMMUAccessFlags::NONE;
2230 ret.fault_type = req.event_type;
2231 ret.fault_level = req.fault_level;
2232 bool stallable = (req.event_type == SMMUV3_EVT_F_TRANSLATION || req.event_type == SMMUV3_EVT_F_PERMISSION ||
2233 req.event_type == SMMUV3_EVT_F_ACCESS || req.event_type == SMMUV3_EVT_F_ADDR_SIZE);
2234 ret.stallable_fault = stallable && (req.ste.config == STE_CONFIG_S1 || req.ste.config == STE_CONFIG_NESTED) &&
2236 uint32_t class_ = (req.ste.config == STE_CONFIG_S2) ? SMMUV3_CLASS_IN : SMMUV3_CLASS_TT;
2237 if (!ret.stallable_fault) {
2238 record_event_full(req.event_type, sid, substream_id, iova, txn, class_, req.fault_level, 0,
false);
2243 ret.translated_addr = req.pa;
2244 ret.perm = req.prot;
2245 ret.addr_mask = (1ULL << req.page_size) - 1;
2246 ret.descriptor = req.descriptor;
2247 ret.table_attrs = req.tableattrs;
2250 key.vmid = req.ste.vmid;
2251 key.iova = iova & ~ret.addr_mask;
2252 key.level = IOTLB_LEAF_LEVEL;
2253 key.secure = is_secure;
2254 key.regime = iotlb_regime;
2256 if (req.ste.config == STE_CONFIG_S1 || req.ste.config == STE_CONFIG_NESTED) {
2257 key.asid = req.cd.asid;
2258 key.tg =
static_cast<uint8_t
>(req.cd.tg[0]);
2261 key.tg =
static_cast<uint8_t
>(req.ste.s2tg);
2263 iotlb_insert(key, ret);
2268template <
unsigned int BUSWIDTH>
2269void smmuv3<BUSWIDTH>::iotlb_insert(
const IOTLBKey& key,
const IOMMUTLBEntry& entry)
2271 if (m_iotlb.find(key) != m_iotlb.end()) {
2272 m_iotlb_lru.remove(key);
2273 }
else if (m_iotlb.size() >= p_iotlb_size.get_value()) {
2274 auto lru_key = m_iotlb_lru.back();
2275 m_iotlb_lru.pop_back();
2276 m_iotlb.erase(lru_key);
2278 m_iotlb[key] = entry;
2279 m_iotlb_lru.push_front(key);
2282template <
unsigned int BUSWIDTH>
2283bool smmuv3<BUSWIDTH>::iotlb_lookup(
const IOTLBKey& key, IOMMUTLBEntry& entry)
2285 auto it = m_iotlb.find(key);
2286 if (it != m_iotlb.end()) {
2293template <
unsigned int BUSWIDTH>
2294void smmuv3<BUSWIDTH>::iotlb_inv_all()
2297 m_iotlb_lru.clear();
2300template <
unsigned int BUSWIDTH>
2301void smmuv3<BUSWIDTH>::iotlb_inv_asid(uint16_t asid)
2303 for (
auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2304 if (it->first.asid == asid) {
2305 m_iotlb_lru.remove(it->first);
2306 it = m_iotlb.erase(it);
2313template <
unsigned int BUSWIDTH>
2314void smmuv3<BUSWIDTH>::iotlb_inv_vmid(uint16_t vmid)
2316 for (
auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2317 if (it->first.vmid == vmid) {
2318 m_iotlb_lru.remove(it->first);
2319 it = m_iotlb.erase(it);
2326template <
unsigned int BUSWIDTH>
2327void smmuv3<BUSWIDTH>::iotlb_inv_iova(uint16_t vmid, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t addr_mask)
2329 uint64_t iova_masked = iova & ~addr_mask;
2330 for (
auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2331 uint64_t entry_masked = it->first.iova & ~addr_mask;
2332 if (it->first.vmid == vmid && it->first.asid == asid && entry_masked == iova_masked && it->first.tg == tg) {
2333 m_iotlb_lru.remove(it->first);
2334 it = m_iotlb.erase(it);
2341template <
unsigned int BUSWIDTH>
2342void smmuv3<BUSWIDTH>::consume_cmdq()
2344 constexpr uint64_t CMDQ_BASE_LOG2_BITS_MASK = 0x1FULL;
2346 uint64_t hi =
static_cast<uint64_t
>(CMDQ_BASE_HI) & CMDQ_BASE_HI_MASK;
2347 uint64_t lo =
static_cast<uint64_t
>(CMDQ_BASE_LO) & ~CMDQ_BASE_LOG2_BITS_MASK;
2348 uint64_t base = (hi << 32) | lo;
2349 uint32_t log2size = clamp_queue_log2size(
static_cast<uint32_t
>(CMDQ_BASE_LO[CMDQ_BASE_LOG2SIZE]));
2350 if (log2size == 0)
return;
2352 uint32_t prod =
static_cast<uint32_t
>(CMDQ_PROD);
2353 uint32_t cons =
static_cast<uint32_t
>(CMDQ_CONS);
2354 uint32_t wrap_mask = (1u << (log2size + 1)) - 1;
2355 uint32_t idx_mask = (1u << log2size) - 1;
2357 SCP_INFO(()) <<
"SMMUv3: consume_cmdq base=0x" << std::hex << base <<
" log2size=" << std::dec << log2size
2358 <<
" prod=0x" << std::hex << prod <<
" cons=0x" << cons;
2360 while ((prod & wrap_mask) != (cons & wrap_mask)) {
2361 uint32_t idx = cons & idx_mask;
2362 uint64_t cmd_addr = base + (idx * SMMUV3_CMD_SIZE);
2363 std::array<uint8_t, SMMUV3_CMD_SIZE> cmd{};
2365 if (!dma_read(cmd_addr, cmd.data(), SMMUV3_CMD_SIZE)) {
2366 SCP_WARN(()) <<
"SMMUv3: dma_read failed at 0x" << std::hex << cmd_addr;
2368 set_cmdq_err(SMMUV3_CERROR_ABT);
2372 uint8_t opcode = cmd[0];
2373 SCP_INFO(()) <<
"SMMUv3: cmd opcode=0x" << std::hex << (int)opcode <<
" at idx=" << std::dec << idx;
2376 case CMD_OP_PREFETCH_CONFIG:
2377 handle_cmd_prefetch_config(cmd.data());
2379 case CMD_OP_PREFETCH_ADDR:
2380 handle_cmd_prefetch_addr(cmd.data());
2382 case CMD_OP_CFGI_STE:
2383 handle_cmd_cfgi_ste(cmd.data());
2385 case CMD_OP_CFGI_STE_RANGE:
2386 handle_cmd_cfgi_ste_range(cmd.data());
2388 case CMD_OP_CFGI_CD:
2389 handle_cmd_cfgi_cd(cmd.data());
2391 case CMD_OP_CFGI_CD_ALL:
2392 handle_cmd_cfgi_cd_all(cmd.data());
2394 case CMD_OP_CFGI_ALL:
2395 handle_cmd_cfgi_all(cmd.data());
2397 case CMD_OP_TLBI_NH_ALL:
2398 handle_cmd_tlbi_nh_all(cmd.data());
2400 case CMD_OP_TLBI_NH_ASID:
2401 handle_cmd_tlbi_nh_asid(cmd.data());
2403 case CMD_OP_TLBI_NH_VA:
2404 handle_cmd_tlbi_nh_va(cmd.data());
2406 case CMD_OP_TLBI_NH_VAA:
2407 handle_cmd_tlbi_nh_vaa(cmd.data());
2409 case CMD_OP_TLBI_EL3_ALL:
2410 handle_cmd_tlbi_nh_all(cmd.data());
2412 case CMD_OP_TLBI_EL3_VA:
2413 handle_cmd_tlbi_nh_va(cmd.data());
2415 case CMD_OP_TLBI_EL2_ALL:
2416 handle_cmd_tlbi_nh_all(cmd.data());
2418 case CMD_OP_TLBI_EL2_ASID:
2419 handle_cmd_tlbi_nh_asid(cmd.data());
2421 case CMD_OP_TLBI_EL2_VA:
2422 handle_cmd_tlbi_nh_va(cmd.data());
2424 case CMD_OP_TLBI_EL2_VAA:
2425 handle_cmd_tlbi_nh_vaa(cmd.data());
2427 case CMD_OP_TLBI_S12_VMALL:
2428 handle_cmd_tlbi_s12_vmall(cmd.data());
2430 case CMD_OP_TLBI_S2_IPA:
2431 handle_cmd_tlbi_s2_ipa(cmd.data());
2433 case CMD_OP_TLBI_NSNH_ALL:
2434 handle_cmd_tlbi_nsnh_all(cmd.data());
2436 case CMD_OP_ATC_INV:
2437 handle_cmd_atc_inv(cmd.data());
2439 case CMD_OP_PRI_RESP:
2440 handle_cmd_pri_resp(cmd.data());
2443 handle_cmd_resume(cmd.data());
2445 case CMD_OP_STALL_TERM:
2448 handle_cmd_sync(cmd.data());
2451 SCP_WARN(()) <<
"SMMUv3: unknown opcode 0x" << std::hex << (int)opcode;
2453 set_cmdq_err(SMMUV3_CERROR_ILL);
2457 cons = (cons + 1) & wrap_mask;
2461 SCP_INFO(()) <<
"SMMUv3: consume_cmdq done, CMDQ_CONS=0x" << std::hex << cons;
2464template <
unsigned int BUSWIDTH>
2465void smmuv3<BUSWIDTH>::consume_secure_cmdq()
2467 constexpr uint64_t CMDQ_BASE_LOG2_BITS_MASK = 0x1FULL;
2469 uint64_t hi =
static_cast<uint64_t
>(S_CMDQ_BASE_HI) & CMDQ_BASE_HI_MASK;
2470 uint64_t lo =
static_cast<uint64_t
>(S_CMDQ_BASE_LO) & ~CMDQ_BASE_LOG2_BITS_MASK;
2471 uint64_t base = (hi << 32) | lo;
2472 uint32_t log2size = clamp_queue_log2size(
static_cast<uint32_t
>(S_CMDQ_BASE_LO[S_CMDQ_BASE_LOG2SIZE]));
2473 if (log2size == 0)
return;
2475 uint32_t prod =
static_cast<uint32_t
>(S_CMDQ_PROD);
2476 uint32_t cons =
static_cast<uint32_t
>(S_CMDQ_CONS);
2477 uint32_t wrap_mask = (1u << (log2size + 1)) - 1;
2478 uint32_t idx_mask = (1u << log2size) - 1;
2480 while ((prod & wrap_mask) != (cons & wrap_mask)) {
2481 uint32_t idx = cons & idx_mask;
2482 uint64_t cmd_addr = base + (idx * SMMUV3_CMD_SIZE);
2483 std::array<uint8_t, SMMUV3_CMD_SIZE> cmd{};
2485 if (!dma_read(cmd_addr, cmd.data(), SMMUV3_CMD_SIZE)) {
2487 set_secure_cmdq_err(SMMUV3_CERROR_ABT);
2491 constexpr size_t CMD_SID_OFFSET = 4;
2492 constexpr size_t CMD_WORD1_OFFSET = 8;
2493 constexpr uint32_t CMD_SYNC_CS_SHIFT_LOCAL = 12;
2494 constexpr uint32_t CMD_SYNC_CS_MASK_LOCAL = 0x3;
2496 uint8_t opcode = cmd[0];
2498 case CMD_OP_PREFETCH_CONFIG:
2499 case CMD_OP_PREFETCH_ADDR:
2501 case CMD_OP_CFGI_STE: {
2502 uint32_t sid = load_le<uint32_t>(cmd.data() + CMD_SID_OFFSET);
2503 m_ste_cache_secure.erase(sid);
2506 case CMD_OP_CFGI_STE_RANGE:
2507 m_ste_cache_secure.clear();
2509 case CMD_OP_CFGI_CD:
2510 case CMD_OP_CFGI_CD_ALL:
2511 m_cd_cache_secure.clear();
2513 case CMD_OP_CFGI_ALL:
2514 m_ste_cache_secure.clear();
2515 m_cd_cache_secure.clear();
2517 case CMD_OP_TLBI_NSNH_ALL:
2518 case CMD_OP_TLBI_NH_ALL:
2519 case CMD_OP_TLBI_NH_ASID:
2520 case CMD_OP_TLBI_NH_VA:
2521 case CMD_OP_TLBI_NH_VAA:
2522 case CMD_OP_TLBI_S12_VMALL:
2523 case CMD_OP_TLBI_S2_IPA:
2527 uint64_t dw0 = load_le<uint64_t>(cmd.data());
2528 uint64_t dw1 = load_le<uint64_t>(cmd.data() + CMD_WORD1_OFFSET);
2529 uint32_t cs = (
static_cast<uint32_t
>(dw0) >> CMD_SYNC_CS_SHIFT_LOCAL) & CMD_SYNC_CS_MASK_LOCAL;
2530 if (cs == SYNC_CS_IRQ) {
2531 constexpr uint64_t CMD_SYNC_MSIADDR_MASK = 0x000FFFFFFFFFFFFCULL;
2532 constexpr uint32_t CMD_SYNC_MSIDATA_SHIFT = 32;
2533 uint64_t msiaddr = dw1 & CMD_SYNC_MSIADDR_MASK;
2534 uint32_t msidata =
static_cast<uint32_t
>(dw0 >> CMD_SYNC_MSIDATA_SHIFT);
2536 if (!dma_write(msiaddr, &msidata,
sizeof(msidata))) {
2537 set_secure_gerror(GERROR_BIT_MSI_CMDQ_ABT);
2545 set_secure_cmdq_err(SMMUV3_CERROR_ILL);
2549 cons = (cons + 1) & wrap_mask;
2555template <
unsigned int BUSWIDTH>
2556void smmuv3<BUSWIDTH>::record_secure_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info)
2558 if (
static_cast<uint32_t
>(S_CR0[S_CR0_EVENTQEN]) == 0)
return;
2560 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2561 constexpr size_t EVT_SID_OFFSET = 4;
2562 constexpr size_t EVT_IOVA_OFFSET = 8;
2563 constexpr size_t EVT_INFO_OFFSET = 16;
2565 uint64_t base = (
static_cast<uint64_t
>(S_EVENTQ_BASE_HI) << 32) |
2566 (
static_cast<uint64_t
>(S_EVENTQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2567 uint32_t log2size = clamp_queue_log2size(
static_cast<uint32_t
>(S_EVENTQ_BASE_LO[S_EVENTQ_BASE_LOG2SIZE]));
2568 uint32_t prod =
static_cast<uint32_t
>(S_EVENTQ_PROD);
2569 uint32_t cons =
static_cast<uint32_t
>(S_EVENTQ_CONS);
2571 if (queue_full(prod, cons, log2size)) {
2572 set_secure_gerror(GERROR_BIT_EVENTQ_ABT);
2576 uint32_t idx = prod & ((1u << log2size) - 1);
2577 uint64_t evt_addr = base + (idx * SMMUV3_EVENT_SIZE);
2578 std::array<uint8_t, SMMUV3_EVENT_SIZE> evt{};
2581 store_le<uint32_t>(evt.data() + EVT_SID_OFFSET, sid);
2582 store_le<uint64_t>(evt.data() + EVT_IOVA_OFFSET, iova);
2583 store_le<uint32_t>(evt.data() + EVT_INFO_OFFSET, info);
2585 if (!dma_write(evt_addr, evt.data(), SMMUV3_EVENT_SIZE)) {
2586 set_secure_gerror(GERROR_BIT_EVENTQ_ABT);
2590 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2591 S_EVENTQ_PROD = prod;
2592 update_secure_irq_levels();
2595template <
unsigned int BUSWIDTH>
2596bool smmuv3<BUSWIDTH>::queue_full(uint32_t prod, uint32_t cons, uint32_t size)
2598 size = clamp_queue_log2size(size);
2599 uint32_t mask = (1u << size) - 1;
2600 uint32_t wrap = 1u << size;
2601 return ((prod ^ cons) == wrap) && ((prod & mask) == (cons & mask));
2604template <
unsigned int BUSWIDTH>
2605void smmuv3<BUSWIDTH>::record_event(uint32_t type, uint32_t sid, uint64_t iova, uint32_t info)
2607 if (
static_cast<uint32_t
>(CR0[CR0_EVENTQEN]) == 0) {
2611 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2612 constexpr size_t EVT_SID_OFFSET = 4;
2613 constexpr size_t EVT_IOVA_OFFSET = 8;
2614 constexpr size_t EVT_INFO_OFFSET = 16;
2616 uint64_t base = (
static_cast<uint64_t
>(EVENTQ_BASE_HI) << 32) |
2617 (
static_cast<uint64_t
>(EVENTQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2618 uint32_t log2size = clamp_queue_log2size(
static_cast<uint32_t
>(EVENTQ_BASE_LO[EVENTQ_BASE_LOG2SIZE]));
2619 uint32_t prod =
static_cast<uint32_t
>(EVENTQ_PROD);
2620 uint32_t cons =
static_cast<uint32_t
>(EVENTQ_CONS);
2622 if (queue_full(prod, cons, log2size)) {
2623 set_gerror(GERROR_BIT_EVENTQ_ABT);
2627 uint32_t idx = prod & ((1u << log2size) - 1);
2628 uint64_t evt_addr = base + (idx * SMMUV3_EVENT_SIZE);
2629 std::array<uint8_t, SMMUV3_EVENT_SIZE> evt{};
2632 store_le<uint32_t>(evt.data() + EVT_SID_OFFSET, sid);
2633 store_le<uint64_t>(evt.data() + EVT_IOVA_OFFSET, iova);
2634 store_le<uint32_t>(evt.data() + EVT_INFO_OFFSET, info);
2636 if (!dma_write(evt_addr, evt.data(), SMMUV3_EVENT_SIZE)) {
2637 set_gerror(GERROR_BIT_EVENTQ_ABT);
2641 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2643 update_irq_levels();
2646template <
unsigned int BUSWIDTH>
2647bool smmuv3<BUSWIDTH>::stall_and_wait(uint32_t sid, uint32_t ssid, uint64_t iova, uint32_t fault_type,
2648 uint32_t fault_level, uint32_t class_, tlm::tlm_generic_payload& txn)
2650 uint32_t stag = m_next_stag++;
2651 auto& stalled = m_stalled_txns[stag];
2652 stalled.stag = stag;
2654 stalled.substream_id = ssid;
2655 stalled.aborted =
false;
2657 record_event_full(fault_type, sid, ssid, iova, txn, class_, fault_level, stag,
true);
2659 sc_core::wait(stalled.resume_event);
2661 bool aborted = stalled.aborted;
2662 m_stalled_txns.erase(stag);
2666template <
unsigned int BUSWIDTH>
2667void smmuv3<BUSWIDTH>::record_event_full(uint32_t type, uint32_t sid, uint32_t ssid, uint64_t iova,
2668 tlm::tlm_generic_payload& txn, uint32_t class_, uint32_t fault_level,
2669 uint32_t stag,
bool stall)
2671 if (
static_cast<uint32_t
>(CR0[CR0_EVENTQEN]) == 0)
return;
2673 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2674 uint64_t base = (
static_cast<uint64_t
>(EVENTQ_BASE_HI) << 32) |
2675 (
static_cast<uint64_t
>(EVENTQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2676 uint32_t log2size = clamp_queue_log2size(
static_cast<uint32_t
>(EVENTQ_BASE_LO[EVENTQ_BASE_LOG2SIZE]));
2677 uint32_t prod =
static_cast<uint32_t
>(EVENTQ_PROD);
2678 uint32_t cons =
static_cast<uint32_t
>(EVENTQ_CONS);
2680 if (queue_full(prod, cons, log2size)) {
2681 set_gerror(GERROR_BIT_EVENTQ_ABT);
2685 uint32_t idx = prod & ((1u << log2size) - 1);
2686 uint64_t evt_addr = base + (idx * SMMUV3_EVENT_SIZE);
2687 std::array<uint64_t, 4> evt{};
2689 bool rnw = (txn.get_command() == tlm::TLM_READ_COMMAND);
2690 bool ssv = (ssid != 0);
2692 evt[0] = (
static_cast<uint64_t
>(type) & EVT_W0_TYPE_MASK) | (ssv ? (1ULL << EVT_W0_SSV_BIT) : 0ULL) |
2693 ((
static_cast<uint64_t
>(ssid) & EVT_W0_SSID_MASK) << EVT_W0_SSID_SHIFT) |
2694 (
static_cast<uint64_t
>(sid) << EVT_W0_STREAMID_SHIFT);
2696 evt[1] = (
static_cast<uint64_t
>(stag) & EVT_W1_STAG_MASK) | (rnw ? (1ULL << EVT_W1_RNW_BIT) : 0ULL) |
2697 ((
static_cast<uint64_t
>(class_) & EVT_W1_CLASS_MASK) << EVT_W1_CLASS_SHIFT) |
2698 (stall ? (1ULL << EVT_W1_STALL_BIT) : 0ULL) |
2699 ((
static_cast<uint64_t
>(fault_level) & EVT_W1_REASON_MASK) << EVT_W1_REASON_SHIFT);
2704 if (!dma_write(evt_addr, evt.data(), SMMUV3_EVENT_SIZE)) {
2705 set_gerror(GERROR_BIT_EVENTQ_ABT);
2709 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2711 update_irq_levels();
2714template <
unsigned int BUSWIDTH>
2715void smmuv3<BUSWIDTH>::record_pri(uint32_t sid, uint64_t iova, uint32_t flags)
2717 if (
static_cast<uint32_t
>(CR0[CR0_PRIQEN]) == 0) {
2721 constexpr uint64_t QUEUE_BASE_LOG2_BITS = 5;
2722 constexpr size_t PRI_SID_OFFSET = 0;
2723 constexpr size_t PRI_FLAGS_OFFSET = 4;
2724 constexpr size_t PRI_IOVA_OFFSET = 8;
2726 uint64_t base = (
static_cast<uint64_t
>(PRIQ_BASE_HI) << 32) |
2727 (
static_cast<uint64_t
>(PRIQ_BASE_LO) & ~((1ULL << QUEUE_BASE_LOG2_BITS) - 1));
2728 uint32_t log2size = clamp_queue_log2size(
static_cast<uint32_t
>(PRIQ_BASE_LO[PRIQ_BASE_LOG2SIZE]));
2729 uint32_t prod =
static_cast<uint32_t
>(PRIQ_PROD);
2730 uint32_t cons =
static_cast<uint32_t
>(PRIQ_CONS);
2732 if (queue_full(prod, cons, log2size)) {
2733 set_gerror(GERROR_BIT_PRIQ_ABT);
2737 uint32_t idx = prod & ((1u << log2size) - 1);
2738 uint64_t pri_addr = base + (idx * SMMUV3_PRI_SIZE);
2739 std::array<uint8_t, SMMUV3_PRI_SIZE> pri{};
2741 store_le<uint32_t>(pri.data() + PRI_SID_OFFSET, sid);
2742 store_le<uint32_t>(pri.data() + PRI_FLAGS_OFFSET, flags);
2743 store_le<uint64_t>(pri.data() + PRI_IOVA_OFFSET, iova);
2745 if (!dma_write(pri_addr, pri.data(), SMMUV3_PRI_SIZE)) {
2746 set_gerror(GERROR_BIT_PRIQ_ABT);
2750 prod = (prod + 1) & ((1u << (log2size + 1)) - 1);
2752 update_irq_levels();
2755template <
unsigned int BUSWIDTH>
2756void smmuv3<BUSWIDTH>::handle_cmd_prefetch_config(
const uint8_t*)
2760template <
unsigned int BUSWIDTH>
2761void smmuv3<BUSWIDTH>::handle_cmd_prefetch_addr(
const uint8_t*)
2765namespace detail_smmuv3 {
2766constexpr size_t CMD_SID_OFFSET = 4;
2767constexpr size_t CMD_ASID_OFFSET = 6;
2768constexpr size_t CMD_WORD1_OFFSET = 8;
2769constexpr size_t CMD_STAG_OFFSET = 4;
2770constexpr size_t CMD_RESUME_RESP_OFFSET = 11;
2771constexpr uint32_t CMD_WORD0_TG_SHIFT = 10;
2772constexpr uint32_t CMD_WORD0_TG_MASK = 0x3;
2773constexpr uint32_t CMD_WORD0_SCALE_SHIFT = 20;
2774constexpr uint32_t CMD_WORD0_SCALE_MASK = 0xF;
2775constexpr uint32_t CMD_WORD0_VMID_SHIFT = 32;
2776constexpr uint32_t CMD_WORD0_VMID_MASK = 0xFFFF;
2777constexpr uint32_t CMD_WORD0_ASID_SHIFT = 48;
2778constexpr uint32_t CMD_WORD0_ASID_MASK = 0xFFFF;
2779constexpr uint32_t CMD_SYNC_CS_SHIFT = 12;
2780constexpr uint32_t CMD_SYNC_CS_MASK = 0x3;
2781constexpr uint64_t CMD_VA_LOW_MASK = 0xFFFULL;
2784 uint32_t page_shift;
2792 return { PAGE_SHIFT_4K, CD_TG_4K };
2794 return { PAGE_SHIFT_16K, CD_TG_16K };
2796 return { PAGE_SHIFT_64K, CD_TG_64K };
2798 return { PAGE_SHIFT_4K, CD_TG_4K };
2803template <
unsigned int BUSWIDTH>
2804void smmuv3<BUSWIDTH>::handle_cmd_cfgi_ste(
const uint8_t* cmd)
2806 uint32_t sid = load_le<uint32_t>(cmd + detail_smmuv3::CMD_SID_OFFSET);
2807 m_ste_cache.erase(sid);
2808 m_ste_cache_lru.remove(sid);
2811template <
unsigned int BUSWIDTH>
2812void smmuv3<BUSWIDTH>::handle_cmd_cfgi_ste_range(
const uint8_t*)
2814 m_ste_cache.clear();
2815 m_ste_cache_lru.clear();
2818template <
unsigned int BUSWIDTH>
2819void smmuv3<BUSWIDTH>::handle_cmd_cfgi_cd(
const uint8_t*)
2822 m_cd_cache_lru.clear();
2825template <
unsigned int BUSWIDTH>
2826void smmuv3<BUSWIDTH>::handle_cmd_cfgi_cd_all(
const uint8_t*)
2829 m_cd_cache_lru.clear();
2832template <
unsigned int BUSWIDTH>
2833void smmuv3<BUSWIDTH>::handle_cmd_cfgi_all(
const uint8_t*)
2835 m_ste_cache.clear();
2836 m_ste_cache_lru.clear();
2838 m_cd_cache_lru.clear();
2841template <
unsigned int BUSWIDTH>
2842void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_all(
const uint8_t*)
2847template <
unsigned int BUSWIDTH>
2848void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_asid(
const uint8_t* cmd)
2850 uint16_t asid = load_le<uint16_t>(cmd + detail_smmuv3::CMD_ASID_OFFSET);
2851 iotlb_inv_asid(asid);
2854template <
unsigned int BUSWIDTH>
2855void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_va(
const uint8_t* cmd)
2857 uint64_t word0 = load_le<uint64_t>(cmd);
2858 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2859 uint16_t asid = (word0 >> detail_smmuv3::CMD_WORD0_ASID_SHIFT) & detail_smmuv3::CMD_WORD0_ASID_MASK;
2860 uint8_t tlbi_tg = (word0 >> detail_smmuv3::CMD_WORD0_TG_SHIFT) & detail_smmuv3::CMD_WORD0_TG_MASK;
2861 uint8_t scale = (word0 >> detail_smmuv3::CMD_WORD0_SCALE_SHIFT) & detail_smmuv3::CMD_WORD0_SCALE_MASK;
2862 uint64_t va = word1 & ~detail_smmuv3::CMD_VA_LOW_MASK;
2865 uint64_t mask = safe_shl1_u64(tg.page_shift + scale) - 1;
2866 iotlb_inv_iova(0, asid, va, tg.key_tg, mask);
2869template <
unsigned int BUSWIDTH>
2870void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nh_vaa(
const uint8_t* cmd)
2872 uint64_t word0 = load_le<uint64_t>(cmd);
2873 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2874 uint8_t tlbi_tg = (word0 >> detail_smmuv3::CMD_WORD0_TG_SHIFT) & detail_smmuv3::CMD_WORD0_TG_MASK;
2875 uint8_t scale = (word0 >> detail_smmuv3::CMD_WORD0_SCALE_SHIFT) & detail_smmuv3::CMD_WORD0_SCALE_MASK;
2876 uint64_t va = word1 & ~detail_smmuv3::CMD_VA_LOW_MASK;
2879 uint64_t mask = safe_shl1_u64(tg.page_shift + scale) - 1;
2880 uint64_t va_masked = va & ~mask;
2881 uint8_t key_tg = tg.key_tg;
2883 for (
auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2884 uint64_t entry_masked = it->first.iova & ~mask;
2885 if (entry_masked == va_masked && it->first.tg == key_tg) {
2886 m_iotlb_lru.remove(it->first);
2887 it = m_iotlb.erase(it);
2894template <
unsigned int BUSWIDTH>
2895void smmuv3<BUSWIDTH>::handle_cmd_atc_inv(
const uint8_t* cmd)
2897 constexpr uint32_t CMD_WORD0_SID_SHIFT = 32;
2898 constexpr uint64_t CMD_ATC_SIZE_MASK = 0x3FULL;
2899 constexpr uint64_t CMD_ATC_ADDR_MASK = ~((1ULL << PAGE_SHIFT_4K) - 1ULL);
2901 uint64_t word0 = load_le<uint64_t>(cmd);
2902 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2903 uint32_t sid =
static_cast<uint32_t
>(word0 >> CMD_WORD0_SID_SHIFT);
2904 uint8_t size = word1 & CMD_ATC_SIZE_MASK;
2905 uint64_t addr_lo = word1 & CMD_ATC_ADDR_MASK;
2907 uint64_t range = safe_shl1_u64(size + PAGE_SHIFT_4K);
2908 uint64_t end = addr_lo + range - 1;
2910 SCP_INFO(()) <<
"SMMUv3: ATC_INV sid=0x" << std::hex << sid <<
" addr=0x" << addr_lo <<
" range=0x" << range;
2912 for (
auto* tbu : tbus) {
2914 tbu->upstream_invalidate(addr_lo, end);
2918template <
unsigned int BUSWIDTH>
2919void smmuv3<BUSWIDTH>::handle_cmd_tlbi_s12_vmall(
const uint8_t* cmd)
2921 uint64_t word0 = load_le<uint64_t>(cmd);
2922 uint16_t vmid = (word0 >> detail_smmuv3::CMD_WORD0_VMID_SHIFT) & detail_smmuv3::CMD_WORD0_VMID_MASK;
2923 iotlb_inv_vmid(vmid);
2926template <
unsigned int BUSWIDTH>
2927void smmuv3<BUSWIDTH>::handle_cmd_tlbi_s2_ipa(
const uint8_t* cmd)
2929 uint64_t word0 = load_le<uint64_t>(cmd);
2930 uint64_t word1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2931 uint16_t vmid = (word0 >> detail_smmuv3::CMD_WORD0_VMID_SHIFT) & detail_smmuv3::CMD_WORD0_VMID_MASK;
2932 uint8_t tlbi_tg = (word0 >> detail_smmuv3::CMD_WORD0_TG_SHIFT) & detail_smmuv3::CMD_WORD0_TG_MASK;
2933 uint8_t scale = (word0 >> detail_smmuv3::CMD_WORD0_SCALE_SHIFT) & detail_smmuv3::CMD_WORD0_SCALE_MASK;
2934 uint64_t ipa = word1 & IPA_ADDR_MASK;
2937 uint64_t mask = safe_shl1_u64(tg.page_shift + scale) - 1;
2938 uint64_t ipa_masked = ipa & ~mask;
2939 uint8_t key_tg = tg.key_tg;
2941 for (
auto it = m_iotlb.begin(); it != m_iotlb.end();) {
2942 uint64_t entry_masked = it->first.iova & ~mask;
2943 if (it->first.vmid == vmid && entry_masked == ipa_masked && it->first.tg == key_tg) {
2944 m_iotlb_lru.remove(it->first);
2945 it = m_iotlb.erase(it);
2952template <
unsigned int BUSWIDTH>
2953void smmuv3<BUSWIDTH>::handle_cmd_tlbi_nsnh_all(
const uint8_t*)
2958template <
unsigned int BUSWIDTH>
2959void smmuv3<BUSWIDTH>::handle_cmd_sync(
const uint8_t* cmd)
2961 uint64_t dw0 = load_le<uint64_t>(cmd);
2962 uint64_t dw1 = load_le<uint64_t>(cmd + detail_smmuv3::CMD_WORD1_OFFSET);
2964 uint32_t cs = (
static_cast<uint32_t
>(dw0) >> detail_smmuv3::CMD_SYNC_CS_SHIFT) & detail_smmuv3::CMD_SYNC_CS_MASK;
2966 if (cs == SYNC_CS_IRQ) {
2967 constexpr uint64_t CMD_SYNC_MSIADDR_MASK = 0x000FFFFFFFFFFFFCULL;
2968 constexpr uint32_t CMD_SYNC_MSIDATA_SHIFT = 32;
2969 uint64_t msiaddr = dw1 & CMD_SYNC_MSIADDR_MASK;
2970 uint32_t msidata =
static_cast<uint32_t
>(dw0 >> CMD_SYNC_MSIDATA_SHIFT);
2972 if (!dma_write(msiaddr, &msidata,
sizeof(msidata))) {
2973 set_gerror(GERROR_BIT_MSI_CMDQ_ABT);
2976 trigger_irq(irq_cmd_sync);
2980template <
unsigned int BUSWIDTH>
2981void smmuv3<BUSWIDTH>::handle_cmd_resume(
const uint8_t* cmd)
2983 uint32_t stag = load_le<uint32_t>(cmd + detail_smmuv3::CMD_STAG_OFFSET);
2984 uint8_t resp = cmd[detail_smmuv3::CMD_RESUME_RESP_OFFSET];
2986 auto it = m_stalled_txns.find(stag);
2987 if (it != m_stalled_txns.end()) {
2989 it->second.aborted =
true;
2991 it->second.resume_event.notify();
2995template <
unsigned int BUSWIDTH>
2996void smmuv3<BUSWIDTH>::handle_cmd_pri_resp(
const uint8_t*)
3000template <
unsigned int BUSWIDTH>
3001void smmuv3<BUSWIDTH>::do_gatos()
3003 GATOS_CTRL[GATOS_CTRL_INPRG] = 1;
3005 uint32_t sid =
static_cast<uint32_t
>(GATOS_SID);
3006 uint64_t iova = (
static_cast<uint64_t
>(GATOS_ADDR_HI) << 32) |
static_cast<uint64_t
>(GATOS_ADDR_LO);
3007 bool wr =
static_cast<uint32_t
>(GATOS_CTRL[GATOS_CTRL_READ_NWRITE]) == 0;
3009 tlm::tlm_generic_payload dummy_txn;
3010 dummy_txn.set_command(wr ? tlm::TLM_WRITE_COMMAND : tlm::TLM_READ_COMMAND);
3011 dummy_txn.set_address(iova);
3013 IOMMUTLBEntry te = smmuv3_translate(dummy_txn, sid, 0);
3015 if (te.perm == IOMMUAccessFlags::NONE) {
3016 GATOS_PAR_LO[GATOS_PAR_F] = 1;
3017 GATOS_PAR_LO[GATOS_PAR_FST] = GATOS_PAR_FAULT_TRANSLATION;
3019 GATOS_PAR_LO =
static_cast<uint32_t
>(te.translated_addr);
3020 GATOS_PAR_HI =
static_cast<uint32_t
>(te.translated_addr >> 32);
3021 GATOS_PAR_LO[GATOS_PAR_F] = 0;
3024 GATOS_CTRL[GATOS_CTRL_RUN] = 0;
3025 GATOS_CTRL[GATOS_CTRL_INPRG] = 0;
Definition target-signal-socket.h:163
Definition reg_model_maker.h:457
Definition reg_model_maker.h:35
Definition smmuv3_extensions.h:35
Definition smmuv3_gen.h:21
Definition smmuv3_memory_attrs.h:19
Definition smmuv3_extensions.h:18
Tool which reads a Lua configuration file and sets parameters.
Definition biflow.cc:10