83class smmu500 :
public sc_core::sc_module,
public smmu500_gen
87 unsigned char* ptr =
nullptr;
98 cci::cci_broker_handle m_broker;
102 tlm_utils::simple_initiator_socket<smmu500, BUSWIDTH> m_register_socket;
114 uint64_t translated_addr;
116 IOMMUAccessFlags perm;
119 cci::cci_param<uint32_t> p_pamax;
120 cci::cci_param<uint16_t> p_num_smr;
121 cci::cci_param<uint16_t> p_num_cb;
122 cci::cci_param<uint16_t> p_num_pages;
123 cci::cci_param<bool> p_ato;
124 cci::cci_param<uint8_t> p_version;
125 cci::cci_param<uint32_t> p_idr2;
126 cci::cci_param<uint8_t> p_num_tbu;
128 tlm_utils::multi_passthrough_target_socket<smmu500, BUSWIDTH> socket;
129 tlm_utils::simple_initiator_socket<smmu500> dma_socket;
131 InitiatorSignalSocket<bool> irq_global;
132 sc_core::sc_vector<InitiatorSignalSocket<bool>> irq_context;
135 std::vector<smmu500_tbu<BUSWIDTH>*> tbus;
138 static uint64_t smmu500_ttbr(uint32_t high, uint32_t low)
140 return (
static_cast<uint64_t
>(high & 0xffffu) << 32) | (
static_cast<uint64_t
>(low) & 0xffffffffu);
143 smmu500(sc_core::sc_module_name name);
146 enum class FaultKind {
154 static constexpr uint32_t CB_FSR_TF_MASK = 1u << 1;
155 static constexpr uint32_t CB_FSR_AFF_MASK = 1u << 2;
156 static constexpr uint32_t CB_FSR_PF_MASK = 1u << 3;
157 static constexpr uint32_t CB_FSR_EF_MASK = 1u << 4;
158 static constexpr uint32_t CB_FSR_ASF_MASK = 1u << 7;
159 static constexpr uint32_t CB_FSR_MULTI_MASK = 1u << 31;
160 static constexpr uint32_t CB_FSR_FAULT_MASK = CB_FSR_TF_MASK | CB_FSR_AFF_MASK | CB_FSR_PF_MASK | CB_FSR_EF_MASK |
162 static constexpr uint32_t GLOBAL_FAULT_MASK = (1u << 0) | (1u << 1) | (1u << 2);
163 static constexpr uint32_t SCR0_CLIENTPD_MASK = 1u << 0;
164 static constexpr uint32_t SCR0_GFRE_MASK = 1u << 1;
165 static constexpr uint32_t SCR0_GFIE_MASK = 1u << 2;
166 static constexpr uint32_t SCR0_USFCFG_MASK = 1u << 10;
167 static constexpr uint32_t SCR0_SMCFCFG_MASK = 1u << 21;
168 static constexpr uint64_t NS_GLOBAL_REG_START = 0x400;
169 static constexpr uint64_t NS_GLOBAL_REG_END = 0x800;
170 static constexpr uint64_t SMR_BASE = 0x800;
171 static constexpr uint64_t S2CR_BASE = 0xc00;
172 static constexpr uint64_t CBAR_BASE = 0x1000;
173 static constexpr uint64_t CBA2R_BASE = 0x1800;
174 static constexpr uint64_t TBU_PWR_STATUS = 0x2204;
175 static constexpr uint64_t SMMU_SCR2_OFFSET = 0x008;
176 static constexpr uint64_t SMMU_GFSR_OFFSET = 0x040;
177 static constexpr uint64_t SMMU_GFSR_SIZE = 0x01c;
178 static constexpr uint64_t SMMU_TLBIVAH_RANGE_OFFSET = 0x070;
179 static constexpr uint64_t SMMU_TLBIVAH_RANGE_SIZE = 0x008;
180 static constexpr uint64_t SMMU_GPAR_RANGE_SIZE = 0x00c;
182 std::array<uint32_t, SMMU_MAX_CB> m_cb_fsr_before_write = {};
183 uint32_t m_sgfsr_before_write = 0;
184 uint32_t m_nsgfsr_before_write = 0;
186 std::array<uint8_t, 0x190> m_secure_gr0_shadow = {};
187 std::array<uint8_t, 0x190> m_non_secure_gr0_shadow = {};
189 std::atomic<uint64_t> m_translation_seq{ 0 };
192 std::recursive_mutex m_translation_lock;
194 std::mutex m_dma_dmi_lock;
196 void invalidate_dma_dmi(sc_dt::uint64 start, sc_dt::uint64 end)
198 std::lock_guard<std::mutex> lock(m_dma_dmi_lock);
199 if (m_dma_dmi.ptr !=
nullptr && m_dma_dmi.end >= start && m_dma_dmi.start <= end) m_dma_dmi =
DmaDmi{};
202 typedef struct PageAttr {
204 unsigned int block : 1;
212 bool privileged =
true;
213 bool instruction =
false;
214 bool non_secure =
true;
215 bool address_translation =
false;
216 bool asynchronous =
false;
217 bool suppress_fault =
false;
223 typedef struct TransReq {
235 FaultKind fault = FaultKind::Translation;
236 bool walk_fault =
false;
237 bool privileged =
true;
238 bool instruction =
false;
239 bool non_secure =
true;
240 bool address_translation =
false;
241 bool asynchronous =
false;
243 bool internal_walk =
false;
244 bool suppress_fault =
false;
245 bool ipafar_valid =
false;
247 unsigned int s1_cb = 0;
251 uint64_t trace_id = 0;
255 static uint32_t extract32(uint32_t val,
int start,
int length) {
return (val >> start) & ((1u << length) - 1); }
257 static uint64_t extract64(uint64_t val,
int start,
int length) {
return (val >> start) & ((1ULL << length) - 1); }
259 static int clz32(uint32_t val) {
return val ? __builtin_clz(val) : 32; }
261 void smmu500_update_ctx_irq(
unsigned int cb)
263 bool fault = (
static_cast<uint32_t
>(SMMU_CB_FSR[cb]) & CB_FSR_FAULT_MASK) != 0;
264 bool ie = SMMU_CB_SCTLR[cb][CB_SCTLR_CFIE];
265 if (irq_context[cb].size() > 0) irq_context[cb]->write(fault && ie);
268 uint32_t smmu500_global_control(
bool non_secure)
270 return non_secure ?
static_cast<uint32_t
>(SMMU_NSCR0) : static_cast<uint32_t>(SMMU_SCR0);
274 void smmu500_update_global_irq()
276 const bool secure_pending = (
static_cast<uint32_t
>(SMMU_SGFSR) & GLOBAL_FAULT_MASK) != 0;
277 const bool non_secure_pending = (
static_cast<uint32_t
>(SMMU_NSGFSR) & GLOBAL_FAULT_MASK) != 0;
278 const bool secure_irq = secure_pending && (smmu500_global_control(
false) & SCR0_GFIE_MASK);
279 const bool non_secure_irq = non_secure_pending && (smmu500_global_control(
true) & SCR0_GFIE_MASK);
280 if (irq_global.size() > 0) irq_global->write(secure_irq || non_secure_irq);
283 void smmu500_record_global_fault(uint32_t fault_mask,
bool non_secure)
286 SMMU_NSGFSR =
static_cast<uint32_t
>(SMMU_NSGFSR) | fault_mask;
288 SMMU_SGFSR =
static_cast<uint32_t
>(SMMU_SGFSR) | fault_mask;
290 smmu500_update_global_irq();
293 void smmu500_report_global_fault(
IOMMUTLBEntry& ret, uint32_t fault_mask,
bool non_secure, uint32_t global_control,
294 bool record_fault =
true)
296 if (record_fault) smmu500_record_global_fault(fault_mask, non_secure);
297 if (global_control & SCR0_GFRE_MASK) {
298 ret.perm = IOMMU_NONE;
304 FaultAttrs smmu500_extract_fault_attrs(tlm::tlm_generic_payload& txn,
bool record_fault)
const
307 attrs.suppress_fault = !record_fault;
308 if (
const auto* ext = txn.get_extension<smmu500_transaction_attrs_extension>()) {
309 attrs.privileged = ext->privileged;
310 attrs.instruction = ext->instruction;
311 attrs.non_secure = ext->non_secure;
312 attrs.address_translation = ext->address_translation;
313 attrs.asynchronous = ext->asynchronous;
314 attrs.mid = ext->mid;
315 attrs.pid = ext->pid;
316 attrs.bid = ext->bid;
318 if (
const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>()) {
319 attrs.non_secure = !secure->value;
324 static bool smmu500_range_within(uint64_t address, uint64_t length, uint64_t start, uint64_t size)
326 return address >= start && length <= size && address - start <= size - length;
329 struct RegisterRange {
335 static bool smmu500_range_within_any(uint64_t address, uint64_t length,
const std::array<RegisterRange, N>& ranges)
337 return std::any_of(ranges.cbegin(), ranges.cend(), [address, length](
const RegisterRange& range) {
338 return smmu500_range_within(address, length, range.offset, range.size);
343 static bool smmu500_offset_in(uint64_t address,
const std::array<uint64_t, N>& offsets)
345 return std::find(offsets.cbegin(), offsets.cend(), address) != offsets.cend();
348 enum class RegisterAccessTarget { Model, Shadow, IdRegister, Denied };
350 struct RegisterAccess {
351 RegisterAccessTarget target = RegisterAccessTarget::Model;
352 uint64_t address = 0;
353 bool non_secure =
false;
356 bool smmu500_is_secure_only_gr0_register(uint64_t address, uint64_t length)
358 const std::array<RegisterRange, 5> secure_only_ranges = { {
359 { SMMU_SCR1.get_offset(),
sizeof(uint32_t) },
360 { SMMU_STLBIALL.get_offset(),
sizeof(uint32_t) },
361 { SMMU_STLBIVALM_LOW.get_offset(),
sizeof(uint64_t) },
362 { SMMU_STLBIVAM_LOW.get_offset(),
sizeof(uint64_t) },
363 { SMMU_STLBIALLM.get_offset(),
sizeof(uint32_t) },
365 return smmu500_range_within_any(address, length, secure_only_ranges);
368 bool smmu500_is_non_secure_alias(uint64_t address, uint64_t length)
370 const std::array<RegisterRange, 7> non_secure_alias_ranges = { {
371 { SMMU_SCR0.get_offset(),
sizeof(uint32_t) },
372 { SMMU_SCR2_OFFSET,
sizeof(uint32_t) },
373 { SMMU_SACR.get_offset(),
sizeof(uint32_t) },
374 { SMMU_GFSR_OFFSET, SMMU_GFSR_SIZE },
375 { SMMU_TLBIVAH_RANGE_OFFSET, SMMU_TLBIVAH_RANGE_SIZE },
376 { SMMU_GATS1UR.get_offset(), SMMU_GATS12PW_H.get_offset() - SMMU_GATS1UR.get_offset() +
sizeof(uint32_t) },
377 { SMMU_GPAR.get_offset(), SMMU_GPAR_RANGE_SIZE },
379 return smmu500_range_within_any(address, length, non_secure_alias_ranges);
382 bool smmu500_is_banked_gr0_register(uint64_t address, uint64_t length)
384 const std::array<RegisterRange, 10> banked_ranges = { {
385 { SMMU_SCR0.get_offset(),
sizeof(uint32_t) },
386 { SMMU_SCR2_OFFSET,
sizeof(uint32_t) },
387 { SMMU_SACR.get_offset(),
sizeof(uint32_t) },
388 { SMMU_GFSR_OFFSET, SMMU_GFSR_SIZE },
389 { SMMU_TLBIVMID.get_offset(),
390 SMMU_TLBIVAH_LOW.get_offset() - SMMU_TLBIVMID.get_offset() +
sizeof(uint32_t) },
391 { SMMU_TLBIVALH64_LOW.get_offset(),
sizeof(uint64_t) },
392 { SMMU_TLBIVMIDS1.get_offset(),
sizeof(uint32_t) },
393 { SMMU_TLBIVAH64_LOW.get_offset(),
sizeof(uint64_t) },
394 { SMMU_GATS1UR.get_offset(), SMMU_GATS12PW_H.get_offset() - SMMU_GATS1UR.get_offset() +
sizeof(uint32_t) },
395 { SMMU_GPAR.get_offset(), SMMU_GPAR_RANGE_SIZE },
397 return smmu500_range_within_any(address, length, banked_ranges);
400 bool smmu500_is_modelled_banked_gr0_register(uint64_t address, uint64_t length)
402 const std::array<RegisterRange, 10> modelled_banked_ranges = { {
403 { SMMU_SCR0.get_offset(),
sizeof(uint32_t) },
404 { SMMU_SACR.get_offset(),
sizeof(uint32_t) },
405 { SMMU_SGFSR.get_offset(),
sizeof(uint32_t) },
406 { SMMU_TLBIVMID.get_offset(), SMMU_TLBIALLH.get_offset() - SMMU_TLBIVMID.get_offset() +
sizeof(uint32_t) },
407 { SMMU_TLBIVAH_LOW.get_offset(),
sizeof(uint32_t) },
408 { SMMU_TLBIVALH64_LOW.get_offset(),
sizeof(uint64_t) },
409 { SMMU_TLBIVMIDS1.get_offset(),
sizeof(uint32_t) },
410 { SMMU_TLBIVAH64_LOW.get_offset(),
sizeof(uint64_t) },
411 { SMMU_GATS1UR.get_offset(), SMMU_GATS12PW_H.get_offset() - SMMU_GATS1UR.get_offset() +
sizeof(uint32_t) },
412 { SMMU_GPAR.get_offset(),
sizeof(uint64_t) },
414 return smmu500_range_within_any(address, length, modelled_banked_ranges);
417 bool smmu500_is_shadowed_gr0_register(uint64_t address, uint64_t length,
bool non_secure)
419 if (!smmu500_is_banked_gr0_register(address, length))
return false;
420 if (non_secure)
return address != SMMU_SCR0.get_offset() && address != SMMU_SGFSR.get_offset();
421 return !smmu500_is_modelled_banked_gr0_register(address, length);
424 RegisterAccess smmu500_resolve_register_access(
const tlm::tlm_generic_payload& txn)
426 const uint64_t address = txn.get_address();
427 const uint64_t length = std::max<uint64_t>(txn.get_data_length(), 1);
428 RegisterAccess access{ RegisterAccessTarget::Model, address,
false };
429 const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
430 if (!secure)
return access;
432 const bool non_secure = !secure->value;
433 const uint64_t cb_base =
static_cast<uint64_t
>(p_num_pages) * SMMU_PAGESIZE;
434 if (non_secure && (
static_cast<uint32_t
>(SMMU_SCR1) & (1u << 24)) && address < cb_base) {
435 access.target = RegisterAccessTarget::Denied;
438 const bool alias = smmu500_range_within(address, length, NS_GLOBAL_REG_START,
439 NS_GLOBAL_REG_END - NS_GLOBAL_REG_START);
440 const uint64_t primary = alias ? address - NS_GLOBAL_REG_START : address;
443 if (!smmu500_is_non_secure_alias(primary, length)) {
444 access.target = RegisterAccessTarget::Denied;
445 }
else if (smmu500_is_shadowed_gr0_register(primary, length,
true)) {
446 access = { RegisterAccessTarget::Shadow, primary,
true };
447 }
else if (primary == SMMU_SCR0.get_offset() || primary == SMMU_SGFSR.get_offset()) {
448 access.address = address;
450 access.address = primary;
456 if (smmu500_is_shadowed_gr0_register(address, length,
false)) {
457 access = { RegisterAccessTarget::Shadow, address,
false };
462 if (smmu500_range_within(address, length, 0, NS_GLOBAL_REG_START)) {
463 if (smmu500_is_secure_only_gr0_register(address, length)) {
464 access.target = RegisterAccessTarget::Denied;
465 }
else if (address == SMMU_SIDR0.get_offset() || address == SMMU_SIDR1.get_offset()) {
466 access.target = RegisterAccessTarget::IdRegister;
467 }
else if (smmu500_is_shadowed_gr0_register(address, length,
true)) {
468 access = { RegisterAccessTarget::Shadow, address,
true };
469 }
else if (address == SMMU_SCR0.get_offset() || address == SMMU_SGFSR.get_offset()) {
470 access.address = address + NS_GLOBAL_REG_START;
475 const uint64_t num_ns_smr = std::min<uint64_t>(
static_cast<uint32_t
>(SCR1_NSNUMSMRGO), p_num_smr);
476 const uint64_t num_ns_cb = std::min<uint64_t>(
static_cast<uint32_t
>(SCR1_NSNUMCBO), p_num_cb);
477 const bool permitted = smmu500_range_within(address, length, SMR_BASE, num_ns_smr *
sizeof(uint32_t)) ||
478 smmu500_range_within(address, length, S2CR_BASE, num_ns_smr *
sizeof(uint32_t)) ||
479 smmu500_range_within(address, length, CBAR_BASE, num_ns_cb *
sizeof(uint32_t)) ||
480 smmu500_range_within(address, length, CBA2R_BASE, num_ns_cb *
sizeof(uint32_t)) ||
481 smmu500_range_within(address, length, TBU_PWR_STATUS,
sizeof(uint32_t)) ||
482 smmu500_range_within(address, length, cb_base, num_ns_cb * SMMU_PAGESIZE);
483 const bool context_access = address >= cb_base;
484 const uint64_t cb = context_access ? (address - cb_base) / SMMU_PAGESIZE : 0;
485 const bool restricted_stage2_cb = context_access && cb < p_num_cb &&
486 (
static_cast<uint32_t
>(SMMU_SCR1) & (1u << 24)) &&
487 static_cast<uint32_t
>(SMMU_CBAR[cb][CBAR_TYPE]) == 0;
488 if (!permitted || restricted_stage2_cb) access.target = RegisterAccessTarget::Denied;
492 bool smmu500_access_gr0_shadow(
const RegisterAccess& access, tlm::tlm_generic_payload& txn)
494 auto* data = txn.get_data_ptr();
495 const uint64_t length = txn.get_data_length();
496 if (!data || !length || access.address + length > m_secure_gr0_shadow.size()) {
497 txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE);
500 auto& state = access.non_secure ? m_non_secure_gr0_shadow : m_secure_gr0_shadow;
502 std::memcpy(data, state.data() + access.address, length);
503 }
else if (txn.is_write()) {
504 std::memcpy(state.data() + access.address, data, length);
505 if (access.non_secure) smmu500_handle_non_secure_gr0_write(access.address);
507 txn.set_response_status(tlm::TLM_COMMAND_ERROR_RESPONSE);
510 txn.set_dmi_allowed(
false);
511 txn.set_response_status(tlm::TLM_OK_RESPONSE);
515 bool smmu500_access_non_secure_id_register(
const RegisterAccess& access, tlm::tlm_generic_payload& txn)
517 if (txn.get_data_length() !=
sizeof(uint32_t) || !txn.get_data_ptr())
return false;
518 if (txn.is_write()) {
519 txn.set_dmi_allowed(
false);
520 txn.set_response_status(tlm::TLM_OK_RESPONSE);
523 if (!txn.is_read())
return false;
524 uint32_t value = access.address == SMMU_SIDR0.get_offset() ?
static_cast<uint32_t
>(SMMU_SIDR0)
525 : static_cast<uint32_t>(SMMU_SIDR1);
526 const uint32_t count = access.address == SMMU_SIDR0.get_offset() ?
static_cast<uint32_t
>(SCR1_NSNUMSMRGO)
527 :
static_cast<uint32_t
>(SCR1_NSNUMCBO);
528 value = (value & ~0xffu) | count;
529 std::memcpy(txn.get_data_ptr(), &value,
sizeof(value));
530 txn.set_dmi_allowed(
false);
531 txn.set_response_status(tlm::TLM_OK_RESPONSE);
535 uint32_t smmu500_non_secure_gr0_read32(uint64_t address)
const
538 std::memcpy(&value, m_non_secure_gr0_shadow.data() + address,
sizeof(value));
542 void smmu500_non_secure_gr0_write32(uint64_t address, uint32_t value)
544 std::memcpy(m_non_secure_gr0_shadow.data() + address, &value,
sizeof(value));
547 void smmu500_handle_non_secure_gr0_write(uint64_t address)
550 uint64_t high_offset;
555 const std::array<GatCommand, 8> gat_commands = { {
556 { SMMU_GATS1UR_H.get_offset(),
false,
false,
true },
557 { SMMU_GATS1UW_H.get_offset(),
true,
false,
true },
558 { SMMU_GATS1PR_H.get_offset(),
false,
false,
false },
559 { SMMU_GATS1PW_H.get_offset(),
true,
false,
false },
560 { SMMU_GATS12UR_H.get_offset(),
false,
true,
true },
561 { SMMU_GATS12UW_H.get_offset(),
true,
true,
true },
562 { SMMU_GATS12PR_H.get_offset(),
false,
true,
false },
563 { SMMU_GATS12PW_H.get_offset(),
true,
true,
false },
565 const std::array<uint64_t, 2> vmid_invalidate_offsets = { {
566 SMMU_TLBIVMID.get_offset(),
567 SMMU_TLBIVMIDS1.get_offset(),
569 const std::array<uint64_t, 7> all_invalidate_offsets = { {
570 SMMU_TLBIALLNSNH.get_offset(),
571 SMMU_TLBIALLH.get_offset(),
572 SMMU_TLBIVAH_LOW.get_offset(),
573 SMMU_TLBIVALH64_LOW.get_offset(),
574 SMMU_TLBIVALH64_HIGH.get_offset(),
575 SMMU_TLBIVAH64_LOW.get_offset(),
576 SMMU_TLBIVAH64_HIGH.get_offset(),
578 const auto gat = [
this, address](
bool write,
bool stage2,
bool unprivileged) {
579 const uint64_t value = (
static_cast<uint64_t
>(smmu500_non_secure_gr0_read32(address)) << 32) |
580 smmu500_non_secure_gr0_read32(address -
sizeof(uint32_t));
581 smmu500_gat(value, write, stage2, unprivileged,
true);
584 const auto gat_command = std::find_if(
585 gat_commands.cbegin(), gat_commands.cend(),
586 [address](
const GatCommand& command) { return command.high_offset == address; });
587 if (gat_command != gat_commands.cend()) {
588 gat(gat_command->write, gat_command->stage2, gat_command->unprivileged);
591 if (smmu500_offset_in(address, vmid_invalidate_offsets)) {
592 smmu500_invalidate_vmid(smmu500_non_secure_gr0_read32(address) & 0xffu);
593 }
else if (smmu500_offset_in(address, all_invalidate_offsets)) {
594 smmu500_invalidate_all();
598 std::fill(m_non_secure_gr0_shadow.begin() + SMMU_TLBIVMID.get_offset(),
599 m_non_secure_gr0_shadow.begin() + SMMU_TLBIVAH64_HIGH.get_offset() +
sizeof(uint32_t), 0);
602 void smmu500_register_b_transport(
int, tlm::tlm_generic_payload& txn, sc_core::sc_time& delay)
604 const RegisterAccess access = smmu500_resolve_register_access(txn);
605 if (access.target == RegisterAccessTarget::Denied) {
606 const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
607 SCP_WARN(()) << std::hex <<
"SMMU denied "
608 << (txn.is_read() ?
"read"
609 : txn.is_write() ?
"write"
611 <<
" at 0x" << txn.get_address() <<
" length 0x" << txn.get_data_length() <<
" from "
612 << (secure && !secure->value ?
"non-secure" :
"secure") <<
" context";
613 txn.set_dmi_allowed(
false);
614 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
617 const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
618 if (secure && secure->value && txn.get_address() >= NS_GLOBAL_REG_START &&
619 txn.get_address() < NS_GLOBAL_REG_END) {
620 SCP_DEBUG(()) << std::hex <<
"SMMU secure "
621 << (txn.is_read() ?
"read"
622 : txn.is_write() ?
"write"
624 <<
" of non-secure alias at 0x" << txn.get_address() <<
" length 0x" << txn.get_data_length();
626 if (access.target == RegisterAccessTarget::Shadow) {
627 smmu500_access_gr0_shadow(access, txn);
630 if (access.target == RegisterAccessTarget::IdRegister) {
631 if (!smmu500_access_non_secure_id_register(access, txn)) {
632 txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE);
636 const uint64_t address = txn.get_address();
637 txn.set_address(access.address);
638 m_register_socket->b_transport(txn, delay);
639 txn.set_address(address);
640 txn.set_dmi_allowed(
false);
643 unsigned int smmu500_register_transport_dbg(
int, tlm::tlm_generic_payload& txn)
645 const RegisterAccess access = smmu500_resolve_register_access(txn);
646 if (access.target == RegisterAccessTarget::Denied)
return 0;
647 if (access.target == RegisterAccessTarget::Shadow) {
648 return smmu500_access_gr0_shadow(access, txn) ? txn.get_data_length() : 0;
650 if (access.target == RegisterAccessTarget::IdRegister) {
651 return smmu500_access_non_secure_id_register(access, txn) ? txn.get_data_length() : 0;
653 const uint64_t address = txn.get_address();
654 txn.set_address(access.address);
655 const unsigned int transferred = m_register_socket->transport_dbg(txn);
656 txn.set_address(address);
660 bool smmu500_register_get_direct_mem_ptr(
int, tlm::tlm_generic_payload& txn, tlm::tlm_dmi& dmi_data)
665 txn.set_dmi_allowed(
false);
669 void smmu500_program_id_registers()
671 const unsigned int num_pages_log2 = 31 - clz32(p_num_pages);
672 SIDR0_ATOSNS =
static_cast<uint32_t
>(p_ato);
673 SIDR0_NUMSMRG = std::min<uint32_t>(p_num_smr, SMMU_MAX_SMR);
674 SIDR1_NUMCB = std::min<uint32_t>(p_num_cb, SMMU_MAX_CB);
675 SIDR1_NUMPAGENDXB = num_pages_log2 - 1;
676 SMMU_SIDR2 =
static_cast<uint32_t
>(p_idr2);
677 SMMU_SIDR7 =
static_cast<uint32_t
>(p_version);
680 void smmu500_invalidate_cb(
unsigned int cb)
682 if (cb >= std::min<unsigned int>(p_num_cb, SMMU_MAX_CB))
return;
683 for (
auto tbu : tbus) tbu->invalidate(cb);
686 void smmu500_invalidate_all()
688 for (
auto tbu : tbus) tbu->reset_dmi();
691 void smmu500_invalidate_vmid(uint32_t vmid)
693 const unsigned int nr_cb = std::min<unsigned int>(p_num_cb, SMMU_MAX_CB);
694 for (
unsigned int cb = 0; cb < nr_cb; ++cb) {
695 if (
static_cast<uint32_t
>(SMMU_CBAR[cb][CBAR_VMID]) == (vmid & 0xffu)) {
696 smmu500_invalidate_cb(cb);
701 void smmu500_clear_global_tlbi_regs()
705 SMMU_TLBIALLNSNH = 0;
707 SMMU_TLBIVAH_LOW = 0;
708 SMMU_STLBIVALM_LOW = 0;
709 SMMU_STLBIVALM_HIGH = 0;
710 SMMU_STLBIVAM_LOW = 0;
711 SMMU_STLBIVAM_HIGH = 0;
712 SMMU_TLBIVALH64_LOW = 0;
713 SMMU_TLBIVALH64_HIGH = 0;
716 SMMU_TLBIVAH64_LOW = 0;
717 SMMU_TLBIVAH64_HIGH = 0;
720 void smmu500_clear_cb_tlbi_regs(
unsigned int cb)
722 if (cb >= SMMU_MAX_CB)
return;
723 SMMU_CB_TLBIVA_LOW[cb] = 0;
724 SMMU_CB_TLBIVA_HIGH[cb] = 0;
725 SMMU_CB_TLBIVAA_LOW[cb] = 0;
726 SMMU_CB_TLBIVAA_HIGH[cb] = 0;
727 SMMU_CB_TLBIASID[cb] = 0;
728 SMMU_CB_TLBIALL[cb] = 0;
729 SMMU_CB_TLBIVAL_LOW[cb] = 0;
730 SMMU_CB_TLBIVAL_HIGH[cb] = 0;
731 SMMU_CB_TLBIVAAL_LOW[cb] = 0;
732 SMMU_CB_TLBIVAAL_HIGH[cb] = 0;
733 SMMU_CB_TLBIIPAS2_LOW[cb] = 0;
734 SMMU_CB_TLBIIPAS2_HIGH[cb] = 0;
735 SMMU_CB_TLBIIPAS2L_LOW[cb] = 0;
736 SMMU_CB_TLBIIPAS2L_HIGH[cb] = 0;
739 static uint32_t smmu500_fault_status(FaultKind kind)
742 case FaultKind::Translation:
743 return CB_FSR_TF_MASK;
744 case FaultKind::AccessFlag:
745 return CB_FSR_AFF_MASK;
746 case FaultKind::Permission:
747 return CB_FSR_PF_MASK;
748 case FaultKind::External:
749 return CB_FSR_EF_MASK;
750 case FaultKind::AddressSize:
751 return CB_FSR_ASF_MASK;
753 return CB_FSR_TF_MASK;
756 static const char* smmu500_fault_name(FaultKind kind)
759 case FaultKind::Translation:
760 return "translation";
761 case FaultKind::AccessFlag:
762 return "access flag";
763 case FaultKind::Permission:
765 case FaultKind::External:
767 case FaultKind::AddressSize:
768 return "address size";
773 void smmu500_fault(
unsigned int cb, TransReq* req,
int level)
775 const uint32_t old_fsr =
static_cast<uint32_t
>(SMMU_CB_FSR[cb]);
776 const bool multiple = (old_fsr & CB_FSR_FAULT_MASK) != 0;
777 uint32_t new_fsr = old_fsr | smmu500_fault_status(req->fault);
778 if (multiple) new_fsr |= CB_FSR_MULTI_MASK;
779 SMMU_CB_FSR[cb] = new_fsr;
782 SCP_WARN(()) <<
"SMMU context fault: CB " << std::dec << cb <<
" SID 0x" << std::hex << req->sid <<
" VA 0x"
783 << req->va <<
" stage " << std::dec << req->stage <<
" level " << level <<
" reason "
784 << smmu500_fault_name(req->fault) << (req->walk_fault ?
" during page-table walk" :
"");
785 const uint64_t ipafar = req->ipafar_valid ? req->ipafar : req->va;
786 SMMU_CB_IPAFAR_LOW[cb] =
static_cast<uint32_t
>(ipafar);
787 SMMU_CB_IPAFAR_HIGH[cb] =
static_cast<uint32_t
>(ipafar >> 32);
788 if (req->stage == 2) {
789 SMMU_CB_FAR_LOW[cb] =
static_cast<uint32_t
>(req->va);
790 SMMU_CB_FAR_HIGH[cb] =
static_cast<uint32_t
>(req->va >> 32);
793 uint32_t syn =
static_cast<uint32_t
>(level) & 0x3;
794 syn |= (req->access == IOMMU_WO) ? (1u << 4) : 0;
795 syn |= req->privileged ? (1u << 5) : 0;
796 syn |= req->instruction ? (1u << 6) : 0;
797 syn |= req->non_secure ? (1u << 8) : 0;
798 syn |= req->address_translation ? (1u << 9) : 0;
799 syn |= req->walk_fault ? (1u << 10) : 0;
800 syn |= req->asynchronous ? (1u << 11) : 0;
801 if (req->stage == 1 || req->nested) {
802 const unsigned int s1_cb = req->nested ? req->s1_cb : cb;
803 syn |= (s1_cb & 0x7Fu) << 16;
805 SMMU_CB_FSYNR0[cb] = syn;
806 SMMU_CB_FSYNR1[cb] = req->mid | (
static_cast<uint32_t
>(req->pid) << 8) |
807 (
static_cast<uint32_t
>(req->bid & 0x7u) << 13);
809 smmu500_update_ctx_irq(cb);
812 bool check_s2_startlevel(
bool is_aa64,
unsigned int pamax_val,
int level,
int inputsize,
int stride)
814 if (level < 0)
return false;
818 if (level == 0 || (level == 1 && pamax_val <= 42))
return false;
821 if (level == 0 || (level == 1 && pamax_val <= 40))
return false;
824 if (level == 0 && pamax_val <= 42)
return false;
830 const int grainsize = stride + 3;
832 if (level == 0)
return false;
833 int startsizecheck = inputsize - ((3 - level) * stride + grainsize);
834 if (startsizecheck < 1 || startsizecheck > stride + 4)
return false;
839 bool check_out_addr(uint64_t addr,
unsigned int outputsize)
841 if (outputsize != 48 && extract64(addr, outputsize, 48 - outputsize))
return false;
845 void dump_trans_req(TransReq
const& tr)
847 SCP_DEBUG(()) <<
"Translation Req\n"
848 << std::hex <<
"VA: 0x" << tr.va <<
"\n"
849 <<
"TCR[0]: 0x" << tr.tcr[0] <<
"\n"
850 <<
"TCR[1]: 0x" << tr.tcr[1] <<
"\n"
851 <<
"TCR[2]: 0x" << tr.tcr[2] <<
"\n"
852 <<
"ACCESS: 0x" << tr.access <<
"\n"
853 <<
"Stage: " << std::dec << tr.stage <<
"\n"
854 <<
"S2_enabled: " << (tr.s2_enabled ?
"true" :
"false") <<
"\n"
855 <<
"S2_CB: " << tr.s2_cb <<
"\n"
856 << std::hex <<
"PA: 0x" << tr.pa <<
"\n"
857 <<
"Prot: 0x" << tr.prot <<
"\n"
858 <<
"Page size: 0x" << tr.page_size <<
"\n"
859 <<
"Error: " << (tr.err ?
"true" :
"false") <<
"\n";
862 void dump_cb_state(
unsigned int cb)
864 SCP_DEBUG(()) <<
"CB" << cb <<
":\n"
865 << std::hex <<
"CB_SCTLR = 0x" <<
static_cast<uint32_t
>(SMMU_CB_SCTLR[cb]) <<
"\n"
866 <<
"CB_TCR2 = 0x" <<
static_cast<uint32_t
>(SMMU_CB_TCR2[cb]) <<
"\n"
867 <<
"CB_TTBR0_LOW = 0x" <<
static_cast<uint32_t
>(SMMU_CB_TTBR0_LOW[cb]) <<
"\n"
868 <<
"CB_TTBR0_HIGH = 0x" <<
static_cast<uint32_t
>(SMMU_CB_TTBR0_HIGH[cb]) <<
"\n"
869 <<
"CB_TTBR1_LOW = 0x" <<
static_cast<uint32_t
>(SMMU_CB_TTBR1_LOW[cb]) <<
"\n"
870 <<
"CB_TTBR1_HIGH = 0x" <<
static_cast<uint32_t
>(SMMU_CB_TTBR1_HIGH[cb]) <<
"\n"
871 <<
"CB_TCR_LPAE = 0x" <<
static_cast<uint32_t
>(SMMU_CB_TCR_LPAE[cb]) <<
"\n"
872 <<
"CB_FSR = 0x" <<
static_cast<uint32_t
>(SMMU_CB_FSR[cb]) <<
"\n"
873 <<
"CB_FAR_LOW = 0x" <<
static_cast<uint32_t
>(SMMU_CB_FAR_LOW[cb]) <<
"\n"
874 <<
"CB_FAR_HIGH = 0x" <<
static_cast<uint32_t
>(SMMU_CB_FAR_HIGH[cb]) <<
"\n"
875 <<
"CB_IPAFAR_LOW = 0x" <<
static_cast<uint32_t
>(SMMU_CB_IPAFAR_LOW[cb]) <<
"\n"
876 <<
"CB_IPAFAR_HIGH = 0x" <<
static_cast<uint32_t
>(SMMU_CB_IPAFAR_HIGH[cb]) <<
"\n";
881 SCP_DEBUG(()) <<
"smmu regs:" << std::hex <<
"SMMU_SCR0 = 0x" <<
static_cast<uint32_t
>(SMMU_SCR0) <<
"\n"
882 <<
"SMMU_SCR1 = 0x" <<
static_cast<uint32_t
>(SMMU_SCR1) <<
"\n"
883 <<
"SMMU_SACR = 0x" <<
static_cast<uint32_t
>(SMMU_SACR) <<
"\n"
884 <<
"SMMU_SIDR0 = 0x" <<
static_cast<uint32_t
>(SMMU_SIDR0) <<
"\n"
885 <<
"SMMU_SIDR1 = 0x" <<
static_cast<uint32_t
>(SMMU_SIDR1) <<
"\n"
886 <<
"SMMU_SIDR2 = 0x" <<
static_cast<uint32_t
>(SMMU_SIDR2) <<
"\n"
887 <<
"SMMU_SIDR7 = 0x" <<
static_cast<uint32_t
>(SMMU_SIDR7) <<
"\n"
888 <<
"SMMU_NSCR0 = 0x" <<
static_cast<uint32_t
>(SMMU_NSCR0) <<
"\n";
891 void smmu500_ptw64(
unsigned int cb, TransReq* req)
893 const unsigned int outsize_map[] = { 32, 36, 40, 42, 44, 48, 48, 48 };
897 unsigned int inputsize;
898 unsigned int outputsize;
899 unsigned int grainsize = -1;
902 unsigned int firstblocklevel = 0;
905 unsigned int baselowerbound;
906 unsigned int stage = req->stage;
907 bool blocktranslate =
false;
909 uint32_t tableattrs = 0;
916 uint64_t last_descaddr = 0;
917 uint64_t last_desc = 0;
919 bool have_descriptor =
false;
923 if (SMMU_CB_SCTLR[cb][CB_SCTLR_M] == 0) {
925 req->prot = IOMMU_RW;
929 ttbr = req->ttbr[stage][0];
930 tg = extract32(req->tcr[stage], 14, 2);
932 ps = extract64(req->tcr[stage], 32, 3);
934 ps = extract64(req->tcr[stage], 16, 3);
936 t0sz = extract32(req->tcr[stage], 0, 6);
940 if (req->stage == 1) {
941 assert(SMMU_CBA2R[cb][CBA2R_VA64] || extract32(req->tcr[1], 31, 1));
943 if ((req->va & (1ULL << 63)) == 0) {
947 const unsigned int tg1map[] = { 3, 2, 0, 1 };
948 tg = tg1map[extract32(req->tcr[stage], 30, 2)];
949 t1sz = extract32(req->tcr[stage], 16, 6);
950 ttbr = req->ttbr[stage][1];
953 epd = extract32(req->tcr[1], ttbr1 ? 23 : 7, 1);
956 if (epd)
goto do_fault;
958 inputsize = 64 - tsz;
959 if (req->stage == 1) {
960 if (inputsize < 25 || inputsize > 48) {
961 req->fault = FaultKind::AddressSize;
964 const uint64_t upper = extract64(req->va, inputsize, 64 - inputsize);
965 const uint64_t expected = ttbr1 ? ((1ULL << (64 - inputsize)) - 1) : 0;
966 if (upper != expected) {
967 req->fault = FaultKind::AddressSize;
970 }
else if (inputsize < 64 && (req->va >> inputsize) != 0) {
974 req->fault = FaultKind::AddressSize;
977 walk_va = (inputsize >= 64) ? req->va : (req->va & ((1ULL << inputsize) - 1));
995 SCP_ERR(()) <<
"Wrong pagesize";
999 outputsize = outsize_map[ps];
1000 if (outputsize > p_pamax) outputsize = p_pamax;
1002 stride = grainsize - 3;
1003 if (req->stage == 1) {
1004 if (grainsize < 16 && (inputsize > (grainsize + 3 * stride)))
1006 else if (inputsize > (grainsize + 2 * stride))
1008 else if (inputsize > (grainsize + stride))
1011 if (inputsize < 25 || inputsize > 48) {
1012 req->fault = FaultKind::AddressSize;
1016 unsigned int startlevel = extract32(req->tcr[stage], 6, 2);
1017 level = 3 - startlevel;
1018 if (grainsize == 12) level = 2 - startlevel;
1019 if (!check_s2_startlevel(
true, outputsize, level, inputsize, stride))
goto do_fault;
1022 if (req->stage == 2) {
1023 ttbr &= ~((1ULL << grainsize) - 1);
1024 const unsigned int level_coverage = grainsize + (4 - level) * stride;
1025 if (inputsize > level_coverage) {
1026 const unsigned int concat_bits = inputsize - level_coverage;
1027 const uint64_t concat_index = extract64(walk_va, level_coverage, concat_bits);
1028 ttbr += concat_index << grainsize;
1031 baselowerbound = 3 + inputsize - ((3 - level) * stride + grainsize);
1032 ttbr &= ~((1ULL << baselowerbound) - 1);
1037 if (!check_out_addr(ttbr, p_pamax)) {
1038 req->fault = FaultKind::AddressSize;
1042 descmask = (1ULL << grainsize) - 1;
1044 unsigned int addrselectbottom = (3 - level) * stride + grainsize;
1045 uint64_t index = (walk_va >> (addrselectbottom - 3)) & descmask;
1047 uint64_t descaddr = ttbr | index;
1049 if (req->stage == 1 && req->s2_enabled) {
1050 TransReq s2req = *req;
1052 s2req.internal_walk =
true;
1053 s2req.va = descaddr;
1054 s2req.access = IOMMU_RO;
1055 smmu500_ptw64(s2req.s2_cb, &s2req);
1057 req->ipafar = descaddr;
1058 req->ipafar_valid =
true;
1059 req->fault = s2req.fault;
1060 req->walk_fault =
true;
1063 descaddr = s2req.pa;
1068 std::lock_guard<std::mutex> lock(m_dma_dmi_lock);
1069 dma_dmi = m_dma_dmi;
1071 const bool descriptor_cached = dma_dmi.ptr !=
nullptr && dma_dmi.secure == !req->non_secure &&
1072 descaddr >= dma_dmi.start && descaddr <= dma_dmi.end &&
1073 dma_dmi.end - descaddr >=
sizeof(desc) - 1;
1074 if (descriptor_cached) {
1075 std::memcpy(&desc, dma_dmi.ptr + (descaddr - dma_dmi.start),
sizeof(desc));
1076 }
else if (!m_dma_on_sysc.is_on_sysc()) {
1080 tlm::tlm_response_status dma_status = tlm::TLM_OK_RESPONSE;
1081 if (!descriptor_cached) {
1082 tlm::tlm_generic_payload txn;
1083 txn.set_command(tlm::TLM_READ_COMMAND);
1084 txn.set_address(descaddr);
1085 txn.set_data_ptr(
reinterpret_cast<unsigned char*
>(&desc));
1086 txn.set_data_length(
sizeof(desc));
1087 txn.set_streaming_width(
sizeof(desc));
1088 txn.set_byte_enable_length(0);
1089 txn.set_dmi_allowed(
false);
1090 txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
1092 QemuMemTxSecureTlmExtension secure_ext;
1093 secure_ext.value = !req->non_secure;
1094 txn.set_extension(&secure_ext);
1097 sc_core::sc_time now = sc_core::SC_ZERO_TIME;
1098 dma_socket->b_transport(txn, now);
1099 dma_status = txn.get_response_status();
1100 if (dma_status == tlm::TLM_OK_RESPONSE && txn.is_dmi_allowed()) {
1102 if (dma_socket->get_direct_mem_ptr(txn, dmi) && dmi.is_read_allowed()) {
1103 const DmaDmi new_dmi{ dmi.get_dmi_ptr(), dmi.get_start_address(), dmi.get_end_address(),
1105 std::lock_guard<std::mutex> lock(m_dma_dmi_lock);
1106 m_dma_dmi = new_dmi;
1109 txn.clear_extension(&secure_ext);
1112 if (dma_status != tlm::TLM_OK_RESPONSE) {
1113 SCP_INFO(()) << std::hex <<
"Bad DMA response CB " << cb <<
" SID 0x" << req->sid <<
" VA 0x" << req->va
1114 <<
" descaddr 0x" << descaddr <<
" TTBR 0x" << ttbr <<
" status " << dma_status;
1115 req->fault = FaultKind::External;
1116 req->walk_fault =
true;
1120 last_descaddr = descaddr;
1122 have_descriptor =
true;
1124 if (req->trace_id != 0) {
1125 SCP_INFO(()) << std::hex <<
"SMMU PTW id " << req->trace_id <<
" SID 0x" << req->sid <<
" CB" << cb
1126 <<
" VA 0x" << req->va <<
" L" << level <<
" table 0x" << ttbr <<
" index 0x" << index
1127 <<
" descaddr 0x" << descaddr <<
" desc 0x" << desc;
1130 unsigned int type = desc & 3;
1131 ttbr = extract64(desc, 0, 48);
1134 if (!(type & 2) && level == 3) {
1135 SCP_INFO(()) <<
"bad level 3 desc";
1136 req->walk_fault =
true;
1140 if (level == 3)
break;
1145 SCP_INFO(()) <<
"bad desc id " << req->trace_id <<
" SID 0x" << std::hex << req->sid <<
" CB " << cb;
1146 req->walk_fault =
true;
1150 blocktranslate =
true;
1151 if (level <
static_cast<int>(firstblocklevel)) {
1152 req->walk_fault =
true;
1157 tableattrs |= extract64(desc, 59, 5);
1158 if (!check_out_addr(ttbr, p_pamax)) {
1159 req->fault = FaultKind::AddressSize;
1165 }
while (!blocktranslate);
1167 if (!check_out_addr(ttbr, outputsize)) {
1168 req->fault = FaultKind::AddressSize;
1173 unsigned long page_size;
1174 page_size = (1ULL << ((stride * (4 - level)) + 3));
1175 ttbr |= (walk_va & (page_size - 1));
1176 req->page_size = ((stride * (4 - level)) + 3);
1179 s2attrs = attrs = extract64(desc, 2, 10) | (extract64(desc, 52, 12) << 10);
1180 if (req->stage == 1) {
1181 attrs |= extract32(tableattrs, 0, 2) << 11;
1182 attrs |= extract32(tableattrs, 3, 1) << 5;
1183 if (0 && extract32(tableattrs, 2, 1)) {
1188 req->prot = IOMMU_RW;
1189 if ((attrs & (1 << 8)) == 0) {
1190 SCP_INFO(()) <<
"access forbidden " << std::hex << attrs;
1191 req->fault = FaultKind::AccessFlag;
1195 if (req->stage == 1) {
1196 const bool unprivileged_access = !req->privileged;
1197 const bool user_access = (attrs & (1 << 4)) != 0;
1198 if (unprivileged_access && !user_access) {
1199 SCP_INFO(()) <<
"Unprivileged access forbidden by AP[1]";
1200 req->fault = FaultKind::Permission;
1203 if (attrs & (1 << 5)) {
1204 if (req->access == IOMMU_WO) {
1205 SCP_INFO(()) <<
"Write access forbidden " << std::hex << attrs;
1206 req->fault = FaultKind::Permission;
1209 req->prot &= ~IOMMU_WO;
1212 switch ((s2attrs >> 4) & 3) {
1214 req->fault = FaultKind::Permission;
1218 if (req->access == IOMMU_WO) {
1219 req->fault = FaultKind::Permission;
1222 req->prot &= ~IOMMU_WO;
1225 if (req->access == IOMMU_RO) {
1226 req->fault = FaultKind::Permission;
1229 req->prot &= ~IOMMU_RO;
1240 if (req->walk_fault && have_descriptor) {
1241 SCP_WARN(()) <<
"SMMU failed descriptor: CB " << std::dec << cb <<
" address 0x" << std::hex
1242 << last_descaddr <<
" value 0x" << last_desc;
1244 SCP_INFO(()) <<
"smmu fault id " << req->trace_id <<
" SID 0x" << std::hex << req->sid <<
" CB" << cb;
1245 dump_trans_req(*req);
1249 if (!req->internal_walk && !req->suppress_fault) {
1250 const unsigned int fault_cb = (req->nested && req->stage == 2) ? req->s1_cb : cb;
1251 smmu500_fault(fault_cb, req, level);
1255 bool smmu500_at64(
unsigned int cb, uint64_t va,
bool wr,
bool s2, uint64_t* pa,
int* prot, uint64_t* page_size,
1256 uint64_t* fault_addr,
const FaultAttrs& fault_attrs, uint64_t trace_id = 0, uint64_t sid = 0)
1258 unsigned int s2_cb = 0;
1259 const unsigned int nr_cb = std::min<unsigned int>(p_num_cb, SMMU_MAX_CB);
1262 int s1_prot = IOMMU_RW;
1263 uint64_t s1_page_size;
1265 t = SMMU_CBAR[cb][CBAR_TYPE];
1270 req.s2_enabled =
true;
1276 req.s2_enabled =
false;
1280 req.s2_enabled =
false;
1284 req.s2_enabled =
true;
1285 req.s2_cb = SMMU_CBAR[cb][CBAR_S2_CBNDX];
1293 req.trace_id = trace_id;
1295 req.tcr[1] =
static_cast<uint32_t
>(SMMU_CB_TCR2[cb]);
1297 req.tcr[1] |=
static_cast<uint32_t
>(SMMU_CB_TCR_LPAE[cb]);
1299 req.ttbr[1][0] = smmu500_ttbr(
static_cast<uint32_t
>(SMMU_CB_TTBR0_HIGH[cb]),
1300 static_cast<uint32_t
>(SMMU_CB_TTBR0_LOW[cb]));
1302 req.ttbr[1][1] = smmu500_ttbr(
static_cast<uint32_t
>(SMMU_CB_TTBR1_HIGH[cb]),
1303 static_cast<uint32_t
>(SMMU_CB_TTBR1_LOW[cb]));
1305 req.access = wr ? IOMMU_WO : IOMMU_RO;
1306 req.page_size = *page_size;
1307 s1_page_size = req.page_size;
1308 req.privileged = fault_attrs.privileged;
1309 req.instruction = fault_attrs.instruction;
1310 req.non_secure = fault_attrs.non_secure;
1311 req.address_translation = fault_attrs.address_translation;
1312 req.asynchronous = fault_attrs.asynchronous;
1313 req.suppress_fault = fault_attrs.suppress_fault;
1314 req.mid = fault_attrs.mid;
1315 req.pid = fault_attrs.pid;
1316 req.bid = fault_attrs.bid;
1318 if (req.s2_enabled && req.s2_cb >= nr_cb) {
1320 req.fault = FaultKind::Translation;
1322 req.prot = IOMMU_NONE;
1323 if (!req.suppress_fault) smmu500_fault(cb, &req, 0);
1326 *page_size = req.page_size;
1327 if (fault_addr) *fault_addr = va;
1331 if (req.s2_enabled) {
1332 req.tcr[2] =
static_cast<uint32_t
>(SMMU_CB_TCR_LPAE[s2_cb]);
1333 req.ttbr[2][0] = smmu500_ttbr(
static_cast<uint32_t
>(SMMU_CB_TTBR0_HIGH[s2_cb]),
1334 static_cast<uint32_t
>(SMMU_CB_TTBR0_LOW[s2_cb]));
1337 if (req.stage == 1) {
1338 smmu500_ptw64(cb, &req);
1341 s1_page_size = req.page_size;
1348 if (!req.err && s2 && req.s2_enabled) {
1352 smmu500_ptw64(req.s2_cb, &req);
1354 req.prot &= s1_prot;
1355 if (s1_page_size < req.page_size) req.page_size = s1_page_size;
1361 *page_size = req.page_size;
1362 if (fault_addr) *fault_addr = req.ipafar_valid ? req.ipafar : (req.err ? req.va : va);
1366 bool smmu500_at(
unsigned int cb, uint64_t va,
bool wr,
bool s2, uint64_t* pa,
int* prot, uint64_t* page_size,
1367 uint64_t* fault_addr,
const FaultAttrs& fault_attrs, uint64_t trace_id = 0, uint64_t sid = 0)
1369 return smmu500_at64(cb, va, wr, s2, pa, prot, page_size, fault_addr, fault_attrs, trace_id, sid);
1372 void smmu500_gat(uint64_t v,
bool wr,
bool s2,
bool unprivileged,
bool non_secure =
false)
1374 std::lock_guard<std::recursive_mutex> translation_lock(m_translation_lock);
1375 uint64_t va = v & ~SMMU_ADDRMASK;
1376 unsigned int cb = v & SMMU_ADDRMASK;
1379 uint64_t page_size = 12;
1380 FaultAttrs fault_attrs;
1381 fault_attrs.privileged = !unprivileged;
1382 fault_attrs.non_secure = non_secure;
1383 fault_attrs.address_translation =
true;
1385 if (cb >= p_num_cb) {
1386 smmu500_record_global_fault(1u << 0, non_secure);
1388 smmu500_non_secure_gr0_write32(SMMU_GPAR.get_offset(), 1);
1389 smmu500_non_secure_gr0_write32(SMMU_GPAR_H.get_offset(), 0);
1397 SCP_INFO(()) <<
"ATS: va=0x" << std::hex << va <<
" cb=0x" << cb <<
" wr=" << wr <<
" s2=" << s2;
1398 bool err = smmu500_at(cb, va, wr, s2, &pa, &prot, &page_size,
nullptr, fault_attrs);
1401 smmu500_non_secure_gr0_write32(SMMU_GPAR.get_offset(),
static_cast<uint32_t
>(pa | err));
1402 smmu500_non_secure_gr0_write32(SMMU_GPAR_H.get_offset(),
static_cast<uint32_t
>(pa >> 32));
1404 SMMU_GPAR =
static_cast<uint32_t
>(pa | err);
1405 SMMU_GPAR_H =
static_cast<uint32_t
>(pa >> 32);
1410 static constexpr uint32_t SMMU_STREAM_ID_MASK = 0x7FFFu;
1411 static constexpr uint64_t SMMU_DSP_WINDOW_START = 0x90000000ULL;
1412 static constexpr uint64_t SMMU_DSP_WINDOW_END = 0xa0000000ULL;
1414 static uint16_t smmu500_compose_stream_id(uint32_t topology_id, uint64_t addr,
bool address_carries_csid =
true)
1419 const uint32_t tbu_id = (topology_id >> 10) & 0x1fu;
1420 const uint32_t topo_id = (topology_id >> 5) & 0x1fu;
1421 const uint32_t configured_csid = topology_id & 0xfu;
1422 const uint32_t transport_csid = address_carries_csid ?
static_cast<uint32_t
>((addr >> 32) & 0xfu) : 0;
1423 const uint32_t csid = transport_csid != 0 ? transport_csid : configured_csid;
1424 return static_cast<uint16_t
>((tbu_id << 10) | (topo_id << 5) | csid);
1427 uint16_t smmu500_resolve_stream_id(uint32_t topology_id, uint64_t addr,
int* cb,
1428 bool* stream_match_conflict =
nullptr, uint32_t* s2cr_type =
nullptr,
1429 bool address_carries_csid =
true)
1431 std::lock_guard<std::recursive_mutex> translation_lock(m_translation_lock);
1432 const uint16_t stream_id = smmu500_compose_stream_id(topology_id, addr, address_carries_csid);
1433 bool conflict =
false;
1435 const int resolved_cb = smmu500_stream_id_match(stream_id, &conflict, &type);
1436 if (cb) *cb = resolved_cb;
1437 if (stream_match_conflict) *stream_match_conflict = conflict;
1438 if (s2cr_type) *s2cr_type = type;
1442 IOMMUTLBEntry smmu500_translate(tlm::tlm_generic_payload& txn, uint64_t sid,
bool transport_stream_address =
false,
1443 bool record_fault =
true)
1445 std::lock_guard<std::recursive_mutex> translation_lock(m_translation_lock);
1446 const uint64_t trace_id = ++m_translation_seq;
1447 uint64_t addr = txn.get_address();
1448 uint64_t page_size = 12;
1449 IOMMUTLBEntry ret = {
1453 .translated_addr = addr,
1454 .addr_mask = (1ULL << page_size) - 1,
1461 const bool address_carries_csid = transport_stream_address && (addr >> 36) == 0;
1462 uint64_t va = addr & ~SMMU_ADDRMASK;
1463 if (address_carries_csid) {
1464 va &= ((1ULL << SMMU_VA_WIDTH) - 1);
1468 uint64_t fault_addr = va;
1470 const uint16_t master_id = smmu500_compose_stream_id(
static_cast<uint32_t
>(sid), addr, address_carries_csid);
1471 const uint32_t transport_stream_bits = address_carries_csid ?
static_cast<uint32_t
>(master_id & 0xfu) : 0;
1472 const FaultAttrs fault_attrs = smmu500_extract_fault_attrs(txn, record_fault);
1473 const uint32_t global_control = smmu500_global_control(fault_attrs.non_secure);
1474 bool stream_match_conflict =
false;
1475 uint32_t s2cr_type = 0;
1476 cb = smmu500_stream_id_match(master_id, &stream_match_conflict, &s2cr_type);
1477 if (transport_stream_address && (addr >> 36) != 0) {
1478 SCP_INFO(()) << std::hex <<
"SMMU high-VA TBU request id " << trace_id <<
" address 0x" << addr
1479 <<
" configured SID 0x" << sid <<
" effective SID 0x" << master_id <<
" CSID carrier "
1480 << address_carries_csid <<
" CB " << cb <<
" S2CR type " << s2cr_type;
1482 bool clientpd = global_control & SCR0_CLIENTPD_MASK;
1494 const bool has_transport_csid = address_carries_csid && ((addr >> 32) & 0xfu) != 0;
1495 if ((global_control & SCR0_USFCFG_MASK) && (!transport_stream_address || has_transport_csid)) {
1496 smmu500_report_global_fault(ret, 1u << 1, fault_attrs.non_secure, global_control, record_fault);
1502 if (stream_match_conflict) {
1503 if (global_control & SCR0_SMCFCFG_MASK) {
1504 smmu500_report_global_fault(ret, 1u << 2, fault_attrs.non_secure, global_control, record_fault);
1510 if (s2cr_type == 1) {
1514 const unsigned int nr_cb = std::min<unsigned int>(p_num_cb, SMMU_MAX_CB);
1515 if (s2cr_type != 0 || cb >=
static_cast<int>(nr_cb)) {
1516 smmu500_report_global_fault(ret, 1u << 0, fault_attrs.non_secure, global_control, record_fault);
1519 if (!SMMU_CBA2R[cb][CBA2R_VA64]) {
1520 va &= ((1ULL << SMMU_VA_WIDTH) - 1);
1523 bool wr = (txn.get_command() == tlm::TLM_WRITE_COMMAND);
1524 const bool fault_latched = (
static_cast<uint32_t
>(SMMU_CB_FSR[cb]) & CB_FSR_FAULT_MASK) != 0;
1528 err = smmu500_at(cb, va, wr,
true, &pa, &prot, &page_size, &fault_addr, fault_attrs, trace_id, master_id);
1529 ret.translated_addr = pa;
1530 ret.perm = (IOMMUAccessFlags)prot;
1532 if (record_fault && !fault_latched) {
1533 const uint64_t recorded_addr = (fault_addr == va) ? (va | (addr & SMMU_PAGEMASK)) : fault_addr;
1534 SMMU_CB_IPAFAR_LOW[cb] =
static_cast<uint32_t
>(recorded_addr);
1535 SMMU_CB_IPAFAR_HIGH[cb] =
static_cast<uint32_t
>(recorded_addr >> 32);
1536 if (SMMU_CBAR[cb][CBAR_TYPE] == 0 || SMMU_CBAR[cb][CBAR_TYPE] == 3) {
1537 SMMU_CB_FAR_LOW[cb] =
static_cast<uint32_t
>(recorded_addr);
1538 SMMU_CB_FAR_HIGH[cb] =
static_cast<uint32_t
>(recorded_addr >> 32);
1541 memset(&ret, 0,
sizeof ret);
1543 ret.perm = IOMMU_NONE;
1545 ret.addr_mask = (page_size >= 64) ? UINT64_MAX : ((1ULL << page_size) - 1);
1546 ret.translated_addr &= ~ret.addr_mask;
1551 int smmu500_stream_id_match(uint32_t stream_id,
bool* stream_match_conflict =
nullptr,
1552 uint32_t* s2cr_type =
nullptr)
1554 const uint32_t effective_stream_id = stream_id & SMMU_STREAM_ID_MASK;
1558 const unsigned int nr_smr = std::min<unsigned int>(
static_cast<unsigned int>(p_num_smr), SMMU_MAX_SMR);
1560 bool conflict =
false;
1563 for (
unsigned int i = 0; i < nr_smr; i++) {
1564 bool valid = SMMU_SMR[i][SMR_VALID];
1565 const uint32_t mask =
static_cast<uint32_t
>(SMMU_SMR[i][SMR_MASK]) & SMMU_STREAM_ID_MASK;
1566 const uint32_t
id =
static_cast<uint32_t
>(SMMU_SMR[i][SMR_ID]) & SMMU_STREAM_ID_MASK;
1568 if (valid && ((
id ^ effective_stream_id) & ~mask & SMMU_STREAM_ID_MASK) == 0) {
1570 cbndx =
static_cast<int>(SMMU_S2CR[i][S2CR_CBNDX_VMID]);
1571 type =
static_cast<uint32_t
>(SMMU_S2CR[i][S2CR_TYPE]);
1578 if (stream_match_conflict) *stream_match_conflict = conflict;
1579 if (s2cr_type) *s2cr_type = type;
1584 void start_of_simulation();
1585 void before_end_of_elaboration();
1589class smmu500_tbu :
public sc_core::sc_module
1594 static constexpr unsigned int TRANSPORT_IOVA_WIDTH = 32;
1598 cci::cci_broker_handle m_broker;
1600 std::pair<uint64_t, uint64_t> dmi_range[SMMU_MAX_CB] = {};
1601 bool dmi_range_valid[SMMU_MAX_CB] = {
false };
1602 std::pair<uint64_t, uint64_t> bypass_dmi_range = {};
1603 bool bypass_dmi_range_valid =
false;
1607 uint64_t page_start;
1615 struct QemuIommuRegion {
1616 const uint64_t start;
1617 const uint64_t size;
1619 std::shared_ptr<qemu::IOMMUMemoryRegion> mr;
1620 std::map<uint64_t, qemu::IOMMUMemoryRegion::IOMMUTLBEntry> ranges[2];
1621 std::mutex cache_mutex;
1623 QemuIommuRegion(uint64_t region_start, uint64_t region_size): start(region_start), size(region_size) {}
1626 std::unique_ptr<QemuIommuRegion> m_qemu_iommu_region;
1628 void smmu500_qemu_setup_region()
1630 auto& region = *m_qemu_iommu_region;
1631 auto& inst = m_qemu_instance->get();
1633 region.owner = inst.template object_new_unparented<QemuInstanceDmiManager::QemuContainer>();
1634 region.mr = std::make_shared<qemu::IOMMUMemoryRegion>(
1635 inst.template object_new_unparented<qemu::IOMMUMemoryRegion>());
1637 region.owner,
"smmu500-iommu", region.size, downstream_socket.create_io_ops(),
1639 qemu::IOMMUMemoryRegion::IOMMUAccessFlags flags,
1640 int idx) { smmu500_qemu_translate(*te, addr, flags, idx); },
1643 qemu::MemoryRegionOps::MemTxAttrs attrs;
1644 attrs.secure = idx != 0;
1647 downstream_socket.init(*region.mr);
1651 qemu::IOMMUMemoryRegion::IOMMUAccessFlags flags,
int idx)
1653 auto& region = *m_qemu_iommu_region;
1654 entry = { .target_as = downstream_socket.get_address_space().get_ptr(),
1655 .iova = addr & ~SMMU_PAGEMASK,
1656 .translated_addr = addr & ~SMMU_PAGEMASK,
1657 .addr_mask = SMMU_PAGEMASK,
1658 .perm = qemu::IOMMUMemoryRegion::IOMMU_NONE };
1659 if (addr >= region.size) {
1663 const bool cache_eligible = flags == qemu::IOMMUMemoryRegion::IOMMU_NONE && (idx == 0 || idx == 1);
1664 if (cache_eligible) {
1665 std::lock_guard<std::mutex> cache_lock(region.cache_mutex);
1666 auto& ranges = region.ranges[idx];
1667 auto it = ranges.upper_bound(addr);
1668 if (it != ranges.begin()) {
1670 if (addr - it->second.iova <= it->second.addr_mask) {
1677 auto translated = entry;
1678 tlm::tlm_generic_payload txn;
1679 const uint64_t iova = region.start + addr;
1680 txn.set_address(iova);
1682 txn.set_command(flags == qemu::IOMMUMemoryRegion::IOMMU_NONE ? tlm::TLM_IGNORE_COMMAND
1683 : (flags & qemu::IOMMUMemoryRegion::IOMMU_WO) ? tlm::TLM_WRITE_COMMAND
1684 : tlm::TLM_READ_COMMAND);
1687 const bool record_fault = flags == qemu::IOMMUMemoryRegion::IOMMU_RO ||
1688 flags == qemu::IOMMUMemoryRegion::IOMMU_WO;
1690 te = m_smmu->smmu500_translate(txn, p_topology_id,
true, record_fault);
1692 auto& inst = m_qemu_instance->get();
1693 const bool iothread_locked = inst.is_iothread_locked();
1694 if (iothread_locked) inst.unlock_iothread();
1695 m_smmu->m_dma_on_sysc.run_on_sysc(
1696 [&, record_fault] { te = m_smmu->smmu500_translate(txn, p_topology_id,
true, record_fault); });
1697 if (iothread_locked) inst.lock_iothread();
1699 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE)
return;
1700 const uint64_t pa = (te.translated_addr & ~te.addr_mask) | (iova & te.addr_mask);
1701 uint64_t mask = te.addr_mask;
1702 const bool transport_csid = (iova >> 36) == 0;
1703 const uint64_t csid_window_mask =
static_cast<uint64_t
>(UINT32_MAX);
1704 const uint64_t csid_window_end = (iova & ~csid_window_mask) | csid_window_mask;
1705 while ((addr & mask) != (pa & mask) || (addr | mask) >= region.size ||
1706 (transport_csid && (iova | mask) > csid_window_end)) {
1710 translated.target_as = downstream_socket.get_address_space().get_ptr();
1711 translated.iova = addr & ~mask;
1712 translated.translated_addr = pa & ~mask;
1713 translated.addr_mask = mask;
1715 case smmu500<BUSWIDTH>::IOMMU_RO:
1716 translated.perm = qemu::IOMMUMemoryRegion::IOMMU_RO;
1718 case smmu500<BUSWIDTH>::IOMMU_WO:
1719 translated.perm = qemu::IOMMUMemoryRegion::IOMMU_WO;
1721 case smmu500<BUSWIDTH>::IOMMU_RW:
1722 translated.perm = qemu::IOMMUMemoryRegion::IOMMU_RW;
1728 if (cache_eligible && entry.perm != qemu::IOMMUMemoryRegion::IOMMU_NONE) {
1729 std::lock_guard<std::mutex> cache_lock(region.cache_mutex);
1730 region.ranges[idx][entry.iova] = entry;
1734 void set_iommu_hint(tlm::tlm_generic_payload& txn)
1736 const uint64_t iova = txn.get_address();
1738 if (!origin || !origin->instance)
return;
1740 if (m_qemu_instance ==
nullptr) {
1741 downstream_socket.set_qemu_inst(*origin->instance);
1742 m_qemu_instance = origin->instance;
1744 if (origin->instance != m_qemu_instance) {
1745 SCP_WARN(()) <<
"Cannot replace QEMU MR hint from a different QEMU instance";
1748 if (!m_qemu_iommu_region) {
1749 SCP_WARN(()) <<
"Cannot replace QEMU MR hint: upstream socket size is zero";
1752 auto& region = *m_qemu_iommu_region;
1753 if (iova < region.start || iova - region.start >= region.size) {
1754 SCP_WARN(()) <<
"Cannot replace QEMU MR hint for IOVA 0x" << std::hex << iova
1755 <<
": it is outside upstream socket [0x" << region.start <<
", 0x"
1756 << region.start + region.size - 1 <<
"]";
1761 downstream_socket.set_cpu(cpu_hint->get_cpu());
1765 auto& inst = m_qemu_instance->get();
1766 const bool locked = inst.is_iothread_locked();
1767 if (!locked) inst.lock_iothread();
1768 smmu500_qemu_setup_region();
1769 if (!locked) inst.unlock_iothread();
1775 hint->copy_from(iommu_hint);
1781 void smmu500_qemu_invalidate()
1783 if (!m_qemu_iommu_region || !m_qemu_iommu_region->mr)
return;
1785 auto& region = *m_qemu_iommu_region;
1786 auto& inst = region.mr->get_inst();
1787 const bool iothread_locked = inst.is_iothread_locked();
1788 if (!iothread_locked) inst.lock_iothread();
1790 std::lock_guard<std::mutex> cache_lock(region.cache_mutex);
1791 region.ranges[0].clear();
1792 region.ranges[1].clear();
1795 .target_as = downstream_socket.get_address_space().get_ptr(),
1797 .translated_addr = 0,
1798 .addr_mask = UINT64_MAX,
1799 .perm = qemu::IOMMUMemoryRegion::IOMMU_NONE,
1801 region.mr->iommu_unmap(&flush);
1802 if (!iothread_locked) inst.unlock_iothread();
1808 void b_transport(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay)
1810 sc_dt::uint64 addr = txn.get_address();
1811 tlm::tlm_command cmd = txn.get_command();
1812 const bool high_va = (addr >> 36) != 0;
1814 SCP_INFO(()) << std::hex <<
"TBU high-VA b_transport address 0x" << addr <<
" topology ID 0x"
1815 <<
static_cast<uint32_t
>(p_topology_id);
1819 SCP_INFO(()) << std::hex <<
"TBU high-VA b_transport result PA 0x" << te.translated_addr <<
" mask 0x"
1820 << te.addr_mask <<
" permissions " << te.perm;
1822 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE ||
1823 (cmd == tlm::TLM_WRITE_COMMAND && te.perm == smmu500<BUSWIDTH>::IOMMU_RO) ||
1824 (cmd == tlm::TLM_READ_COMMAND && te.perm == smmu500<BUSWIDTH>::IOMMU_WO)) {
1825 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
1826 wait(sc_core::SC_ZERO_TIME);
1828 txn.set_address((te.translated_addr & ~te.addr_mask) | (addr & te.addr_mask));
1829 downstream_socket->b_transport(txn, delay);
1830 txn.set_address(addr);
1832 set_iommu_hint(txn);
1837 virtual unsigned int transport_dbg(tlm::tlm_generic_payload& txn)
1839 sc_dt::uint64 addr = txn.get_address();
1841 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE) {
1844 txn.set_address((te.translated_addr & ~te.addr_mask) | (addr & te.addr_mask));
1845 int ret = downstream_socket->transport_dbg(txn);
1846 txn.set_address(addr);
1850 virtual bool get_direct_mem_ptr(tlm::tlm_generic_payload& txn, tlm::tlm_dmi& dmi_data)
1856 const uint64_t addr = txn.get_address();
1857 if (m_qemu_iommu_region && addr >= m_qemu_iommu_region->start &&
1858 addr - m_qemu_iommu_region->start < m_qemu_iommu_region->size) {
1859 dmi_data.set_start_address(m_qemu_iommu_region->start);
1860 dmi_data.set_end_address(m_qemu_iommu_region->start + m_qemu_iommu_region->size - 1);
1862 dmi_data.set_start_address(addr & ~SMMU_PAGEMASK);
1863 dmi_data.set_end_address(addr | SMMU_PAGEMASK);
1865 if (origin->instance == m_qemu_instance) {
1867 downstream_socket.set_cpu(cpu_hint->get_cpu());
1876 VIRT.address = txn.get_address();
1877 const bool high_va = (VIRT.address >> 36) != 0;
1879 SCP_INFO(()) << std::hex <<
"TBU high-VA DMI request address 0x" << VIRT.address <<
" topology ID 0x"
1880 <<
static_cast<uint32_t
>(p_topology_id);
1884 SCP_INFO(()) << std::hex <<
"TBU high-VA DMI result PA 0x" << te.translated_addr <<
" mask 0x"
1885 << te.addr_mask <<
" permissions " << te.perm;
1888 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE) {
1890 const bool address_carries_csid = (VIRT.address >> 36) == 0;
1891 m_smmu->smmu500_resolve_stream_id(p_topology_id, VIRT.address, &CB,
nullptr,
nullptr, address_carries_csid);
1892 if (CB >= 0 &&
static_cast<unsigned int>(CB) < SMMU_MAX_CB) {
1893 const uint64_t page = VIRT.address & ~SMMU_PAGEMASK;
1894 if (!dmi_range_valid[CB] || dmi_range[CB].first > page) {
1895 dmi_range[CB].first = page;
1897 if (!dmi_range_valid[CB] || dmi_range[CB].second < page + SMMU_PAGEMASK) {
1898 dmi_range[CB].second = page + SMMU_PAGEMASK;
1900 dmi_range_valid[CB] =
true;
1905 if (te.addr_mask ==
static_cast<uint64_t
>(-1)) {
1906 assert(te.translated_addr == VIRT.address);
1907 bool ret = downstream_socket->get_direct_mem_ptr(txn, dmi_data);
1911 if (
reinterpret_cast<uintptr_t
>(dmi_data.get_dmi_ptr()) < 4096) {
1914 VIRT.page_end = dmi_data.get_end_address();
1915 if (VIRT.page_end >> TRANSPORT_IOVA_WIDTH != VIRT.address >> TRANSPORT_IOVA_WIDTH) {
1916 dmi_data.set_end_address((((VIRT.address >> TRANSPORT_IOVA_WIDTH) + 1) << TRANSPORT_IOVA_WIDTH) - 1);
1918 VIRT.page_start = dmi_data.get_start_address();
1919 if (VIRT.page_start >> TRANSPORT_IOVA_WIDTH != VIRT.address >> TRANSPORT_IOVA_WIDTH) {
1920 uint64_t newstart = (VIRT.address >> TRANSPORT_IOVA_WIDTH) << TRANSPORT_IOVA_WIDTH;
1921 dmi_data.set_start_address(newstart);
1922 dmi_data.set_dmi_ptr(dmi_data.get_dmi_ptr() + (newstart - VIRT.page_start));
1924 if (!bypass_dmi_range_valid || bypass_dmi_range.first > dmi_data.get_start_address()) {
1925 bypass_dmi_range.first = dmi_data.get_start_address();
1927 if (!bypass_dmi_range_valid || bypass_dmi_range.second < dmi_data.get_end_address()) {
1928 bypass_dmi_range.second = dmi_data.get_end_address();
1930 bypass_dmi_range_valid =
true;
1934 PHYS.address = te.translated_addr;
1935 PHYS.page_start = te.translated_addr & ~te.addr_mask;
1936 PHYS.page_size = (te.addr_mask + 1);
1937 PHYS.page_end = PHYS.page_start + PHYS.page_size - 1;
1938 assert(PHYS.page_start <= PHYS.page_end);
1940 txn.set_address(PHYS.page_start);
1942 int ret = downstream_socket->get_direct_mem_ptr(txn, dmi_data);
1944 txn.set_address(VIRT.address);
1947 if (
reinterpret_cast<uintptr_t
>(dmi_data.get_dmi_ptr()) < 4096) {
1948 txn.set_address(VIRT.address);
1952 uint64_t dmi_offset = PHYS.page_start - dmi_data.get_start_address();
1953 assert(dmi_data.get_start_address() <= PHYS.page_start);
1954 assert(PHYS.page_end <= dmi_data.get_end_address());
1956 VIRT.page_start = VIRT.address & ~te.addr_mask;
1957 VIRT.page_end = VIRT.page_start + te.addr_mask;
1959 dmi_data.set_dmi_ptr(dmi_data.get_dmi_ptr() + dmi_offset);
1960 dmi_data.set_start_address(VIRT.page_start);
1961 dmi_data.set_end_address(VIRT.page_end);
1963 case smmu500<BUSWIDTH>::IOMMU_RO:
1964 dmi_data.allow_read();
1966 case smmu500<BUSWIDTH>::IOMMU_WO:
1967 dmi_data.allow_write();
1969 case smmu500<BUSWIDTH>::IOMMU_RW:
1970 dmi_data.allow_read_write();
1973 txn.set_address(VIRT.address);
1978 const bool address_carries_csid = (VIRT.address >> 36) == 0;
1979 txn.set_address(VIRT.address);
1980 m_smmu->smmu500_resolve_stream_id(p_topology_id, VIRT.address, &CB,
nullptr,
nullptr, address_carries_csid);
1984 txn.set_address(VIRT.address);
1987 if (!dmi_range_valid[CB] || dmi_range[CB].first > VIRT.page_start) {
1988 dmi_range[CB].first = VIRT.page_start;
1990 if (!dmi_range_valid[CB] || dmi_range[CB].second < VIRT.page_end) {
1991 dmi_range[CB].second = VIRT.page_end;
1993 dmi_range_valid[CB] =
true;
1995 if (PHYS.page_start != VIRT.page_start) {
1996 uint64_t bits_masked = VIRT.address - (VIRT.address & ~SMMU_PAGEMASK);
1997 SCP_DEBUG(()) << std::hex <<
"\nVIRT Page Start 0x" << VIRT.page_start <<
"\nPHYS Page Start 0x"
1998 << PHYS.page_start <<
"\nbits_masked 0x" << bits_masked;
2000 SCP_DEBUG(()) << std::hex <<
"smmu TBU DMI: translate 0x" << VIRT.address <<
" to 0x" << te.translated_addr
2001 <<
" pg size 0x" << PHYS.page_size <<
" pg base 0x" << PHYS.page_start <<
" offset 0x"
2002 << dmi_offset <<
" start 0x" << dmi_data.get_start_address() <<
" end 0x"
2003 << dmi_data.get_end_address();
2005 txn.set_address(VIRT.address);
2010 cci::cci_param<uint32_t> p_topology_id;
2012 tlm_utils::simple_target_socket<smmu500_tbu> upstream_socket;
2016 tlm_utils::simple_initiator_socket<smmu500_tbu> downstream_socket;
2019 smmu500_tbu(
const sc_core::sc_module_name& name, sc_core::sc_object* o)
2030 : m_broker(cci::cci_get_broker())
2031 , p_topology_id(
"topology_id", 0x0,
"Topology ID for this TBU")
2032 , upstream_socket(
"upstream_socket")
2034 , downstream_socket(
"downstream_socket", qemu_instance)
2036 , downstream_socket(
"downstream_socket")
2040 m_smmu->tbus.push_back(
this);
2042 m_qemu_instance = qemu_instance;
2044 upstream_socket.register_b_transport(
this, &smmu500_tbu::b_transport);
2045 upstream_socket.register_transport_dbg(
this, &smmu500_tbu::transport_dbg);
2046 upstream_socket.register_get_direct_mem_ptr(
this, &smmu500_tbu::get_direct_mem_ptr);
2048 downstream_socket.set_invalidate_observer(
2049 [
this](sc_dt::uint64 start, sc_dt::uint64 end) { invalidate_direct_mem_ptr(start, end); });
2051 downstream_socket.register_invalidate_direct_mem_ptr(
this, &smmu500_tbu::invalidate_direct_mem_ptr);
2057 : smmu500_tbu(name, _smmu, &qemu_instance)
2063 void end_of_elaboration()
override
2065 const std::string socket_name = std::string(sc_core::sc_module::name()) +
".upstream_socket";
2066 uint64_t region_start = gs::cci_get_d<uint64_t>(m_broker, socket_name +
".address", 0);
2067 if (gs::cci_get_d<bool>(m_broker, socket_name +
".relative_addresses",
true)) region_start = 0;
2068 uint64_t region_size = 0;
2069 const bool has_size = gs::cci_get(m_broker, socket_name +
".size", region_size);
2071 const sc_core::sc_object* socket = &upstream_socket;
2072 std::vector<const sc_core::sc_object*> visited;
2073 while (!has_size && socket) {
2074 if (std::find(visited.begin(), visited.end(), socket) != visited.end())
break;
2075 visited.push_back(socket);
2076 const auto*
module = socket->get_parent_object();
2077 const std::string bind = gs::cci_get_d<std::string>(m_broker, std::string(socket->name()) +
".bind",
"");
2078 if (bind.empty())
break;
2079 const std::string path = bind.front() ==
'&' ? bind.substr(1) : bind;
2080 socket = sc_core::sc_find_object(path.c_str());
2081 for (
auto* scope = module ? module->get_parent_object() :
nullptr; !socket && scope;
2082 scope = scope->get_parent_object()) {
2083 socket = sc_core::sc_find_object((std::string(scope->name()) +
"." + path).c_str());
2086 using TargetSocket = tlm::tlm_base_target_socket_b<BUSWIDTH, tlm::tlm_fw_transport_if<>,
2087 tlm::tlm_bw_transport_if<>>;
2088 if (!
dynamic_cast<const TargetSocket*
>(socket) &&
2089 gs::cci_get_d<std::string>(m_broker, std::string(socket->name()) +
".bind",
"").empty()) {
2090 const auto* parent = socket->get_parent_object();
2091 socket = parent ? sc_core::sc_find_object((std::string(parent->name()) +
".target_socket").c_str())
2094 if (
dynamic_cast<const TargetSocket*
>(socket) &&
2095 gs::cci_get(m_broker, std::string(socket->name()) +
".size", region_size)) {
2096 region_start = gs::cci_get_d<uint64_t>(m_broker, std::string(socket->name()) +
".address", 0);
2097 if (gs::cci_get_d<bool>(m_broker, std::string(socket->name()) +
".relative_addresses",
true)) {
2100 region_start = gs::cci_get_d<uint64_t>(
2101 m_broker, std::string(socket->get_parent_object()->name()) +
".mapped_base_addr", region_start);
2105 if (region_size != 0) {
2106 m_qemu_iommu_region = std::make_unique<QemuIommuRegion>(region_start, region_size);
2111 void invalidate_direct_mem_ptr(sc_dt::uint64 start, sc_dt::uint64 end)
2118 void invalidate(uint32_t CB)
2120 if (CB >= SMMU_MAX_CB)
return;
2122 smmu500_qemu_invalidate();
2124 invalidate_dmi_range(CB);
2127 void invalidate_dmi_range(uint32_t CB)
2129 if (CB >= SMMU_MAX_CB)
return;
2130 if (!dmi_range_valid[CB])
return;
2131 SCP_DEBUG(())(
"TLBIALL invalidate {:x} - {:x}", dmi_range[CB].first, dmi_range[CB].second);
2132 upstream_socket->invalidate_direct_mem_ptr(dmi_range[CB].first, dmi_range[CB].second);
2133 dmi_range_valid[CB] =
false;
2139 smmu500_qemu_invalidate();
2141 for (
unsigned int cb = 0; cb < SMMU_MAX_CB; ++cb) invalidate_dmi_range(cb);
2142 if (bypass_dmi_range_valid) {
2143 upstream_socket->invalidate_direct_mem_ptr(bypass_dmi_range.first, bypass_dmi_range.second);
2144 bypass_dmi_range_valid =
false;
2149 void end_of_simulation()
override
2151 if (m_qemu_instance ==
nullptr || !m_qemu_iommu_region || !m_qemu_iommu_region->mr)
return;
2153 auto& inst = m_qemu_iommu_region->mr->get_inst();
2154 const bool iothread_locked = inst.is_iothread_locked();
2155 if (!iothread_locked) inst.lock_iothread();
2156 smmu500_qemu_invalidate();
2157 if (!iothread_locked) inst.unlock_iothread();