qualcomm/qbox
Loading...
Searching...
No Matches
smmu500.h
1/*
2 * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All Rights Reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#ifndef smmu500_H
8#define smmu500_H
9
10#include <systemc>
11#include <cci_configuration>
12#include <cciutils.h>
13#include <scp/report.h>
14#include <tlm>
15#include <async_event.h>
16#include <runonsysc.h>
17#include <module_factory_registery.h>
18#include <ports/initiator-signal-socket.h>
19#include <ports/target-signal-socket.h>
20#include <tlm_utils/simple_initiator_socket.h>
21#include <tlm_utils/simple_target_socket.h>
22#include <tlm-extensions/qemu-memtx-attrs.h>
23#ifndef WITHOUT_QEMU
24#include <libqemu-cxx/libqemu-cxx.h>
25#include <ports/initiator.h>
26#include <tlm-extensions/qemu-instance-hint.h>
27#include <tlm-extensions/qemu-memtx-attr-extensions.h>
28#include <tlm-extensions/qemu-mr-hint.h>
29#endif
30
31#include <cassert>
32#include <algorithm>
33#include <array>
34#include <atomic>
35#include <cstdint>
36#include <cstring>
37#include <map>
38#include <vector>
39#include <memory>
40#include <mutex>
41#include <sstream>
42
43#include "smmu500_gen.h"
44
45#define SMMU_PAGESIZE 4096
46#define SMMU_PAGEMASK (SMMU_PAGESIZE - 1)
47#define SMMU_MAX_CB 128
48#define SMMU_MAX_SMR 224
49#define SMMU_MAX_TBU 16
50#define SMMU_VA_WIDTH 32
51#define SMMU_ADDRMASK ((1ULL << 12) - 1)
52
53namespace gs {
54
55// Optional transaction attributes used to populate the context-fault syndrome.
56// Ordinary TLM payloads do not carry these bus-level fields, so the defaults
57// describe a synchronous, privileged, non-secure data access.
58class smmu500_transaction_attrs_extension : public tlm::tlm_extension<smmu500_transaction_attrs_extension>
59{
60public:
61 uint8_t mid = 0;
62 uint8_t pid = 0;
63 uint8_t bid = 0;
64 bool privileged = true;
65 bool instruction = false;
66 bool non_secure = true;
67 bool address_translation = false; // Address-translation operation; reports FSYNR0.ATOF on a fault.
68 bool asynchronous = false;
69
70 tlm::tlm_extension_base* clone() const override { return new smmu500_transaction_attrs_extension(*this); }
71
72 void copy_from(const tlm::tlm_extension_base& ext) override
73 {
74 const auto& other = static_cast<const smmu500_transaction_attrs_extension&>(ext);
75 *this = other;
76 }
77};
78
79template <unsigned int BUSWIDTH = 32>
80class smmu500_tbu;
81
82template <unsigned int BUSWIDTH = 32>
83class smmu500 : public sc_core::sc_module, public smmu500_gen
84{
85public:
86 struct DmaDmi {
87 unsigned char* ptr = nullptr;
88 uint64_t start = 0;
89 uint64_t end = 0;
90 bool secure = false;
91 };
92
93 struct DmaDmiMiss {
94 };
95
96private:
97 SCP_LOGGER();
98 cci::cci_broker_handle m_broker;
100 bool loaded_ok;
102 tlm_utils::simple_initiator_socket<smmu500, BUSWIDTH> m_register_socket;
103
104public:
105 typedef enum {
106 IOMMU_NONE = 0,
107 IOMMU_RO = 1,
108 IOMMU_WO = 2,
109 IOMMU_RW = 3,
110 } IOMMUAccessFlags;
111
113 uint64_t iova;
114 uint64_t translated_addr;
115 uint64_t addr_mask;
116 IOMMUAccessFlags perm;
117 };
118
119 cci::cci_param<uint32_t> p_pamax;
120 cci::cci_param<uint16_t> p_num_smr;
121 cci::cci_param<uint16_t> p_num_cb;
122 cci::cci_param<uint16_t> p_num_pages;
123 cci::cci_param<bool> p_ato;
124 cci::cci_param<uint8_t> p_version;
125 cci::cci_param<uint32_t> p_idr2;
126 cci::cci_param<uint8_t> p_num_tbu;
127
128 tlm_utils::multi_passthrough_target_socket<smmu500, BUSWIDTH> socket;
129 tlm_utils::simple_initiator_socket<smmu500> dma_socket;
130 gs::runonsysc m_dma_on_sysc;
131 InitiatorSignalSocket<bool> irq_global;
132 sc_core::sc_vector<InitiatorSignalSocket<bool>> irq_context;
134
135 std::vector<smmu500_tbu<BUSWIDTH>*> tbus;
136
137 /* TTBR_HIGH[15:0] is the upper address; [31:16] is ASID. */
138 static uint64_t smmu500_ttbr(uint32_t high, uint32_t low)
139 {
140 return (static_cast<uint64_t>(high & 0xffffu) << 32) | (static_cast<uint64_t>(low) & 0xffffffffu);
141 }
142
143 smmu500(sc_core::sc_module_name name);
144
145private:
146 enum class FaultKind {
147 Translation, // Invalid descriptor or a disabled translation walk.
148 AccessFlag, // Descriptor access flag is clear.
149 Permission, // Descriptor permissions reject the requested access.
150 External, // Page-table descriptor fetch failed on the DMA path.
151 AddressSize, // Input, output, or table address exceeds its configured width.
152 };
153
154 static constexpr uint32_t CB_FSR_TF_MASK = 1u << 1;
155 static constexpr uint32_t CB_FSR_AFF_MASK = 1u << 2;
156 static constexpr uint32_t CB_FSR_PF_MASK = 1u << 3;
157 static constexpr uint32_t CB_FSR_EF_MASK = 1u << 4;
158 static constexpr uint32_t CB_FSR_ASF_MASK = 1u << 7;
159 static constexpr uint32_t CB_FSR_MULTI_MASK = 1u << 31;
160 static constexpr uint32_t CB_FSR_FAULT_MASK = CB_FSR_TF_MASK | CB_FSR_AFF_MASK | CB_FSR_PF_MASK | CB_FSR_EF_MASK |
161 CB_FSR_ASF_MASK;
162 static constexpr uint32_t GLOBAL_FAULT_MASK = (1u << 0) | (1u << 1) | (1u << 2);
163 static constexpr uint32_t SCR0_CLIENTPD_MASK = 1u << 0;
164 static constexpr uint32_t SCR0_GFRE_MASK = 1u << 1;
165 static constexpr uint32_t SCR0_GFIE_MASK = 1u << 2;
166 static constexpr uint32_t SCR0_USFCFG_MASK = 1u << 10;
167 static constexpr uint32_t SCR0_SMCFCFG_MASK = 1u << 21;
168 static constexpr uint64_t NS_GLOBAL_REG_START = 0x400;
169 static constexpr uint64_t NS_GLOBAL_REG_END = 0x800;
170 static constexpr uint64_t SMR_BASE = 0x800;
171 static constexpr uint64_t S2CR_BASE = 0xc00;
172 static constexpr uint64_t CBAR_BASE = 0x1000;
173 static constexpr uint64_t CBA2R_BASE = 0x1800;
174 static constexpr uint64_t TBU_PWR_STATUS = 0x2204;
175 static constexpr uint64_t SMMU_SCR2_OFFSET = 0x008;
176 static constexpr uint64_t SMMU_GFSR_OFFSET = 0x040;
177 static constexpr uint64_t SMMU_GFSR_SIZE = 0x01c;
178 static constexpr uint64_t SMMU_TLBIVAH_RANGE_OFFSET = 0x070;
179 static constexpr uint64_t SMMU_TLBIVAH_RANGE_SIZE = 0x008;
180 static constexpr uint64_t SMMU_GPAR_RANGE_SIZE = 0x00c;
181
182 std::array<uint32_t, SMMU_MAX_CB> m_cb_fsr_before_write = {};
183 uint32_t m_sgfsr_before_write = 0;
184 uint32_t m_nsgfsr_before_write = 0;
185
186 std::array<uint8_t, 0x190> m_secure_gr0_shadow = {};
187 std::array<uint8_t, 0x190> m_non_secure_gr0_shadow = {};
188
189 std::atomic<uint64_t> m_translation_seq{ 0 };
190 /* Page-table/DMI callbacks can re-enter translation on the same
191 * SystemC thread. A plain mutex deadlocks that legal callback path. */
192 std::recursive_mutex m_translation_lock;
193 DmaDmi m_dma_dmi{};
194 std::mutex m_dma_dmi_lock;
195
196 void invalidate_dma_dmi(sc_dt::uint64 start, sc_dt::uint64 end)
197 {
198 std::lock_guard<std::mutex> lock(m_dma_dmi_lock);
199 if (m_dma_dmi.ptr != nullptr && m_dma_dmi.end >= start && m_dma_dmi.start <= end) m_dma_dmi = DmaDmi{};
200 }
201
202 typedef struct PageAttr {
203 uint64_t pa;
204 unsigned int block : 1;
205 unsigned int rd : 1;
206 unsigned int wr : 1;
207 unsigned int ns : 1;
208 } PageAttr;
209
210 // Request attributes propagated into the translation walk and fault syndrome.
211 struct FaultAttrs {
212 bool privileged = true; // Privileged rather than unprivileged access.
213 bool instruction = false; // Instruction fetch rather than data access.
214 bool non_secure = true; // Non-secure rather than secure transaction.
215 bool address_translation = false; // Address-translation operation; reports FSYNR0.ATOF on a fault.
216 bool asynchronous = false; // Asynchronous fault delivery.
217 bool suppress_fault = false; // Probe only; do not update architectural fault registers.
218 uint8_t mid = 0; // Master ID, reported in FSYNR1.MID.
219 uint8_t pid = 0; // Process ID, reported in FSYNR1.PID.
220 uint8_t bid = 0; // Bus ID, reported in FSYNR1.BID.
221 };
222
223 typedef struct TransReq {
224 uint64_t va;
225 uint64_t tcr[3];
226 uint64_t ttbr[3][2];
227 uint32_t access;
228 unsigned int stage;
229 bool s2_enabled;
230 unsigned int s2_cb;
231 uint64_t pa;
232 uint32_t prot;
233 uint64_t page_size;
234 bool err;
235 FaultKind fault = FaultKind::Translation;
236 bool walk_fault = false;
237 bool privileged = true;
238 bool instruction = false;
239 bool non_secure = true;
240 bool address_translation = false;
241 bool asynchronous = false;
242 bool nested = false;
243 bool internal_walk = false;
244 bool suppress_fault = false;
245 bool ipafar_valid = false;
246 uint64_t ipafar = 0;
247 unsigned int s1_cb = 0;
248 uint8_t mid = 0;
249 uint8_t pid = 0;
250 uint8_t bid = 0;
251 uint64_t trace_id = 0;
252 uint64_t sid = 0;
253 } TransReq;
254
255 static uint32_t extract32(uint32_t val, int start, int length) { return (val >> start) & ((1u << length) - 1); }
256
257 static uint64_t extract64(uint64_t val, int start, int length) { return (val >> start) & ((1ULL << length) - 1); }
258
259 static int clz32(uint32_t val) { return val ? __builtin_clz(val) : 32; }
260
261 void smmu500_update_ctx_irq(unsigned int cb)
262 {
263 bool fault = (static_cast<uint32_t>(SMMU_CB_FSR[cb]) & CB_FSR_FAULT_MASK) != 0;
264 bool ie = SMMU_CB_SCTLR[cb][CB_SCTLR_CFIE];
265 if (irq_context[cb].size() > 0) irq_context[cb]->write(fault && ie);
266 }
267
268 uint32_t smmu500_global_control(bool non_secure)
269 {
270 return non_secure ? static_cast<uint32_t>(SMMU_NSCR0) : static_cast<uint32_t>(SMMU_SCR0);
271 }
272
273private:
274 void smmu500_update_global_irq()
275 {
276 const bool secure_pending = (static_cast<uint32_t>(SMMU_SGFSR) & GLOBAL_FAULT_MASK) != 0;
277 const bool non_secure_pending = (static_cast<uint32_t>(SMMU_NSGFSR) & GLOBAL_FAULT_MASK) != 0;
278 const bool secure_irq = secure_pending && (smmu500_global_control(false) & SCR0_GFIE_MASK);
279 const bool non_secure_irq = non_secure_pending && (smmu500_global_control(true) & SCR0_GFIE_MASK);
280 if (irq_global.size() > 0) irq_global->write(secure_irq || non_secure_irq);
281 }
282
283 void smmu500_record_global_fault(uint32_t fault_mask, bool non_secure)
284 {
285 if (non_secure) {
286 SMMU_NSGFSR = static_cast<uint32_t>(SMMU_NSGFSR) | fault_mask;
287 } else {
288 SMMU_SGFSR = static_cast<uint32_t>(SMMU_SGFSR) | fault_mask;
289 }
290 smmu500_update_global_irq();
291 }
292
293 void smmu500_report_global_fault(IOMMUTLBEntry& ret, uint32_t fault_mask, bool non_secure, uint32_t global_control,
294 bool record_fault = true)
295 {
296 if (record_fault) smmu500_record_global_fault(fault_mask, non_secure);
297 if (global_control & SCR0_GFRE_MASK) {
298 ret.perm = IOMMU_NONE;
299 } else {
300 ret.addr_mask = -1;
301 }
302 }
303
304 FaultAttrs smmu500_extract_fault_attrs(tlm::tlm_generic_payload& txn, bool record_fault) const
305 {
306 FaultAttrs attrs;
307 attrs.suppress_fault = !record_fault;
308 if (const auto* ext = txn.get_extension<smmu500_transaction_attrs_extension>()) {
309 attrs.privileged = ext->privileged;
310 attrs.instruction = ext->instruction;
311 attrs.non_secure = ext->non_secure;
312 attrs.address_translation = ext->address_translation;
313 attrs.asynchronous = ext->asynchronous;
314 attrs.mid = ext->mid;
315 attrs.pid = ext->pid;
316 attrs.bid = ext->bid;
317 }
318 if (const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>()) {
319 attrs.non_secure = !secure->value;
320 }
321 return attrs;
322 }
323
324 static bool smmu500_range_within(uint64_t address, uint64_t length, uint64_t start, uint64_t size)
325 {
326 return address >= start && length <= size && address - start <= size - length;
327 }
328
329 struct RegisterRange {
330 uint64_t offset;
331 uint64_t size;
332 };
333
334 template <size_t N>
335 static bool smmu500_range_within_any(uint64_t address, uint64_t length, const std::array<RegisterRange, N>& ranges)
336 {
337 return std::any_of(ranges.cbegin(), ranges.cend(), [address, length](const RegisterRange& range) {
338 return smmu500_range_within(address, length, range.offset, range.size);
339 });
340 }
341
342 template <size_t N>
343 static bool smmu500_offset_in(uint64_t address, const std::array<uint64_t, N>& offsets)
344 {
345 return std::find(offsets.cbegin(), offsets.cend(), address) != offsets.cend();
346 }
347
348 enum class RegisterAccessTarget { Model, Shadow, IdRegister, Denied };
349
350 struct RegisterAccess {
351 RegisterAccessTarget target = RegisterAccessTarget::Model;
352 uint64_t address = 0;
353 bool non_secure = false;
354 };
355
356 bool smmu500_is_secure_only_gr0_register(uint64_t address, uint64_t length)
357 {
358 const std::array<RegisterRange, 5> secure_only_ranges = { {
359 { SMMU_SCR1.get_offset(), sizeof(uint32_t) },
360 { SMMU_STLBIALL.get_offset(), sizeof(uint32_t) },
361 { SMMU_STLBIVALM_LOW.get_offset(), sizeof(uint64_t) },
362 { SMMU_STLBIVAM_LOW.get_offset(), sizeof(uint64_t) },
363 { SMMU_STLBIALLM.get_offset(), sizeof(uint32_t) },
364 } };
365 return smmu500_range_within_any(address, length, secure_only_ranges);
366 }
367
368 bool smmu500_is_non_secure_alias(uint64_t address, uint64_t length)
369 {
370 const std::array<RegisterRange, 7> non_secure_alias_ranges = { {
371 { SMMU_SCR0.get_offset(), sizeof(uint32_t) },
372 { SMMU_SCR2_OFFSET, sizeof(uint32_t) },
373 { SMMU_SACR.get_offset(), sizeof(uint32_t) },
374 { SMMU_GFSR_OFFSET, SMMU_GFSR_SIZE },
375 { SMMU_TLBIVAH_RANGE_OFFSET, SMMU_TLBIVAH_RANGE_SIZE },
376 { SMMU_GATS1UR.get_offset(), SMMU_GATS12PW_H.get_offset() - SMMU_GATS1UR.get_offset() + sizeof(uint32_t) },
377 { SMMU_GPAR.get_offset(), SMMU_GPAR_RANGE_SIZE },
378 } };
379 return smmu500_range_within_any(address, length, non_secure_alias_ranges);
380 }
381
382 bool smmu500_is_banked_gr0_register(uint64_t address, uint64_t length)
383 {
384 const std::array<RegisterRange, 10> banked_ranges = { {
385 { SMMU_SCR0.get_offset(), sizeof(uint32_t) },
386 { SMMU_SCR2_OFFSET, sizeof(uint32_t) },
387 { SMMU_SACR.get_offset(), sizeof(uint32_t) },
388 { SMMU_GFSR_OFFSET, SMMU_GFSR_SIZE },
389 { SMMU_TLBIVMID.get_offset(),
390 SMMU_TLBIVAH_LOW.get_offset() - SMMU_TLBIVMID.get_offset() + sizeof(uint32_t) },
391 { SMMU_TLBIVALH64_LOW.get_offset(), sizeof(uint64_t) },
392 { SMMU_TLBIVMIDS1.get_offset(), sizeof(uint32_t) },
393 { SMMU_TLBIVAH64_LOW.get_offset(), sizeof(uint64_t) },
394 { SMMU_GATS1UR.get_offset(), SMMU_GATS12PW_H.get_offset() - SMMU_GATS1UR.get_offset() + sizeof(uint32_t) },
395 { SMMU_GPAR.get_offset(), SMMU_GPAR_RANGE_SIZE },
396 } };
397 return smmu500_range_within_any(address, length, banked_ranges);
398 }
399
400 bool smmu500_is_modelled_banked_gr0_register(uint64_t address, uint64_t length)
401 {
402 const std::array<RegisterRange, 10> modelled_banked_ranges = { {
403 { SMMU_SCR0.get_offset(), sizeof(uint32_t) },
404 { SMMU_SACR.get_offset(), sizeof(uint32_t) },
405 { SMMU_SGFSR.get_offset(), sizeof(uint32_t) },
406 { SMMU_TLBIVMID.get_offset(), SMMU_TLBIALLH.get_offset() - SMMU_TLBIVMID.get_offset() + sizeof(uint32_t) },
407 { SMMU_TLBIVAH_LOW.get_offset(), sizeof(uint32_t) },
408 { SMMU_TLBIVALH64_LOW.get_offset(), sizeof(uint64_t) },
409 { SMMU_TLBIVMIDS1.get_offset(), sizeof(uint32_t) },
410 { SMMU_TLBIVAH64_LOW.get_offset(), sizeof(uint64_t) },
411 { SMMU_GATS1UR.get_offset(), SMMU_GATS12PW_H.get_offset() - SMMU_GATS1UR.get_offset() + sizeof(uint32_t) },
412 { SMMU_GPAR.get_offset(), sizeof(uint64_t) },
413 } };
414 return smmu500_range_within_any(address, length, modelled_banked_ranges);
415 }
416
417 bool smmu500_is_shadowed_gr0_register(uint64_t address, uint64_t length, bool non_secure)
418 {
419 if (!smmu500_is_banked_gr0_register(address, length)) return false;
420 if (non_secure) return address != SMMU_SCR0.get_offset() && address != SMMU_SGFSR.get_offset();
421 return !smmu500_is_modelled_banked_gr0_register(address, length);
422 }
423
424 RegisterAccess smmu500_resolve_register_access(const tlm::tlm_generic_payload& txn)
425 {
426 const uint64_t address = txn.get_address();
427 const uint64_t length = std::max<uint64_t>(txn.get_data_length(), 1);
428 RegisterAccess access{ RegisterAccessTarget::Model, address, false };
429 const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
430 if (!secure) return access;
431
432 const bool non_secure = !secure->value;
433 const uint64_t cb_base = static_cast<uint64_t>(p_num_pages) * SMMU_PAGESIZE;
434 if (non_secure && (static_cast<uint32_t>(SMMU_SCR1) & (1u << 24)) && address < cb_base) {
435 access.target = RegisterAccessTarget::Denied;
436 return access;
437 }
438 const bool alias = smmu500_range_within(address, length, NS_GLOBAL_REG_START,
439 NS_GLOBAL_REG_END - NS_GLOBAL_REG_START);
440 const uint64_t primary = alias ? address - NS_GLOBAL_REG_START : address;
441
442 if (alias) {
443 if (!smmu500_is_non_secure_alias(primary, length)) {
444 access.target = RegisterAccessTarget::Denied;
445 } else if (smmu500_is_shadowed_gr0_register(primary, length, true)) {
446 access = { RegisterAccessTarget::Shadow, primary, true };
447 } else if (primary == SMMU_SCR0.get_offset() || primary == SMMU_SGFSR.get_offset()) {
448 access.address = address;
449 } else {
450 access.address = primary;
451 }
452 return access;
453 }
454
455 if (!non_secure) {
456 if (smmu500_is_shadowed_gr0_register(address, length, false)) {
457 access = { RegisterAccessTarget::Shadow, address, false };
458 }
459 return access;
460 }
461
462 if (smmu500_range_within(address, length, 0, NS_GLOBAL_REG_START)) {
463 if (smmu500_is_secure_only_gr0_register(address, length)) {
464 access.target = RegisterAccessTarget::Denied;
465 } else if (address == SMMU_SIDR0.get_offset() || address == SMMU_SIDR1.get_offset()) {
466 access.target = RegisterAccessTarget::IdRegister;
467 } else if (smmu500_is_shadowed_gr0_register(address, length, true)) {
468 access = { RegisterAccessTarget::Shadow, address, true };
469 } else if (address == SMMU_SCR0.get_offset() || address == SMMU_SGFSR.get_offset()) {
470 access.address = address + NS_GLOBAL_REG_START;
471 }
472 return access;
473 }
474
475 const uint64_t num_ns_smr = std::min<uint64_t>(static_cast<uint32_t>(SCR1_NSNUMSMRGO), p_num_smr);
476 const uint64_t num_ns_cb = std::min<uint64_t>(static_cast<uint32_t>(SCR1_NSNUMCBO), p_num_cb);
477 const bool permitted = smmu500_range_within(address, length, SMR_BASE, num_ns_smr * sizeof(uint32_t)) ||
478 smmu500_range_within(address, length, S2CR_BASE, num_ns_smr * sizeof(uint32_t)) ||
479 smmu500_range_within(address, length, CBAR_BASE, num_ns_cb * sizeof(uint32_t)) ||
480 smmu500_range_within(address, length, CBA2R_BASE, num_ns_cb * sizeof(uint32_t)) ||
481 smmu500_range_within(address, length, TBU_PWR_STATUS, sizeof(uint32_t)) ||
482 smmu500_range_within(address, length, cb_base, num_ns_cb * SMMU_PAGESIZE);
483 const bool context_access = address >= cb_base;
484 const uint64_t cb = context_access ? (address - cb_base) / SMMU_PAGESIZE : 0;
485 const bool restricted_stage2_cb = context_access && cb < p_num_cb &&
486 (static_cast<uint32_t>(SMMU_SCR1) & (1u << 24)) &&
487 static_cast<uint32_t>(SMMU_CBAR[cb][CBAR_TYPE]) == 0;
488 if (!permitted || restricted_stage2_cb) access.target = RegisterAccessTarget::Denied;
489 return access;
490 }
491
492 bool smmu500_access_gr0_shadow(const RegisterAccess& access, tlm::tlm_generic_payload& txn)
493 {
494 auto* data = txn.get_data_ptr();
495 const uint64_t length = txn.get_data_length();
496 if (!data || !length || access.address + length > m_secure_gr0_shadow.size()) {
497 txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE);
498 return false;
499 }
500 auto& state = access.non_secure ? m_non_secure_gr0_shadow : m_secure_gr0_shadow;
501 if (txn.is_read()) {
502 std::memcpy(data, state.data() + access.address, length);
503 } else if (txn.is_write()) {
504 std::memcpy(state.data() + access.address, data, length);
505 if (access.non_secure) smmu500_handle_non_secure_gr0_write(access.address);
506 } else {
507 txn.set_response_status(tlm::TLM_COMMAND_ERROR_RESPONSE);
508 return false;
509 }
510 txn.set_dmi_allowed(false);
511 txn.set_response_status(tlm::TLM_OK_RESPONSE);
512 return true;
513 }
514
515 bool smmu500_access_non_secure_id_register(const RegisterAccess& access, tlm::tlm_generic_payload& txn)
516 {
517 if (txn.get_data_length() != sizeof(uint32_t) || !txn.get_data_ptr()) return false;
518 if (txn.is_write()) {
519 txn.set_dmi_allowed(false);
520 txn.set_response_status(tlm::TLM_OK_RESPONSE);
521 return true;
522 }
523 if (!txn.is_read()) return false;
524 uint32_t value = access.address == SMMU_SIDR0.get_offset() ? static_cast<uint32_t>(SMMU_SIDR0)
525 : static_cast<uint32_t>(SMMU_SIDR1);
526 const uint32_t count = access.address == SMMU_SIDR0.get_offset() ? static_cast<uint32_t>(SCR1_NSNUMSMRGO)
527 : static_cast<uint32_t>(SCR1_NSNUMCBO);
528 value = (value & ~0xffu) | count;
529 std::memcpy(txn.get_data_ptr(), &value, sizeof(value));
530 txn.set_dmi_allowed(false);
531 txn.set_response_status(tlm::TLM_OK_RESPONSE);
532 return true;
533 }
534
535 uint32_t smmu500_non_secure_gr0_read32(uint64_t address) const
536 {
537 uint32_t value = 0;
538 std::memcpy(&value, m_non_secure_gr0_shadow.data() + address, sizeof(value));
539 return value;
540 }
541
542 void smmu500_non_secure_gr0_write32(uint64_t address, uint32_t value)
543 {
544 std::memcpy(m_non_secure_gr0_shadow.data() + address, &value, sizeof(value));
545 }
546
547 void smmu500_handle_non_secure_gr0_write(uint64_t address)
548 {
549 struct GatCommand {
550 uint64_t high_offset;
551 bool write;
552 bool stage2;
553 bool unprivileged;
554 };
555 const std::array<GatCommand, 8> gat_commands = { {
556 { SMMU_GATS1UR_H.get_offset(), false, false, true },
557 { SMMU_GATS1UW_H.get_offset(), true, false, true },
558 { SMMU_GATS1PR_H.get_offset(), false, false, false },
559 { SMMU_GATS1PW_H.get_offset(), true, false, false },
560 { SMMU_GATS12UR_H.get_offset(), false, true, true },
561 { SMMU_GATS12UW_H.get_offset(), true, true, true },
562 { SMMU_GATS12PR_H.get_offset(), false, true, false },
563 { SMMU_GATS12PW_H.get_offset(), true, true, false },
564 } };
565 const std::array<uint64_t, 2> vmid_invalidate_offsets = { {
566 SMMU_TLBIVMID.get_offset(),
567 SMMU_TLBIVMIDS1.get_offset(),
568 } };
569 const std::array<uint64_t, 7> all_invalidate_offsets = { {
570 SMMU_TLBIALLNSNH.get_offset(),
571 SMMU_TLBIALLH.get_offset(),
572 SMMU_TLBIVAH_LOW.get_offset(),
573 SMMU_TLBIVALH64_LOW.get_offset(),
574 SMMU_TLBIVALH64_HIGH.get_offset(),
575 SMMU_TLBIVAH64_LOW.get_offset(),
576 SMMU_TLBIVAH64_HIGH.get_offset(),
577 } };
578 const auto gat = [this, address](bool write, bool stage2, bool unprivileged) {
579 const uint64_t value = (static_cast<uint64_t>(smmu500_non_secure_gr0_read32(address)) << 32) |
580 smmu500_non_secure_gr0_read32(address - sizeof(uint32_t));
581 smmu500_gat(value, write, stage2, unprivileged, true);
582 };
583
584 const auto gat_command = std::find_if(
585 gat_commands.cbegin(), gat_commands.cend(),
586 [address](const GatCommand& command) { return command.high_offset == address; });
587 if (gat_command != gat_commands.cend()) {
588 gat(gat_command->write, gat_command->stage2, gat_command->unprivileged);
589 return;
590 }
591 if (smmu500_offset_in(address, vmid_invalidate_offsets)) {
592 smmu500_invalidate_vmid(smmu500_non_secure_gr0_read32(address) & 0xffu);
593 } else if (smmu500_offset_in(address, all_invalidate_offsets)) {
594 smmu500_invalidate_all();
595 } else {
596 return;
597 }
598 std::fill(m_non_secure_gr0_shadow.begin() + SMMU_TLBIVMID.get_offset(),
599 m_non_secure_gr0_shadow.begin() + SMMU_TLBIVAH64_HIGH.get_offset() + sizeof(uint32_t), 0);
600 }
601
602 void smmu500_register_b_transport(int, tlm::tlm_generic_payload& txn, sc_core::sc_time& delay)
603 {
604 const RegisterAccess access = smmu500_resolve_register_access(txn);
605 if (access.target == RegisterAccessTarget::Denied) {
606 const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
607 SCP_WARN(()) << std::hex << "SMMU denied "
608 << (txn.is_read() ? "read"
609 : txn.is_write() ? "write"
610 : "command")
611 << " at 0x" << txn.get_address() << " length 0x" << txn.get_data_length() << " from "
612 << (secure && !secure->value ? "non-secure" : "secure") << " context";
613 txn.set_dmi_allowed(false);
614 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
615 return;
616 }
617 const auto* secure = txn.get_extension<QemuMemTxSecureTlmExtension>();
618 if (secure && secure->value && txn.get_address() >= NS_GLOBAL_REG_START &&
619 txn.get_address() < NS_GLOBAL_REG_END) {
620 SCP_DEBUG(()) << std::hex << "SMMU secure "
621 << (txn.is_read() ? "read"
622 : txn.is_write() ? "write"
623 : "command")
624 << " of non-secure alias at 0x" << txn.get_address() << " length 0x" << txn.get_data_length();
625 }
626 if (access.target == RegisterAccessTarget::Shadow) {
627 smmu500_access_gr0_shadow(access, txn);
628 return;
629 }
630 if (access.target == RegisterAccessTarget::IdRegister) {
631 if (!smmu500_access_non_secure_id_register(access, txn)) {
632 txn.set_response_status(tlm::TLM_GENERIC_ERROR_RESPONSE);
633 }
634 return;
635 }
636 const uint64_t address = txn.get_address();
637 txn.set_address(access.address);
638 m_register_socket->b_transport(txn, delay);
639 txn.set_address(address);
640 txn.set_dmi_allowed(false);
641 }
642
643 unsigned int smmu500_register_transport_dbg(int, tlm::tlm_generic_payload& txn)
644 {
645 const RegisterAccess access = smmu500_resolve_register_access(txn);
646 if (access.target == RegisterAccessTarget::Denied) return 0;
647 if (access.target == RegisterAccessTarget::Shadow) {
648 return smmu500_access_gr0_shadow(access, txn) ? txn.get_data_length() : 0;
649 }
650 if (access.target == RegisterAccessTarget::IdRegister) {
651 return smmu500_access_non_secure_id_register(access, txn) ? txn.get_data_length() : 0;
652 }
653 const uint64_t address = txn.get_address();
654 txn.set_address(access.address);
655 const unsigned int transferred = m_register_socket->transport_dbg(txn);
656 txn.set_address(address);
657 return transferred;
658 }
659
660 bool smmu500_register_get_direct_mem_ptr(int, tlm::tlm_generic_payload& txn, tlm::tlm_dmi& dmi_data)
661 {
662 /* Register behavior depends on the transaction security state and
663 * several writes have side effects. A DMI mapping obtained for one
664 * security state must never service a later access in another. */
665 txn.set_dmi_allowed(false);
666 return false;
667 }
668
669 void smmu500_program_id_registers()
670 {
671 const unsigned int num_pages_log2 = 31 - clz32(p_num_pages);
672 SIDR0_ATOSNS = static_cast<uint32_t>(p_ato);
673 SIDR0_NUMSMRG = std::min<uint32_t>(p_num_smr, SMMU_MAX_SMR);
674 SIDR1_NUMCB = std::min<uint32_t>(p_num_cb, SMMU_MAX_CB);
675 SIDR1_NUMPAGENDXB = num_pages_log2 - 1;
676 SMMU_SIDR2 = static_cast<uint32_t>(p_idr2);
677 SMMU_SIDR7 = static_cast<uint32_t>(p_version);
678 }
679
680 void smmu500_invalidate_cb(unsigned int cb)
681 {
682 if (cb >= std::min<unsigned int>(p_num_cb, SMMU_MAX_CB)) return;
683 for (auto tbu : tbus) tbu->invalidate(cb);
684 }
685
686 void smmu500_invalidate_all()
687 {
688 for (auto tbu : tbus) tbu->reset_dmi();
689 }
690
691 void smmu500_invalidate_vmid(uint32_t vmid)
692 {
693 const unsigned int nr_cb = std::min<unsigned int>(p_num_cb, SMMU_MAX_CB);
694 for (unsigned int cb = 0; cb < nr_cb; ++cb) {
695 if (static_cast<uint32_t>(SMMU_CBAR[cb][CBAR_VMID]) == (vmid & 0xffu)) {
696 smmu500_invalidate_cb(cb);
697 }
698 }
699 }
700
701 void smmu500_clear_global_tlbi_regs()
702 {
703 SMMU_STLBIALL = 0;
704 SMMU_TLBIVMID = 0;
705 SMMU_TLBIALLNSNH = 0;
706 SMMU_TLBIALLH = 0;
707 SMMU_TLBIVAH_LOW = 0;
708 SMMU_STLBIVALM_LOW = 0;
709 SMMU_STLBIVALM_HIGH = 0;
710 SMMU_STLBIVAM_LOW = 0;
711 SMMU_STLBIVAM_HIGH = 0;
712 SMMU_TLBIVALH64_LOW = 0;
713 SMMU_TLBIVALH64_HIGH = 0;
714 SMMU_TLBIVMIDS1 = 0;
715 SMMU_STLBIALLM = 0;
716 SMMU_TLBIVAH64_LOW = 0;
717 SMMU_TLBIVAH64_HIGH = 0;
718 }
719
720 void smmu500_clear_cb_tlbi_regs(unsigned int cb)
721 {
722 if (cb >= SMMU_MAX_CB) return;
723 SMMU_CB_TLBIVA_LOW[cb] = 0;
724 SMMU_CB_TLBIVA_HIGH[cb] = 0;
725 SMMU_CB_TLBIVAA_LOW[cb] = 0;
726 SMMU_CB_TLBIVAA_HIGH[cb] = 0;
727 SMMU_CB_TLBIASID[cb] = 0;
728 SMMU_CB_TLBIALL[cb] = 0;
729 SMMU_CB_TLBIVAL_LOW[cb] = 0;
730 SMMU_CB_TLBIVAL_HIGH[cb] = 0;
731 SMMU_CB_TLBIVAAL_LOW[cb] = 0;
732 SMMU_CB_TLBIVAAL_HIGH[cb] = 0;
733 SMMU_CB_TLBIIPAS2_LOW[cb] = 0;
734 SMMU_CB_TLBIIPAS2_HIGH[cb] = 0;
735 SMMU_CB_TLBIIPAS2L_LOW[cb] = 0;
736 SMMU_CB_TLBIIPAS2L_HIGH[cb] = 0;
737 }
738
739 static uint32_t smmu500_fault_status(FaultKind kind)
740 {
741 switch (kind) {
742 case FaultKind::Translation:
743 return CB_FSR_TF_MASK;
744 case FaultKind::AccessFlag:
745 return CB_FSR_AFF_MASK;
746 case FaultKind::Permission:
747 return CB_FSR_PF_MASK;
748 case FaultKind::External:
749 return CB_FSR_EF_MASK;
750 case FaultKind::AddressSize:
751 return CB_FSR_ASF_MASK;
752 }
753 return CB_FSR_TF_MASK;
754 }
755
756 static const char* smmu500_fault_name(FaultKind kind)
757 {
758 switch (kind) {
759 case FaultKind::Translation:
760 return "translation";
761 case FaultKind::AccessFlag:
762 return "access flag";
763 case FaultKind::Permission:
764 return "permission";
765 case FaultKind::External:
766 return "external";
767 case FaultKind::AddressSize:
768 return "address size";
769 }
770 return "unknown";
771 }
772
773 void smmu500_fault(unsigned int cb, TransReq* req, int level)
774 {
775 const uint32_t old_fsr = static_cast<uint32_t>(SMMU_CB_FSR[cb]);
776 const bool multiple = (old_fsr & CB_FSR_FAULT_MASK) != 0;
777 uint32_t new_fsr = old_fsr | smmu500_fault_status(req->fault);
778 if (multiple) new_fsr |= CB_FSR_MULTI_MASK;
779 SMMU_CB_FSR[cb] = new_fsr;
780 req->err = true;
781 if (!multiple) {
782 SCP_WARN(()) << "SMMU context fault: CB " << std::dec << cb << " SID 0x" << std::hex << req->sid << " VA 0x"
783 << req->va << " stage " << std::dec << req->stage << " level " << level << " reason "
784 << smmu500_fault_name(req->fault) << (req->walk_fault ? " during page-table walk" : "");
785 const uint64_t ipafar = req->ipafar_valid ? req->ipafar : req->va;
786 SMMU_CB_IPAFAR_LOW[cb] = static_cast<uint32_t>(ipafar);
787 SMMU_CB_IPAFAR_HIGH[cb] = static_cast<uint32_t>(ipafar >> 32);
788 if (req->stage == 2) {
789 SMMU_CB_FAR_LOW[cb] = static_cast<uint32_t>(req->va);
790 SMMU_CB_FAR_HIGH[cb] = static_cast<uint32_t>(req->va >> 32);
791 }
792
793 uint32_t syn = static_cast<uint32_t>(level) & 0x3;
794 syn |= (req->access == IOMMU_WO) ? (1u << 4) : 0;
795 syn |= req->privileged ? (1u << 5) : 0;
796 syn |= req->instruction ? (1u << 6) : 0;
797 syn |= req->non_secure ? (1u << 8) : 0;
798 syn |= req->address_translation ? (1u << 9) : 0;
799 syn |= req->walk_fault ? (1u << 10) : 0;
800 syn |= req->asynchronous ? (1u << 11) : 0;
801 if (req->stage == 1 || req->nested) {
802 const unsigned int s1_cb = req->nested ? req->s1_cb : cb;
803 syn |= (s1_cb & 0x7Fu) << 16;
804 }
805 SMMU_CB_FSYNR0[cb] = syn;
806 SMMU_CB_FSYNR1[cb] = req->mid | (static_cast<uint32_t>(req->pid) << 8) |
807 (static_cast<uint32_t>(req->bid & 0x7u) << 13);
808 }
809 smmu500_update_ctx_irq(cb);
810 }
811
812 bool check_s2_startlevel(bool is_aa64, unsigned int pamax_val, int level, int inputsize, int stride)
813 {
814 if (level < 0) return false;
815 if (is_aa64) {
816 switch (stride) {
817 case 13:
818 if (level == 0 || (level == 1 && pamax_val <= 42)) return false;
819 break;
820 case 11:
821 if (level == 0 || (level == 1 && pamax_val <= 40)) return false;
822 break;
823 case 9:
824 if (level == 0 && pamax_val <= 42) return false;
825 break;
826 default:
827 assert(false);
828 }
829 } else {
830 const int grainsize = stride + 3;
831 assert(stride == 9);
832 if (level == 0) return false;
833 int startsizecheck = inputsize - ((3 - level) * stride + grainsize);
834 if (startsizecheck < 1 || startsizecheck > stride + 4) return false;
835 }
836 return true;
837 }
838
839 bool check_out_addr(uint64_t addr, unsigned int outputsize)
840 {
841 if (outputsize != 48 && extract64(addr, outputsize, 48 - outputsize)) return false;
842 return true;
843 }
844
845 void dump_trans_req(TransReq const& tr)
846 {
847 SCP_DEBUG(()) << "Translation Req\n"
848 << std::hex << "VA: 0x" << tr.va << "\n"
849 << "TCR[0]: 0x" << tr.tcr[0] << "\n"
850 << "TCR[1]: 0x" << tr.tcr[1] << "\n"
851 << "TCR[2]: 0x" << tr.tcr[2] << "\n"
852 << "ACCESS: 0x" << tr.access << "\n"
853 << "Stage: " << std::dec << tr.stage << "\n"
854 << "S2_enabled: " << (tr.s2_enabled ? "true" : "false") << "\n"
855 << "S2_CB: " << tr.s2_cb << "\n"
856 << std::hex << "PA: 0x" << tr.pa << "\n"
857 << "Prot: 0x" << tr.prot << "\n"
858 << "Page size: 0x" << tr.page_size << "\n"
859 << "Error: " << (tr.err ? "true" : "false") << "\n";
860 }
861
862 void dump_cb_state(unsigned int cb)
863 {
864 SCP_DEBUG(()) << "CB" << cb << ":\n"
865 << std::hex << "CB_SCTLR = 0x" << static_cast<uint32_t>(SMMU_CB_SCTLR[cb]) << "\n"
866 << "CB_TCR2 = 0x" << static_cast<uint32_t>(SMMU_CB_TCR2[cb]) << "\n"
867 << "CB_TTBR0_LOW = 0x" << static_cast<uint32_t>(SMMU_CB_TTBR0_LOW[cb]) << "\n"
868 << "CB_TTBR0_HIGH = 0x" << static_cast<uint32_t>(SMMU_CB_TTBR0_HIGH[cb]) << "\n"
869 << "CB_TTBR1_LOW = 0x" << static_cast<uint32_t>(SMMU_CB_TTBR1_LOW[cb]) << "\n"
870 << "CB_TTBR1_HIGH = 0x" << static_cast<uint32_t>(SMMU_CB_TTBR1_HIGH[cb]) << "\n"
871 << "CB_TCR_LPAE = 0x" << static_cast<uint32_t>(SMMU_CB_TCR_LPAE[cb]) << "\n"
872 << "CB_FSR = 0x" << static_cast<uint32_t>(SMMU_CB_FSR[cb]) << "\n"
873 << "CB_FAR_LOW = 0x" << static_cast<uint32_t>(SMMU_CB_FAR_LOW[cb]) << "\n"
874 << "CB_FAR_HIGH = 0x" << static_cast<uint32_t>(SMMU_CB_FAR_HIGH[cb]) << "\n"
875 << "CB_IPAFAR_LOW = 0x" << static_cast<uint32_t>(SMMU_CB_IPAFAR_LOW[cb]) << "\n"
876 << "CB_IPAFAR_HIGH = 0x" << static_cast<uint32_t>(SMMU_CB_IPAFAR_HIGH[cb]) << "\n";
877 }
878
879 void dump_state()
880 {
881 SCP_DEBUG(()) << "smmu regs:" << std::hex << "SMMU_SCR0 = 0x" << static_cast<uint32_t>(SMMU_SCR0) << "\n"
882 << "SMMU_SCR1 = 0x" << static_cast<uint32_t>(SMMU_SCR1) << "\n"
883 << "SMMU_SACR = 0x" << static_cast<uint32_t>(SMMU_SACR) << "\n"
884 << "SMMU_SIDR0 = 0x" << static_cast<uint32_t>(SMMU_SIDR0) << "\n"
885 << "SMMU_SIDR1 = 0x" << static_cast<uint32_t>(SMMU_SIDR1) << "\n"
886 << "SMMU_SIDR2 = 0x" << static_cast<uint32_t>(SMMU_SIDR2) << "\n"
887 << "SMMU_SIDR7 = 0x" << static_cast<uint32_t>(SMMU_SIDR7) << "\n"
888 << "SMMU_NSCR0 = 0x" << static_cast<uint32_t>(SMMU_NSCR0) << "\n";
889 }
890
891 void smmu500_ptw64(unsigned int cb, TransReq* req)
892 {
893 const unsigned int outsize_map[] = { 32, 36, 40, 42, 44, 48, 48, 48 };
894 unsigned int tsz;
895 unsigned int t0sz;
896 unsigned int t1sz;
897 unsigned int inputsize;
898 unsigned int outputsize;
899 unsigned int grainsize = -1;
900 unsigned int stride;
901 int level = 0;
902 unsigned int firstblocklevel = 0;
903 unsigned int tg;
904 unsigned int ps;
905 unsigned int baselowerbound;
906 unsigned int stage = req->stage;
907 bool blocktranslate = false;
908 bool epd = false;
909 uint32_t tableattrs = 0;
910 uint32_t attrs;
911 uint32_t s2attrs;
912 uint64_t descmask;
913 uint64_t ttbr;
914 uint64_t desc;
915 uint64_t walk_va;
916 uint64_t last_descaddr = 0;
917 uint64_t last_desc = 0;
918 bool ttbr1 = false;
919 bool have_descriptor = false;
920
921 req->err = false;
922
923 if (SMMU_CB_SCTLR[cb][CB_SCTLR_M] == 0) {
924 req->pa = req->va;
925 req->prot = IOMMU_RW;
926 return;
927 }
928
929 ttbr = req->ttbr[stage][0];
930 tg = extract32(req->tcr[stage], 14, 2);
931 if (stage == 1) {
932 ps = extract64(req->tcr[stage], 32, 3);
933 } else {
934 ps = extract64(req->tcr[stage], 16, 3);
935 }
936 t0sz = extract32(req->tcr[stage], 0, 6);
937 tsz = t0sz;
938 req->pa = req->va;
939
940 if (req->stage == 1) {
941 assert(SMMU_CBA2R[cb][CBA2R_VA64] || extract32(req->tcr[1], 31, 1));
942
943 if ((req->va & (1ULL << 63)) == 0) {
944 /* Use TTBR0. */
945 } else {
946 ttbr1 = true;
947 const unsigned int tg1map[] = { 3, 2, 0, 1 };
948 tg = tg1map[extract32(req->tcr[stage], 30, 2)];
949 t1sz = extract32(req->tcr[stage], 16, 6);
950 ttbr = req->ttbr[stage][1];
951 tsz = t1sz;
952 }
953 epd = extract32(req->tcr[1], ttbr1 ? 23 : 7, 1);
954 }
955
956 if (epd) goto do_fault;
957
958 inputsize = 64 - tsz;
959 if (req->stage == 1) {
960 if (inputsize < 25 || inputsize > 48) {
961 req->fault = FaultKind::AddressSize;
962 goto do_fault;
963 }
964 const uint64_t upper = extract64(req->va, inputsize, 64 - inputsize);
965 const uint64_t expected = ttbr1 ? ((1ULL << (64 - inputsize)) - 1) : 0;
966 if (upper != expected) {
967 req->fault = FaultKind::AddressSize;
968 goto do_fault;
969 }
970 } else if (inputsize < 64 && (req->va >> inputsize) != 0) {
971 /* Stage-2 input addresses are IPAs, not sign-extended VAs. Do
972 * not let the walk_va mask below turn an oversized IPA into a
973 * translation of an unrelated low address. */
974 req->fault = FaultKind::AddressSize;
975 goto do_fault;
976 }
977 walk_va = (inputsize >= 64) ? req->va : (req->va & ((1ULL << inputsize) - 1));
978 switch (tg) {
979 case 1:
980 grainsize = 16;
981 level = 3;
982 firstblocklevel = 2;
983 break;
984 case 2:
985 grainsize = 14;
986 level = 3;
987 firstblocklevel = 2;
988 break;
989 case 0:
990 grainsize = 12;
991 level = 2;
992 firstblocklevel = 1;
993 break;
994 default:
995 SCP_ERR(()) << "Wrong pagesize";
996 goto do_fault;
997 }
998
999 outputsize = outsize_map[ps];
1000 if (outputsize > p_pamax) outputsize = p_pamax;
1001
1002 stride = grainsize - 3;
1003 if (req->stage == 1) {
1004 if (grainsize < 16 && (inputsize > (grainsize + 3 * stride)))
1005 level = 0;
1006 else if (inputsize > (grainsize + 2 * stride))
1007 level = 1;
1008 else if (inputsize > (grainsize + stride))
1009 level = 2;
1010
1011 if (inputsize < 25 || inputsize > 48) {
1012 req->fault = FaultKind::AddressSize;
1013 goto do_fault;
1014 }
1015 } else {
1016 unsigned int startlevel = extract32(req->tcr[stage], 6, 2);
1017 level = 3 - startlevel;
1018 if (grainsize == 12) level = 2 - startlevel;
1019 if (!check_s2_startlevel(true, outputsize, level, inputsize, stride)) goto do_fault;
1020 }
1021
1022 if (req->stage == 2) {
1023 ttbr &= ~((1ULL << grainsize) - 1);
1024 const unsigned int level_coverage = grainsize + (4 - level) * stride;
1025 if (inputsize > level_coverage) {
1026 const unsigned int concat_bits = inputsize - level_coverage;
1027 const uint64_t concat_index = extract64(walk_va, level_coverage, concat_bits);
1028 ttbr += concat_index << grainsize;
1029 }
1030 } else {
1031 baselowerbound = 3 + inputsize - ((3 - level) * stride + grainsize);
1032 ttbr &= ~((1ULL << baselowerbound) - 1);
1033 }
1034
1035 /* Translation tables live in the SMMU physical address space. PS
1036 * constrains a completed translation, not the address of its tables. */
1037 if (!check_out_addr(ttbr, p_pamax)) {
1038 req->fault = FaultKind::AddressSize;
1039 goto do_fault;
1040 }
1041
1042 descmask = (1ULL << grainsize) - 1;
1043 do {
1044 unsigned int addrselectbottom = (3 - level) * stride + grainsize;
1045 uint64_t index = (walk_va >> (addrselectbottom - 3)) & descmask;
1046 index &= ~7ULL;
1047 uint64_t descaddr = ttbr | index;
1048
1049 if (req->stage == 1 && req->s2_enabled) {
1050 TransReq s2req = *req;
1051 s2req.stage = 2;
1052 s2req.internal_walk = true;
1053 s2req.va = descaddr;
1054 s2req.access = IOMMU_RO;
1055 smmu500_ptw64(s2req.s2_cb, &s2req);
1056 if (s2req.err) {
1057 req->ipafar = descaddr;
1058 req->ipafar_valid = true;
1059 req->fault = s2req.fault;
1060 req->walk_fault = true;
1061 goto do_fault;
1062 }
1063 descaddr = s2req.pa;
1064 }
1065
1066 DmaDmi dma_dmi;
1067 {
1068 std::lock_guard<std::mutex> lock(m_dma_dmi_lock);
1069 dma_dmi = m_dma_dmi;
1070 }
1071 const bool descriptor_cached = dma_dmi.ptr != nullptr && dma_dmi.secure == !req->non_secure &&
1072 descaddr >= dma_dmi.start && descaddr <= dma_dmi.end &&
1073 dma_dmi.end - descaddr >= sizeof(desc) - 1;
1074 if (descriptor_cached) {
1075 std::memcpy(&desc, dma_dmi.ptr + (descaddr - dma_dmi.start), sizeof(desc));
1076 } else if (!m_dma_on_sysc.is_on_sysc()) {
1077 throw DmaDmiMiss{};
1078 }
1079
1080 tlm::tlm_response_status dma_status = tlm::TLM_OK_RESPONSE;
1081 if (!descriptor_cached) {
1082 tlm::tlm_generic_payload txn;
1083 txn.set_command(tlm::TLM_READ_COMMAND);
1084 txn.set_address(descaddr);
1085 txn.set_data_ptr(reinterpret_cast<unsigned char*>(&desc));
1086 txn.set_data_length(sizeof(desc));
1087 txn.set_streaming_width(sizeof(desc));
1088 txn.set_byte_enable_length(0);
1089 txn.set_dmi_allowed(false);
1090 txn.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
1091
1092 QemuMemTxSecureTlmExtension secure_ext;
1093 secure_ext.value = !req->non_secure;
1094 txn.set_extension(&secure_ext);
1095
1096 // A TLM delay is relative to the caller, not an absolute timestamp.
1097 sc_core::sc_time now = sc_core::SC_ZERO_TIME;
1098 dma_socket->b_transport(txn, now);
1099 dma_status = txn.get_response_status();
1100 if (dma_status == tlm::TLM_OK_RESPONSE && txn.is_dmi_allowed()) {
1101 tlm::tlm_dmi dmi;
1102 if (dma_socket->get_direct_mem_ptr(txn, dmi) && dmi.is_read_allowed()) {
1103 const DmaDmi new_dmi{ dmi.get_dmi_ptr(), dmi.get_start_address(), dmi.get_end_address(),
1104 !req->non_secure };
1105 std::lock_guard<std::mutex> lock(m_dma_dmi_lock);
1106 m_dma_dmi = new_dmi;
1107 }
1108 }
1109 txn.clear_extension(&secure_ext);
1110 }
1111
1112 if (dma_status != tlm::TLM_OK_RESPONSE) {
1113 SCP_INFO(()) << std::hex << "Bad DMA response CB " << cb << " SID 0x" << req->sid << " VA 0x" << req->va
1114 << " descaddr 0x" << descaddr << " TTBR 0x" << ttbr << " status " << dma_status;
1115 req->fault = FaultKind::External;
1116 req->walk_fault = true;
1117 goto do_fault;
1118 }
1119
1120 last_descaddr = descaddr;
1121 last_desc = desc;
1122 have_descriptor = true;
1123
1124 if (req->trace_id != 0) {
1125 SCP_INFO(()) << std::hex << "SMMU PTW id " << req->trace_id << " SID 0x" << req->sid << " CB" << cb
1126 << " VA 0x" << req->va << " L" << level << " table 0x" << ttbr << " index 0x" << index
1127 << " descaddr 0x" << descaddr << " desc 0x" << desc;
1128 }
1129
1130 unsigned int type = desc & 3;
1131 ttbr = extract64(desc, 0, 48);
1132 ttbr &= ~descmask;
1133
1134 if (!(type & 2) && level == 3) {
1135 SCP_INFO(()) << "bad level 3 desc";
1136 req->walk_fault = true;
1137 goto do_fault;
1138 }
1139
1140 if (level == 3) break;
1141
1142 switch (type) {
1143 case 2:
1144 case 0:
1145 SCP_INFO(()) << "bad desc id " << req->trace_id << " SID 0x" << std::hex << req->sid << " CB " << cb;
1146 req->walk_fault = true;
1147 goto do_fault;
1148 break;
1149 case 1:
1150 blocktranslate = true;
1151 if (level < static_cast<int>(firstblocklevel)) {
1152 req->walk_fault = true;
1153 goto do_fault;
1154 }
1155 break;
1156 case 3:
1157 tableattrs |= extract64(desc, 59, 5);
1158 if (!check_out_addr(ttbr, p_pamax)) {
1159 req->fault = FaultKind::AddressSize;
1160 goto do_fault;
1161 }
1162 level++;
1163 break;
1164 }
1165 } while (!blocktranslate);
1166
1167 if (!check_out_addr(ttbr, outputsize)) {
1168 req->fault = FaultKind::AddressSize;
1169 goto do_fault;
1170 }
1171
1172 {
1173 unsigned long page_size;
1174 page_size = (1ULL << ((stride * (4 - level)) + 3));
1175 ttbr |= (walk_va & (page_size - 1));
1176 req->page_size = ((stride * (4 - level)) + 3);
1177 }
1178
1179 s2attrs = attrs = extract64(desc, 2, 10) | (extract64(desc, 52, 12) << 10);
1180 if (req->stage == 1) {
1181 attrs |= extract32(tableattrs, 0, 2) << 11;
1182 attrs |= extract32(tableattrs, 3, 1) << 5;
1183 if (0 && extract32(tableattrs, 2, 1)) {
1184 attrs &= ~(1 << 4);
1185 }
1186 }
1187
1188 req->prot = IOMMU_RW;
1189 if ((attrs & (1 << 8)) == 0) {
1190 SCP_INFO(()) << "access forbidden " << std::hex << attrs;
1191 req->fault = FaultKind::AccessFlag;
1192 goto do_fault;
1193 }
1194
1195 if (req->stage == 1) {
1196 const bool unprivileged_access = !req->privileged;
1197 const bool user_access = (attrs & (1 << 4)) != 0;
1198 if (unprivileged_access && !user_access) {
1199 SCP_INFO(()) << "Unprivileged access forbidden by AP[1]";
1200 req->fault = FaultKind::Permission;
1201 goto do_fault;
1202 }
1203 if (attrs & (1 << 5)) {
1204 if (req->access == IOMMU_WO) {
1205 SCP_INFO(()) << "Write access forbidden " << std::hex << attrs;
1206 req->fault = FaultKind::Permission;
1207 goto do_fault;
1208 }
1209 req->prot &= ~IOMMU_WO;
1210 }
1211 } else {
1212 switch ((s2attrs >> 4) & 3) {
1213 case 0:
1214 req->fault = FaultKind::Permission;
1215 goto do_fault;
1216 break;
1217 case 1:
1218 if (req->access == IOMMU_WO) {
1219 req->fault = FaultKind::Permission;
1220 goto do_fault;
1221 }
1222 req->prot &= ~IOMMU_WO;
1223 break;
1224 case 2:
1225 if (req->access == IOMMU_RO) {
1226 req->fault = FaultKind::Permission;
1227 goto do_fault;
1228 }
1229 req->prot &= ~IOMMU_RO;
1230 break;
1231 case 3:
1232 break;
1233 }
1234 }
1235
1236 req->pa = ttbr;
1237 return;
1238
1239 do_fault:
1240 if (req->walk_fault && have_descriptor) {
1241 SCP_WARN(()) << "SMMU failed descriptor: CB " << std::dec << cb << " address 0x" << std::hex
1242 << last_descaddr << " value 0x" << last_desc;
1243 }
1244 SCP_INFO(()) << "smmu fault id " << req->trace_id << " SID 0x" << std::hex << req->sid << " CB" << cb;
1245 dump_trans_req(*req);
1246 dump_cb_state(cb);
1247 dump_state();
1248 req->err = true;
1249 if (!req->internal_walk && !req->suppress_fault) {
1250 const unsigned int fault_cb = (req->nested && req->stage == 2) ? req->s1_cb : cb;
1251 smmu500_fault(fault_cb, req, level);
1252 }
1253 }
1254
1255 bool smmu500_at64(unsigned int cb, uint64_t va, bool wr, bool s2, uint64_t* pa, int* prot, uint64_t* page_size,
1256 uint64_t* fault_addr, const FaultAttrs& fault_attrs, uint64_t trace_id = 0, uint64_t sid = 0)
1257 {
1258 unsigned int s2_cb = 0;
1259 const unsigned int nr_cb = std::min<unsigned int>(p_num_cb, SMMU_MAX_CB);
1260 TransReq req{};
1261 unsigned int t;
1262 int s1_prot = IOMMU_RW;
1263 uint64_t s1_page_size;
1264
1265 t = SMMU_CBAR[cb][CBAR_TYPE];
1266
1267 switch (t) {
1268 case 0:
1269 req.stage = 2;
1270 req.s2_enabled = true;
1271 req.s2_cb = cb;
1272 s2_cb = cb;
1273 break;
1274 case 1:
1275 req.stage = 1;
1276 req.s2_enabled = false;
1277 break;
1278 case 2:
1279 req.stage = 1;
1280 req.s2_enabled = false;
1281 break;
1282 case 3:
1283 req.stage = 1;
1284 req.s2_enabled = true;
1285 req.s2_cb = SMMU_CBAR[cb][CBAR_S2_CBNDX];
1286 req.nested = true;
1287 req.s1_cb = cb;
1288 s2_cb = req.s2_cb;
1289 break;
1290 }
1291
1292 req.va = va;
1293 req.trace_id = trace_id;
1294 req.sid = sid;
1295 req.tcr[1] = static_cast<uint32_t>(SMMU_CB_TCR2[cb]);
1296 req.tcr[1] <<= 32;
1297 req.tcr[1] |= static_cast<uint32_t>(SMMU_CB_TCR_LPAE[cb]);
1298
1299 req.ttbr[1][0] = smmu500_ttbr(static_cast<uint32_t>(SMMU_CB_TTBR0_HIGH[cb]),
1300 static_cast<uint32_t>(SMMU_CB_TTBR0_LOW[cb]));
1301
1302 req.ttbr[1][1] = smmu500_ttbr(static_cast<uint32_t>(SMMU_CB_TTBR1_HIGH[cb]),
1303 static_cast<uint32_t>(SMMU_CB_TTBR1_LOW[cb]));
1304
1305 req.access = wr ? IOMMU_WO : IOMMU_RO;
1306 req.page_size = *page_size;
1307 s1_page_size = req.page_size;
1308 req.privileged = fault_attrs.privileged;
1309 req.instruction = fault_attrs.instruction;
1310 req.non_secure = fault_attrs.non_secure;
1311 req.address_translation = fault_attrs.address_translation;
1312 req.asynchronous = fault_attrs.asynchronous;
1313 req.suppress_fault = fault_attrs.suppress_fault;
1314 req.mid = fault_attrs.mid;
1315 req.pid = fault_attrs.pid;
1316 req.bid = fault_attrs.bid;
1317
1318 if (req.s2_enabled && req.s2_cb >= nr_cb) {
1319 req.err = true;
1320 req.fault = FaultKind::Translation;
1321 req.pa = va;
1322 req.prot = IOMMU_NONE;
1323 if (!req.suppress_fault) smmu500_fault(cb, &req, 0);
1324 *pa = req.pa;
1325 *prot = req.prot;
1326 *page_size = req.page_size;
1327 if (fault_addr) *fault_addr = va;
1328 return true;
1329 }
1330
1331 if (req.s2_enabled) {
1332 req.tcr[2] = static_cast<uint32_t>(SMMU_CB_TCR_LPAE[s2_cb]);
1333 req.ttbr[2][0] = smmu500_ttbr(static_cast<uint32_t>(SMMU_CB_TTBR0_HIGH[s2_cb]),
1334 static_cast<uint32_t>(SMMU_CB_TTBR0_LOW[s2_cb]));
1335 }
1336
1337 if (req.stage == 1) {
1338 smmu500_ptw64(cb, &req);
1339 if (!req.err) {
1340 s1_prot = req.prot;
1341 s1_page_size = req.page_size;
1342 req.stage++;
1343 }
1344 } else {
1345 req.pa = req.va;
1346 }
1347
1348 if (!req.err && s2 && req.s2_enabled) {
1349 req.va = req.pa;
1350 /* A nested context uses the CBAR-selected stage-2 context for
1351 * the final walk as well as for translating stage-1 descriptors. */
1352 smmu500_ptw64(req.s2_cb, &req);
1353 if (!req.err) {
1354 req.prot &= s1_prot;
1355 if (s1_page_size < req.page_size) req.page_size = s1_page_size;
1356 }
1357 }
1358
1359 *pa = req.pa;
1360 *prot = req.prot;
1361 *page_size = req.page_size;
1362 if (fault_addr) *fault_addr = req.ipafar_valid ? req.ipafar : (req.err ? req.va : va);
1363 return req.err;
1364 }
1365
1366 bool smmu500_at(unsigned int cb, uint64_t va, bool wr, bool s2, uint64_t* pa, int* prot, uint64_t* page_size,
1367 uint64_t* fault_addr, const FaultAttrs& fault_attrs, uint64_t trace_id = 0, uint64_t sid = 0)
1368 {
1369 return smmu500_at64(cb, va, wr, s2, pa, prot, page_size, fault_addr, fault_attrs, trace_id, sid);
1370 }
1371
1372 void smmu500_gat(uint64_t v, bool wr, bool s2, bool unprivileged, bool non_secure = false)
1373 {
1374 std::lock_guard<std::recursive_mutex> translation_lock(m_translation_lock);
1375 uint64_t va = v & ~SMMU_ADDRMASK;
1376 unsigned int cb = v & SMMU_ADDRMASK;
1377 uint64_t pa;
1378 int prot;
1379 uint64_t page_size = 12; // log2 default page size: 4 KiB
1380 FaultAttrs fault_attrs;
1381 fault_attrs.privileged = !unprivileged;
1382 fault_attrs.non_secure = non_secure;
1383 fault_attrs.address_translation = true;
1384
1385 if (cb >= p_num_cb) {
1386 smmu500_record_global_fault(1u << 0, non_secure);
1387 if (non_secure) {
1388 smmu500_non_secure_gr0_write32(SMMU_GPAR.get_offset(), 1);
1389 smmu500_non_secure_gr0_write32(SMMU_GPAR_H.get_offset(), 0);
1390 } else {
1391 SMMU_GPAR = 1;
1392 SMMU_GPAR_H = 0;
1393 }
1394 return;
1395 }
1396
1397 SCP_INFO(()) << "ATS: va=0x" << std::hex << va << " cb=0x" << cb << " wr=" << wr << " s2=" << s2;
1398 bool err = smmu500_at(cb, va, wr, s2, &pa, &prot, &page_size, nullptr, fault_attrs);
1399
1400 if (non_secure) {
1401 smmu500_non_secure_gr0_write32(SMMU_GPAR.get_offset(), static_cast<uint32_t>(pa | err));
1402 smmu500_non_secure_gr0_write32(SMMU_GPAR_H.get_offset(), static_cast<uint32_t>(pa >> 32));
1403 } else {
1404 SMMU_GPAR = static_cast<uint32_t>(pa | err);
1405 SMMU_GPAR_H = static_cast<uint32_t>(pa >> 32);
1406 }
1407 }
1408
1409public:
1410 static constexpr uint32_t SMMU_STREAM_ID_MASK = 0x7FFFu;
1411 static constexpr uint64_t SMMU_DSP_WINDOW_START = 0x90000000ULL;
1412 static constexpr uint64_t SMMU_DSP_WINDOW_END = 0xa0000000ULL;
1413
1414 static uint16_t smmu500_compose_stream_id(uint32_t topology_id, uint64_t addr, bool address_carries_csid = true)
1415 {
1416 /* When the address carries CSID[3:0] in bits [35:32], keep the TBU
1417 * and topology fields from the configured base SID and assemble the
1418 * architectural SID[14:0] explicitly. */
1419 const uint32_t tbu_id = (topology_id >> 10) & 0x1fu;
1420 const uint32_t topo_id = (topology_id >> 5) & 0x1fu;
1421 const uint32_t configured_csid = topology_id & 0xfu;
1422 const uint32_t transport_csid = address_carries_csid ? static_cast<uint32_t>((addr >> 32) & 0xfu) : 0;
1423 const uint32_t csid = transport_csid != 0 ? transport_csid : configured_csid;
1424 return static_cast<uint16_t>((tbu_id << 10) | (topo_id << 5) | csid);
1425 }
1426
1427 uint16_t smmu500_resolve_stream_id(uint32_t topology_id, uint64_t addr, int* cb,
1428 bool* stream_match_conflict = nullptr, uint32_t* s2cr_type = nullptr,
1429 bool address_carries_csid = true)
1430 {
1431 std::lock_guard<std::recursive_mutex> translation_lock(m_translation_lock);
1432 const uint16_t stream_id = smmu500_compose_stream_id(topology_id, addr, address_carries_csid);
1433 bool conflict = false;
1434 uint32_t type = 0;
1435 const int resolved_cb = smmu500_stream_id_match(stream_id, &conflict, &type);
1436 if (cb) *cb = resolved_cb;
1437 if (stream_match_conflict) *stream_match_conflict = conflict;
1438 if (s2cr_type) *s2cr_type = type;
1439 return stream_id;
1440 }
1441
1442 IOMMUTLBEntry smmu500_translate(tlm::tlm_generic_payload& txn, uint64_t sid, bool transport_stream_address = false,
1443 bool record_fault = true)
1444 {
1445 std::lock_guard<std::recursive_mutex> translation_lock(m_translation_lock);
1446 const uint64_t trace_id = ++m_translation_seq;
1447 uint64_t addr = txn.get_address();
1448 uint64_t page_size = 12;
1449 IOMMUTLBEntry ret = {
1450 /* Keep the request address in the TLB entry. Consumers use this
1451 * to associate the translated page with the upstream IOVA. */
1452 .iova = addr,
1453 .translated_addr = addr,
1454 .addr_mask = (1ULL << page_size) - 1,
1455 .perm = IOMMU_RW,
1456 };
1457 int cb;
1458 /* Legacy TBU traffic encodes CSID[3:0] in address bits [35:32]. A
1459 * genuine high VA cannot use that encoding, so retain it in full and
1460 * select the configured CSID instead. */
1461 const bool address_carries_csid = transport_stream_address && (addr >> 36) == 0;
1462 uint64_t va = addr & ~SMMU_ADDRMASK;
1463 if (address_carries_csid) {
1464 va &= ((1ULL << SMMU_VA_WIDTH) - 1);
1465 }
1466 uint64_t pa = va;
1467 int prot;
1468 uint64_t fault_addr = va;
1469 bool err = false;
1470 const uint16_t master_id = smmu500_compose_stream_id(static_cast<uint32_t>(sid), addr, address_carries_csid);
1471 const uint32_t transport_stream_bits = address_carries_csid ? static_cast<uint32_t>(master_id & 0xfu) : 0;
1472 const FaultAttrs fault_attrs = smmu500_extract_fault_attrs(txn, record_fault);
1473 const uint32_t global_control = smmu500_global_control(fault_attrs.non_secure);
1474 bool stream_match_conflict = false;
1475 uint32_t s2cr_type = 0;
1476 cb = smmu500_stream_id_match(master_id, &stream_match_conflict, &s2cr_type);
1477 if (transport_stream_address && (addr >> 36) != 0) {
1478 SCP_INFO(()) << std::hex << "SMMU high-VA TBU request id " << trace_id << " address 0x" << addr
1479 << " configured SID 0x" << sid << " effective SID 0x" << master_id << " CSID carrier "
1480 << address_carries_csid << " CB " << cb << " S2CR type " << s2cr_type;
1481 }
1482 bool clientpd = global_control & SCR0_CLIENTPD_MASK;
1483 if (clientpd) {
1484 ret.addr_mask = -1;
1485 return ret;
1486 }
1487
1488 if (cb < 0) {
1489 /* USFCFG selects whether unidentified streams generate a global
1490 * fault. GFRE selects whether that recorded fault rejects the
1491 * transaction or permits a bypass response. A TBU request with
1492 * no transport carrier has no CSID to match; preserve the
1493 * identity boot traffic instead of turning it into a fault. */
1494 const bool has_transport_csid = address_carries_csid && ((addr >> 32) & 0xfu) != 0;
1495 if ((global_control & SCR0_USFCFG_MASK) && (!transport_stream_address || has_transport_csid)) {
1496 smmu500_report_global_fault(ret, 1u << 1, fault_attrs.non_secure, global_control, record_fault);
1497 } else {
1498 ret.addr_mask = -1;
1499 }
1500 return ret;
1501 }
1502 if (stream_match_conflict) {
1503 if (global_control & SCR0_SMCFCFG_MASK) {
1504 smmu500_report_global_fault(ret, 1u << 2, fault_attrs.non_secure, global_control, record_fault);
1505 return ret;
1506 }
1507 }
1508
1509 /* S2CR.TYPE selects the action for a matched stream. */
1510 if (s2cr_type == 1) { // Bypass
1511 ret.addr_mask = -1;
1512 return ret;
1513 }
1514 const unsigned int nr_cb = std::min<unsigned int>(p_num_cb, SMMU_MAX_CB);
1515 if (s2cr_type != 0 || cb >= static_cast<int>(nr_cb)) { // Fault/reserved/invalid context
1516 smmu500_report_global_fault(ret, 1u << 0, fault_attrs.non_secure, global_control, record_fault);
1517 return ret;
1518 }
1519 if (!SMMU_CBA2R[cb][CBA2R_VA64]) {
1520 va &= ((1ULL << SMMU_VA_WIDTH) - 1);
1521 }
1522
1523 bool wr = (txn.get_command() == tlm::TLM_WRITE_COMMAND);
1524 const bool fault_latched = (static_cast<uint32_t>(SMMU_CB_FSR[cb]) & CB_FSR_FAULT_MASK) != 0;
1525 /* A later fault must still reach smmu500_fault so the TRM-defined
1526 * MULTI status bit is set. Only non-recording probes, such as DMI
1527 * lookups, suppress fault status updates. */
1528 err = smmu500_at(cb, va, wr, true, &pa, &prot, &page_size, &fault_addr, fault_attrs, trace_id, master_id);
1529 ret.translated_addr = pa;
1530 ret.perm = (IOMMUAccessFlags)prot;
1531 if (err) {
1532 if (record_fault && !fault_latched) {
1533 const uint64_t recorded_addr = (fault_addr == va) ? (va | (addr & SMMU_PAGEMASK)) : fault_addr;
1534 SMMU_CB_IPAFAR_LOW[cb] = static_cast<uint32_t>(recorded_addr);
1535 SMMU_CB_IPAFAR_HIGH[cb] = static_cast<uint32_t>(recorded_addr >> 32);
1536 if (SMMU_CBAR[cb][CBAR_TYPE] == 0 || SMMU_CBAR[cb][CBAR_TYPE] == 3) {
1537 SMMU_CB_FAR_LOW[cb] = static_cast<uint32_t>(recorded_addr);
1538 SMMU_CB_FAR_HIGH[cb] = static_cast<uint32_t>(recorded_addr >> 32);
1539 }
1540 }
1541 memset(&ret, 0, sizeof ret);
1542 ret.iova = addr;
1543 ret.perm = IOMMU_NONE;
1544 } else {
1545 ret.addr_mask = (page_size >= 64) ? UINT64_MAX : ((1ULL << page_size) - 1);
1546 ret.translated_addr &= ~ret.addr_mask;
1547 }
1548 return ret;
1549 }
1550
1551 int smmu500_stream_id_match(uint32_t stream_id, bool* stream_match_conflict = nullptr,
1552 uint32_t* s2cr_type = nullptr)
1553 {
1554 const uint32_t effective_stream_id = stream_id & SMMU_STREAM_ID_MASK;
1555 /* SIDR0.NUMSMRG is an 8-bit architectural identification field.
1556 * Use the configured model capacity for the software-programmable
1557 * SMR/S2CR arrays instead of making matching depend on that field. */
1558 const unsigned int nr_smr = std::min<unsigned int>(static_cast<unsigned int>(p_num_smr), SMMU_MAX_SMR);
1559 int cbndx = -1;
1560 bool conflict = false;
1561 uint32_t type = 0;
1562
1563 for (unsigned int i = 0; i < nr_smr; i++) {
1564 bool valid = SMMU_SMR[i][SMR_VALID];
1565 const uint32_t mask = static_cast<uint32_t>(SMMU_SMR[i][SMR_MASK]) & SMMU_STREAM_ID_MASK;
1566 const uint32_t id = static_cast<uint32_t>(SMMU_SMR[i][SMR_ID]) & SMMU_STREAM_ID_MASK;
1567
1568 if (valid && ((id ^ effective_stream_id) & ~mask & SMMU_STREAM_ID_MASK) == 0) {
1569 if (cbndx < 0) {
1570 cbndx = static_cast<int>(SMMU_S2CR[i][S2CR_CBNDX_VMID]);
1571 type = static_cast<uint32_t>(SMMU_S2CR[i][S2CR_TYPE]);
1572 } else {
1573 conflict = true;
1574 }
1575 }
1576 }
1577
1578 if (stream_match_conflict) *stream_match_conflict = conflict;
1579 if (s2cr_type) *s2cr_type = type;
1580
1581 return cbndx;
1582 }
1583
1584 void start_of_simulation();
1585 void before_end_of_elaboration();
1586};
1587
1588template <unsigned int BUSWIDTH>
1589class smmu500_tbu : public sc_core::sc_module
1590{
1591 // Transport addresses use bits [35:32] for the CSID carrier. Bypass DMI
1592 // must stay within the 32-bit VA portion; translated IOMMU ranges retain
1593 // their actual reported start and size.
1594 static constexpr unsigned int TRANSPORT_IOVA_WIDTH = 32;
1595
1596 SCP_LOGGER();
1597 smmu500<BUSWIDTH>* m_smmu;
1598 cci::cci_broker_handle m_broker;
1599
1600 std::pair<uint64_t, uint64_t> dmi_range[SMMU_MAX_CB] = {};
1601 bool dmi_range_valid[SMMU_MAX_CB] = { false };
1602 std::pair<uint64_t, uint64_t> bypass_dmi_range = {};
1603 bool bypass_dmi_range_valid = false;
1604
1605 struct MemoryView {
1606 uint64_t address;
1607 uint64_t page_start;
1608 uint64_t page_end;
1609 uint64_t page_size;
1610 };
1611
1612#ifndef WITHOUT_QEMU
1613 QemuInstance* m_qemu_instance = nullptr;
1614
1615 struct QemuIommuRegion {
1616 const uint64_t start;
1617 const uint64_t size;
1619 std::shared_ptr<qemu::IOMMUMemoryRegion> mr;
1620 std::map<uint64_t, qemu::IOMMUMemoryRegion::IOMMUTLBEntry> ranges[2];
1621 std::mutex cache_mutex;
1622
1623 QemuIommuRegion(uint64_t region_start, uint64_t region_size): start(region_start), size(region_size) {}
1624 };
1625
1626 std::unique_ptr<QemuIommuRegion> m_qemu_iommu_region;
1627
1628 void smmu500_qemu_setup_region()
1629 {
1630 auto& region = *m_qemu_iommu_region;
1631 auto& inst = m_qemu_instance->get();
1632
1633 region.owner = inst.template object_new_unparented<QemuInstanceDmiManager::QemuContainer>();
1634 region.mr = std::make_shared<qemu::IOMMUMemoryRegion>(
1635 inst.template object_new_unparented<qemu::IOMMUMemoryRegion>());
1636 region.mr->init(
1637 region.owner, "smmu500-iommu", region.size, downstream_socket.create_io_ops(),
1638 [this](qemu::IOMMUMemoryRegion::IOMMUTLBEntry* te, uint64_t addr,
1639 qemu::IOMMUMemoryRegion::IOMMUAccessFlags flags,
1640 int idx) { smmu500_qemu_translate(*te, addr, flags, idx); },
1641 [](qemu::MemoryRegionOps::MemTxAttrs attrs) { return attrs.secure ? 1 : 0; }, [] { return 2; },
1642 [](int idx) {
1643 qemu::MemoryRegionOps::MemTxAttrs attrs;
1644 attrs.secure = idx != 0;
1645 return attrs;
1646 });
1647 downstream_socket.init(*region.mr);
1648 }
1649
1650 void smmu500_qemu_translate(qemu::IOMMUMemoryRegion::IOMMUTLBEntry& entry, uint64_t addr,
1651 qemu::IOMMUMemoryRegion::IOMMUAccessFlags flags, int idx)
1652 {
1653 auto& region = *m_qemu_iommu_region;
1654 entry = { .target_as = downstream_socket.get_address_space().get_ptr(),
1655 .iova = addr & ~SMMU_PAGEMASK,
1656 .translated_addr = addr & ~SMMU_PAGEMASK,
1657 .addr_mask = SMMU_PAGEMASK,
1658 .perm = qemu::IOMMUMemoryRegion::IOMMU_NONE };
1659 if (addr >= region.size) {
1660 return;
1661 }
1662
1663 const bool cache_eligible = flags == qemu::IOMMUMemoryRegion::IOMMU_NONE && (idx == 0 || idx == 1);
1664 if (cache_eligible) {
1665 std::lock_guard<std::mutex> cache_lock(region.cache_mutex);
1666 auto& ranges = region.ranges[idx];
1667 auto it = ranges.upper_bound(addr);
1668 if (it != ranges.begin()) {
1669 --it;
1670 if (addr - it->second.iova <= it->second.addr_mask) {
1671 entry = it->second;
1672 return;
1673 }
1674 }
1675 }
1676
1677 auto translated = entry;
1678 tlm::tlm_generic_payload txn;
1679 const uint64_t iova = region.start + addr;
1680 txn.set_address(iova);
1681 // Input IOMMU_NONE is a translation probe; output IOMMU_NONE is a failed translation.
1682 txn.set_command(flags == qemu::IOMMUMemoryRegion::IOMMU_NONE ? tlm::TLM_IGNORE_COMMAND
1683 : (flags & qemu::IOMMUMemoryRegion::IOMMU_WO) ? tlm::TLM_WRITE_COMMAND
1684 : tlm::TLM_READ_COMMAND);
1685 QemuMemTxAttrExtensions attrs(txn, region.mr->attrs_for_index(idx));
1687 const bool record_fault = flags == qemu::IOMMUMemoryRegion::IOMMU_RO ||
1688 flags == qemu::IOMMUMemoryRegion::IOMMU_WO;
1689 try {
1690 te = m_smmu->smmu500_translate(txn, p_topology_id, true, record_fault);
1691 } catch (const typename smmu500<BUSWIDTH>::DmaDmiMiss&) {
1692 auto& inst = m_qemu_instance->get();
1693 const bool iothread_locked = inst.is_iothread_locked();
1694 if (iothread_locked) inst.unlock_iothread();
1695 m_smmu->m_dma_on_sysc.run_on_sysc(
1696 [&, record_fault] { te = m_smmu->smmu500_translate(txn, p_topology_id, true, record_fault); });
1697 if (iothread_locked) inst.lock_iothread();
1698 }
1699 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE) return;
1700 const uint64_t pa = (te.translated_addr & ~te.addr_mask) | (iova & te.addr_mask);
1701 uint64_t mask = te.addr_mask;
1702 const bool transport_csid = (iova >> 36) == 0;
1703 const uint64_t csid_window_mask = static_cast<uint64_t>(UINT32_MAX);
1704 const uint64_t csid_window_end = (iova & ~csid_window_mask) | csid_window_mask;
1705 while ((addr & mask) != (pa & mask) || (addr | mask) >= region.size ||
1706 (transport_csid && (iova | mask) > csid_window_end)) {
1707 mask >>= 1;
1708 }
1709
1710 translated.target_as = downstream_socket.get_address_space().get_ptr();
1711 translated.iova = addr & ~mask;
1712 translated.translated_addr = pa & ~mask;
1713 translated.addr_mask = mask;
1714 switch (te.perm) {
1715 case smmu500<BUSWIDTH>::IOMMU_RO:
1716 translated.perm = qemu::IOMMUMemoryRegion::IOMMU_RO;
1717 break;
1718 case smmu500<BUSWIDTH>::IOMMU_WO:
1719 translated.perm = qemu::IOMMUMemoryRegion::IOMMU_WO;
1720 break;
1721 case smmu500<BUSWIDTH>::IOMMU_RW:
1722 translated.perm = qemu::IOMMUMemoryRegion::IOMMU_RW;
1723 break;
1724 default:
1725 return;
1726 }
1727 entry = translated;
1728 if (cache_eligible && entry.perm != qemu::IOMMUMemoryRegion::IOMMU_NONE) {
1729 std::lock_guard<std::mutex> cache_lock(region.cache_mutex);
1730 region.ranges[idx][entry.iova] = entry;
1731 }
1732 }
1733
1734 void set_iommu_hint(tlm::tlm_generic_payload& txn)
1735 {
1736 const uint64_t iova = txn.get_address();
1737 const auto* origin = txn.get_extension<QemuInstanceHintTlmExtension>();
1738 if (!origin || !origin->instance) return;
1739
1740 if (m_qemu_instance == nullptr) {
1741 downstream_socket.set_qemu_inst(*origin->instance);
1742 m_qemu_instance = origin->instance;
1743 }
1744 if (origin->instance != m_qemu_instance) {
1745 SCP_WARN(()) << "Cannot replace QEMU MR hint from a different QEMU instance";
1746 return;
1747 }
1748 if (!m_qemu_iommu_region) {
1749 SCP_WARN(()) << "Cannot replace QEMU MR hint: upstream socket size is zero";
1750 return;
1751 }
1752 auto& region = *m_qemu_iommu_region;
1753 if (iova < region.start || iova - region.start >= region.size) {
1754 SCP_WARN(()) << "Cannot replace QEMU MR hint for IOVA 0x" << std::hex << iova
1755 << ": it is outside upstream socket [0x" << region.start << ", 0x"
1756 << region.start + region.size - 1 << "]";
1757 return;
1758 }
1759
1760 if (const auto* cpu_hint = txn.get_extension<QemuCpuHintTlmExtension>()) {
1761 downstream_socket.set_cpu(cpu_hint->get_cpu());
1762 }
1763
1764 if (!region.mr) {
1765 auto& inst = m_qemu_instance->get();
1766 const bool locked = inst.is_iothread_locked();
1767 if (!locked) inst.lock_iothread();
1768 smmu500_qemu_setup_region();
1769 if (!locked) inst.unlock_iothread();
1770 }
1771
1772 const QemuMrHintTlmExtension iommu_hint(*region.mr, iova - region.start);
1773 auto* hint = txn.get_extension<QemuMrHintTlmExtension>();
1774 if (hint) {
1775 hint->copy_from(iommu_hint);
1776 } else {
1777 txn.set_extension(new QemuMrHintTlmExtension(iommu_hint));
1778 }
1779 }
1780
1781 void smmu500_qemu_invalidate()
1782 {
1783 if (!m_qemu_iommu_region || !m_qemu_iommu_region->mr) return;
1784
1785 auto& region = *m_qemu_iommu_region;
1786 auto& inst = region.mr->get_inst();
1787 const bool iothread_locked = inst.is_iothread_locked();
1788 if (!iothread_locked) inst.lock_iothread();
1789 {
1790 std::lock_guard<std::mutex> cache_lock(region.cache_mutex);
1791 region.ranges[0].clear();
1792 region.ranges[1].clear();
1793 }
1795 .target_as = downstream_socket.get_address_space().get_ptr(),
1796 .iova = 0,
1797 .translated_addr = 0,
1798 .addr_mask = UINT64_MAX,
1799 .perm = qemu::IOMMUMemoryRegion::IOMMU_NONE,
1800 };
1801 region.mr->iommu_unmap(&flush);
1802 if (!iothread_locked) inst.unlock_iothread();
1803 }
1804
1805#endif
1806
1807protected:
1808 void b_transport(tlm::tlm_generic_payload& txn, sc_core::sc_time& delay)
1809 {
1810 sc_dt::uint64 addr = txn.get_address();
1811 tlm::tlm_command cmd = txn.get_command();
1812 const bool high_va = (addr >> 36) != 0;
1813 if (high_va) {
1814 SCP_INFO(()) << std::hex << "TBU high-VA b_transport address 0x" << addr << " topology ID 0x"
1815 << static_cast<uint32_t>(p_topology_id);
1816 }
1817 typename smmu500<BUSWIDTH>::IOMMUTLBEntry te = m_smmu->smmu500_translate(txn, p_topology_id, true);
1818 if (high_va) {
1819 SCP_INFO(()) << std::hex << "TBU high-VA b_transport result PA 0x" << te.translated_addr << " mask 0x"
1820 << te.addr_mask << " permissions " << te.perm;
1821 }
1822 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE ||
1823 (cmd == tlm::TLM_WRITE_COMMAND && te.perm == smmu500<BUSWIDTH>::IOMMU_RO) ||
1824 (cmd == tlm::TLM_READ_COMMAND && te.perm == smmu500<BUSWIDTH>::IOMMU_WO)) {
1825 txn.set_response_status(tlm::TLM_ADDRESS_ERROR_RESPONSE);
1826 wait(sc_core::SC_ZERO_TIME);
1827 } else {
1828 txn.set_address((te.translated_addr & ~te.addr_mask) | (addr & te.addr_mask));
1829 downstream_socket->b_transport(txn, delay);
1830 txn.set_address(addr);
1831#ifndef WITHOUT_QEMU
1832 set_iommu_hint(txn);
1833#endif
1834 }
1835 }
1836
1837 virtual unsigned int transport_dbg(tlm::tlm_generic_payload& txn)
1838 {
1839 sc_dt::uint64 addr = txn.get_address();
1840 typename smmu500<BUSWIDTH>::IOMMUTLBEntry te = m_smmu->smmu500_translate(txn, p_topology_id, true, false);
1841 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE) {
1842 return 0;
1843 }
1844 txn.set_address((te.translated_addr & ~te.addr_mask) | (addr & te.addr_mask));
1845 int ret = downstream_socket->transport_dbg(txn);
1846 txn.set_address(addr);
1847 return ret;
1848 }
1849
1850 virtual bool get_direct_mem_ptr(tlm::tlm_generic_payload& txn, tlm::tlm_dmi& dmi_data)
1851 {
1852#ifndef WITHOUT_QEMU
1853 // qemu_map() probes DMI during discovery. Refuse a PA DMI alias to avoid bypassing the IOMMU;
1854 // return the whole IOVA window so discovery can skip per-page probes.
1855 if (const auto* origin = txn.get_extension<QemuInstanceHintTlmExtension>()) {
1856 const uint64_t addr = txn.get_address();
1857 if (m_qemu_iommu_region && addr >= m_qemu_iommu_region->start &&
1858 addr - m_qemu_iommu_region->start < m_qemu_iommu_region->size) {
1859 dmi_data.set_start_address(m_qemu_iommu_region->start);
1860 dmi_data.set_end_address(m_qemu_iommu_region->start + m_qemu_iommu_region->size - 1);
1861 } else {
1862 dmi_data.set_start_address(addr & ~SMMU_PAGEMASK);
1863 dmi_data.set_end_address(addr | SMMU_PAGEMASK);
1864 }
1865 if (origin->instance == m_qemu_instance) {
1866 if (const auto* cpu_hint = txn.get_extension<QemuCpuHintTlmExtension>()) {
1867 downstream_socket.set_cpu(cpu_hint->get_cpu());
1868 }
1869 }
1870 return false;
1871 }
1872#endif
1873 MemoryView VIRT;
1874 MemoryView PHYS;
1875
1876 VIRT.address = txn.get_address();
1877 const bool high_va = (VIRT.address >> 36) != 0;
1878 if (high_va) {
1879 SCP_INFO(()) << std::hex << "TBU high-VA DMI request address 0x" << VIRT.address << " topology ID 0x"
1880 << static_cast<uint32_t>(p_topology_id);
1881 }
1882 typename smmu500<BUSWIDTH>::IOMMUTLBEntry te = m_smmu->smmu500_translate(txn, p_topology_id, true, false);
1883 if (high_va) {
1884 SCP_INFO(()) << std::hex << "TBU high-VA DMI result PA 0x" << te.translated_addr << " mask 0x"
1885 << te.addr_mask << " permissions " << te.perm;
1886 }
1887
1888 if (te.perm == smmu500<BUSWIDTH>::IOMMU_NONE) {
1889 int CB = -1;
1890 const bool address_carries_csid = (VIRT.address >> 36) == 0;
1891 m_smmu->smmu500_resolve_stream_id(p_topology_id, VIRT.address, &CB, nullptr, nullptr, address_carries_csid);
1892 if (CB >= 0 && static_cast<unsigned int>(CB) < SMMU_MAX_CB) {
1893 const uint64_t page = VIRT.address & ~SMMU_PAGEMASK;
1894 if (!dmi_range_valid[CB] || dmi_range[CB].first > page) {
1895 dmi_range[CB].first = page;
1896 }
1897 if (!dmi_range_valid[CB] || dmi_range[CB].second < page + SMMU_PAGEMASK) {
1898 dmi_range[CB].second = page + SMMU_PAGEMASK;
1899 }
1900 dmi_range_valid[CB] = true;
1901 }
1902 return false;
1903 }
1904
1905 if (te.addr_mask == static_cast<uint64_t>(-1)) {
1906 assert(te.translated_addr == VIRT.address);
1907 bool ret = downstream_socket->get_direct_mem_ptr(txn, dmi_data);
1908 if (!ret) {
1909 return false;
1910 }
1911 if (reinterpret_cast<uintptr_t>(dmi_data.get_dmi_ptr()) < 4096) {
1912 return false;
1913 }
1914 VIRT.page_end = dmi_data.get_end_address();
1915 if (VIRT.page_end >> TRANSPORT_IOVA_WIDTH != VIRT.address >> TRANSPORT_IOVA_WIDTH) {
1916 dmi_data.set_end_address((((VIRT.address >> TRANSPORT_IOVA_WIDTH) + 1) << TRANSPORT_IOVA_WIDTH) - 1);
1917 }
1918 VIRT.page_start = dmi_data.get_start_address();
1919 if (VIRT.page_start >> TRANSPORT_IOVA_WIDTH != VIRT.address >> TRANSPORT_IOVA_WIDTH) {
1920 uint64_t newstart = (VIRT.address >> TRANSPORT_IOVA_WIDTH) << TRANSPORT_IOVA_WIDTH;
1921 dmi_data.set_start_address(newstart);
1922 dmi_data.set_dmi_ptr(dmi_data.get_dmi_ptr() + (newstart - VIRT.page_start));
1923 }
1924 if (!bypass_dmi_range_valid || bypass_dmi_range.first > dmi_data.get_start_address()) {
1925 bypass_dmi_range.first = dmi_data.get_start_address();
1926 }
1927 if (!bypass_dmi_range_valid || bypass_dmi_range.second < dmi_data.get_end_address()) {
1928 bypass_dmi_range.second = dmi_data.get_end_address();
1929 }
1930 bypass_dmi_range_valid = true;
1931 return ret;
1932 }
1933
1934 PHYS.address = te.translated_addr;
1935 PHYS.page_start = te.translated_addr & ~te.addr_mask;
1936 PHYS.page_size = (te.addr_mask + 1);
1937 PHYS.page_end = PHYS.page_start + PHYS.page_size - 1;
1938 assert(PHYS.page_start <= PHYS.page_end);
1939
1940 txn.set_address(PHYS.page_start);
1941
1942 int ret = downstream_socket->get_direct_mem_ptr(txn, dmi_data);
1943 if (!ret) {
1944 txn.set_address(VIRT.address);
1945 return ret;
1946 }
1947 if (reinterpret_cast<uintptr_t>(dmi_data.get_dmi_ptr()) < 4096) {
1948 txn.set_address(VIRT.address);
1949 return false;
1950 }
1951
1952 uint64_t dmi_offset = PHYS.page_start - dmi_data.get_start_address();
1953 assert(dmi_data.get_start_address() <= PHYS.page_start);
1954 assert(PHYS.page_end <= dmi_data.get_end_address());
1955
1956 VIRT.page_start = VIRT.address & ~te.addr_mask;
1957 VIRT.page_end = VIRT.page_start + te.addr_mask;
1958
1959 dmi_data.set_dmi_ptr(dmi_data.get_dmi_ptr() + dmi_offset);
1960 dmi_data.set_start_address(VIRT.page_start);
1961 dmi_data.set_end_address(VIRT.page_end);
1962 switch (te.perm) {
1963 case smmu500<BUSWIDTH>::IOMMU_RO:
1964 dmi_data.allow_read();
1965 break;
1966 case smmu500<BUSWIDTH>::IOMMU_WO:
1967 dmi_data.allow_write();
1968 break;
1969 case smmu500<BUSWIDTH>::IOMMU_RW:
1970 dmi_data.allow_read_write();
1971 break;
1972 default:
1973 txn.set_address(VIRT.address);
1974 return false;
1975 }
1976
1977 int CB = -1;
1978 const bool address_carries_csid = (VIRT.address >> 36) == 0;
1979 txn.set_address(VIRT.address);
1980 m_smmu->smmu500_resolve_stream_id(p_topology_id, VIRT.address, &CB, nullptr, nullptr, address_carries_csid);
1981 if (CB < 0) {
1982 // An unidentified stream reached DMI only through the bypass path, so
1983 // there is no context-bank DMI range to track.
1984 txn.set_address(VIRT.address);
1985 return ret;
1986 }
1987 if (!dmi_range_valid[CB] || dmi_range[CB].first > VIRT.page_start) {
1988 dmi_range[CB].first = VIRT.page_start;
1989 }
1990 if (!dmi_range_valid[CB] || dmi_range[CB].second < VIRT.page_end) {
1991 dmi_range[CB].second = VIRT.page_end;
1992 }
1993 dmi_range_valid[CB] = true;
1994
1995 if (PHYS.page_start != VIRT.page_start) {
1996 uint64_t bits_masked = VIRT.address - (VIRT.address & ~SMMU_PAGEMASK);
1997 SCP_DEBUG(()) << std::hex << "\nVIRT Page Start 0x" << VIRT.page_start << "\nPHYS Page Start 0x"
1998 << PHYS.page_start << "\nbits_masked 0x" << bits_masked;
1999 }
2000 SCP_DEBUG(()) << std::hex << "smmu TBU DMI: translate 0x" << VIRT.address << " to 0x" << te.translated_addr
2001 << " pg size 0x" << PHYS.page_size << " pg base 0x" << PHYS.page_start << " offset 0x"
2002 << dmi_offset << " start 0x" << dmi_data.get_start_address() << " end 0x"
2003 << dmi_data.get_end_address();
2004
2005 txn.set_address(VIRT.address);
2006 return ret;
2007 }
2008
2009public:
2010 cci::cci_param<uint32_t> p_topology_id;
2011
2012 tlm_utils::simple_target_socket<smmu500_tbu> upstream_socket;
2013#ifndef WITHOUT_QEMU
2014 QemuInitiatorSocket<BUSWIDTH> downstream_socket;
2015#else
2016 tlm_utils::simple_initiator_socket<smmu500_tbu> downstream_socket;
2017#endif
2018
2019 smmu500_tbu(const sc_core::sc_module_name& name, sc_core::sc_object* o)
2020 : smmu500_tbu(name, dynamic_cast<smmu500<BUSWIDTH>*>(o))
2021 {
2022 }
2023
2024 smmu500_tbu(sc_core::sc_module_name name, smmu500<BUSWIDTH>* _smmu
2025#ifndef WITHOUT_QEMU
2026 ,
2027 QemuInstance* qemu_instance = nullptr
2028#endif
2029 )
2030 : m_broker(cci::cci_get_broker())
2031 , p_topology_id("topology_id", 0x0, "Topology ID for this TBU")
2032 , upstream_socket("upstream_socket")
2033#ifndef WITHOUT_QEMU
2034 , downstream_socket("downstream_socket", qemu_instance)
2035#else
2036 , downstream_socket("downstream_socket")
2037#endif
2038 {
2039 m_smmu = _smmu;
2040 m_smmu->tbus.push_back(this);
2041#ifndef WITHOUT_QEMU
2042 m_qemu_instance = qemu_instance;
2043#endif
2044 upstream_socket.register_b_transport(this, &smmu500_tbu::b_transport);
2045 upstream_socket.register_transport_dbg(this, &smmu500_tbu::transport_dbg);
2046 upstream_socket.register_get_direct_mem_ptr(this, &smmu500_tbu::get_direct_mem_ptr);
2047#ifndef WITHOUT_QEMU
2048 downstream_socket.set_invalidate_observer(
2049 [this](sc_dt::uint64 start, sc_dt::uint64 end) { invalidate_direct_mem_ptr(start, end); });
2050#else
2051 downstream_socket.register_invalidate_direct_mem_ptr(this, &smmu500_tbu::invalidate_direct_mem_ptr);
2052#endif
2053 }
2054
2055#ifndef WITHOUT_QEMU
2056 smmu500_tbu(sc_core::sc_module_name name, smmu500<BUSWIDTH>* _smmu, QemuInstance& qemu_instance)
2057 : smmu500_tbu(name, _smmu, &qemu_instance)
2058 {
2059 }
2060#endif
2061
2062#ifndef WITHOUT_QEMU
2063 void end_of_elaboration() override
2064 {
2065 const std::string socket_name = std::string(sc_core::sc_module::name()) + ".upstream_socket";
2066 uint64_t region_start = gs::cci_get_d<uint64_t>(m_broker, socket_name + ".address", 0);
2067 if (gs::cci_get_d<bool>(m_broker, socket_name + ".relative_addresses", true)) region_start = 0;
2068 uint64_t region_size = 0;
2069 const bool has_size = gs::cci_get(m_broker, socket_name + ".size", region_size);
2070
2071 const sc_core::sc_object* socket = &upstream_socket;
2072 std::vector<const sc_core::sc_object*> visited;
2073 while (!has_size && socket) {
2074 if (std::find(visited.begin(), visited.end(), socket) != visited.end()) break;
2075 visited.push_back(socket);
2076 const auto* module = socket->get_parent_object();
2077 const std::string bind = gs::cci_get_d<std::string>(m_broker, std::string(socket->name()) + ".bind", "");
2078 if (bind.empty()) break;
2079 const std::string path = bind.front() == '&' ? bind.substr(1) : bind;
2080 socket = sc_core::sc_find_object(path.c_str());
2081 for (auto* scope = module ? module->get_parent_object() : nullptr; !socket && scope;
2082 scope = scope->get_parent_object()) {
2083 socket = sc_core::sc_find_object((std::string(scope->name()) + "." + path).c_str());
2084 }
2085 if (!socket) break;
2086 using TargetSocket = tlm::tlm_base_target_socket_b<BUSWIDTH, tlm::tlm_fw_transport_if<>,
2087 tlm::tlm_bw_transport_if<>>;
2088 if (!dynamic_cast<const TargetSocket*>(socket) &&
2089 gs::cci_get_d<std::string>(m_broker, std::string(socket->name()) + ".bind", "").empty()) {
2090 const auto* parent = socket->get_parent_object();
2091 socket = parent ? sc_core::sc_find_object((std::string(parent->name()) + ".target_socket").c_str())
2092 : nullptr;
2093 }
2094 if (dynamic_cast<const TargetSocket*>(socket) &&
2095 gs::cci_get(m_broker, std::string(socket->name()) + ".size", region_size)) {
2096 region_start = gs::cci_get_d<uint64_t>(m_broker, std::string(socket->name()) + ".address", 0);
2097 if (gs::cci_get_d<bool>(m_broker, std::string(socket->name()) + ".relative_addresses", true)) {
2098 region_start = 0;
2099 }
2100 region_start = gs::cci_get_d<uint64_t>(
2101 m_broker, std::string(socket->get_parent_object()->name()) + ".mapped_base_addr", region_start);
2102 break;
2103 }
2104 }
2105 if (region_size != 0) {
2106 m_qemu_iommu_region = std::make_unique<QemuIommuRegion>(region_start, region_size);
2107 }
2108 }
2109#endif
2110
2111 void invalidate_direct_mem_ptr(sc_dt::uint64 start, sc_dt::uint64 end)
2112 {
2113 (void)start;
2114 (void)end;
2115 reset_dmi();
2116 }
2117
2118 void invalidate(uint32_t CB)
2119 {
2120 if (CB >= SMMU_MAX_CB) return;
2121#ifndef WITHOUT_QEMU
2122 smmu500_qemu_invalidate();
2123#endif
2124 invalidate_dmi_range(CB);
2125 }
2126
2127 void invalidate_dmi_range(uint32_t CB)
2128 {
2129 if (CB >= SMMU_MAX_CB) return;
2130 if (!dmi_range_valid[CB]) return;
2131 SCP_DEBUG(())("TLBIALL invalidate {:x} - {:x}", dmi_range[CB].first, dmi_range[CB].second);
2132 upstream_socket->invalidate_direct_mem_ptr(dmi_range[CB].first, dmi_range[CB].second);
2133 dmi_range_valid[CB] = false;
2134 }
2135
2136 void reset_dmi()
2137 {
2138#ifndef WITHOUT_QEMU
2139 smmu500_qemu_invalidate();
2140#endif
2141 for (unsigned int cb = 0; cb < SMMU_MAX_CB; ++cb) invalidate_dmi_range(cb);
2142 if (bypass_dmi_range_valid) {
2143 upstream_socket->invalidate_direct_mem_ptr(bypass_dmi_range.first, bypass_dmi_range.second);
2144 bypass_dmi_range_valid = false;
2145 }
2146 }
2147
2148#ifndef WITHOUT_QEMU
2149 void end_of_simulation() override
2150 {
2151 if (m_qemu_instance == nullptr || !m_qemu_iommu_region || !m_qemu_iommu_region->mr) return;
2152
2153 auto& inst = m_qemu_iommu_region->mr->get_inst();
2154 const bool iothread_locked = inst.is_iothread_locked();
2155 if (!iothread_locked) inst.lock_iothread();
2156 smmu500_qemu_invalidate();
2157 if (!iothread_locked) inst.unlock_iothread();
2158 }
2159#endif
2160};
2161
2162} // namespace gs
2163
2164extern "C" void module_register();
2165
2166#endif // smmu500_H
Definition qemu-cpu-hint.h:17
TLM-2.0 initiator socket specialisation for QEMU AddressSpace mapping.
Definition initiator.h:63
Definition dmi-manager.h:51
This class encapsulates a libqemu-cxx qemu::LibQemu instance. It handles QEMU parameters and instance...
Definition qemu-instance.h:89
Definition qemu-mr-hint.h:17
Definition target-signal-socket.h:163
Definition reg_model_maker.h:457
Definition reg_model_maker.h:35
Definition runonsysc.h:23
Definition smmu500.h:1590
Definition smmu500.h:84
Tool which reads a Lua configuration file and sets parameters.
Definition biflow.cc:10
Definition qemu-instance-hint.h:16
Definition qemu-memtx-attr-extensions.h:18
Definition smmu500.h:93
Definition smmu500.h:86
Definition smmu500.h:112
Definition libqemu-cxx.h:500
Definition libqemu-cxx.h:352