60class QemuInitiatorSocket
61 :
public tlm::tlm_initiator_socket<BUSWIDTH, tlm::tlm_base_protocol_types, 1, sc_core::SC_ZERO_OR_MORE_BOUND>,
62 public tlm::tlm_bw_transport_if<>
66 std::vector<std::pair<sc_dt::uint64, sc_dt::uint64>> m_ranges;
67 std::thread::id m_thread_id;
72 using TlmInitiatorSocket = tlm::tlm_initiator_socket<BUSWIDTH, tlm::tlm_base_protocol_types, 1,
73 sc_core::SC_ZERO_OR_MORE_BOUND>;
74 using TlmPayload = tlm::tlm_generic_payload;
75 using MemTxResult = qemu::MemoryRegionOps::MemTxResult;
78 using DmiRegionAliasKey = uint64_t;
82 std::unique_ptr<QemuInstanceHintTlmExtension> m_instance_hint;
87 std::atomic<bool> m_finished =
false;
90 std::shared_ptr<qemu::AddressSpace> m_as;
91 std::shared_ptr<qemu::MemoryListener> m_listener;
96 std::shared_ptr<qemu::MemoryRegion> m_root;
98 m_mem_obj(std::shared_ptr<qemu::MemoryRegion> memory): m_root(std::move(memory)) {}
102 std::map<DmiRegionAliasKey, DmiRegionAlias::Ptr> m_dmi_aliases;
103 using AliasesIterator = std::map<DmiRegionAliasKey, DmiRegionAlias::Ptr>::iterator;
106 std::set<std::pair<uint64_t, uintptr_t>> m_qemu_mr_aliases;
107 std::function<void(sc_dt::uint64, sc_dt::uint64)> m_invalidate_observer;
109 void init_payload(TlmPayload& trans, tlm::tlm_command command, uint64_t addr, uint64_t* val,
unsigned int size)
111 trans.set_command(command);
112 trans.set_address(addr);
113 trans.set_data_ptr(
reinterpret_cast<unsigned char*
>(val));
114 trans.set_data_length(size);
115 trans.set_streaming_width(size);
116 trans.set_byte_enable_length(0);
117 trans.set_dmi_allowed(
false);
118 trans.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
119 trans.set_extension(m_instance_hint.get());
120 if (m_initiator !=
nullptr) m_initiator->initiator_customize_tlm_payload(trans);
124 void add_dmi_mr_alias(DmiRegionAlias::Ptr alias)
126 SCP_INFO(()) <<
"Adding " << *alias;
128 m_r->m_root->add_subregion(alias_mr, alias->get_start());
129 alias->set_installed();
132 void del_dmi_mr_alias(
const DmiRegionAlias::Ptr alias)
134 if (!alias->is_installed()) {
137 SCP_INFO(()) <<
"Removing " << *alias;
138 m_r->m_root->del_subregion(alias->get_alias_mr());
141 tlm::tlm_dmi check_dmi_hint_locked(TlmPayload& trans)
143 assert(trans.is_dmi_allowed());
144 tlm::tlm_dmi dmi_data;
145 bool dmi_valid =
false;
146 const bool iothread_locked = m_inst->get().is_iothread_locked();
147 if (iothread_locked) m_inst->get().unlock_iothread();
148 const bool completed = m_on_sysc.run_on_sysc([&] {
149 assert(m_on_sysc.is_on_sysc());
150 dmi_valid = (*this)->get_direct_mem_ptr(trans, dmi_data);
152 if (iothread_locked) m_inst->get().lock_iothread();
154 if (!completed || !dmi_valid)
return dmi_data;
157 if (
const auto* shm_ext = trans.get_extension<gs::ShmemIDExtension>()) shm_fd = shm_ext->m_fd;
159 const uint64_t start = dmi_data.get_start_address();
160 const uint64_t end = dmi_data.get_end_address();
161 if (start > end)
return tlm::tlm_dmi{};
163 const bool topology_locked = m_inst->get().is_iothread_locked();
164 if (!topology_locked) m_inst->get().lock_iothread();
166 std::lock_guard<std::mutex> lock(m_mutex);
167 constexpr size_t max_map = 250;
168 if (m_dmi_aliases.size() > max_map) SCP_FATAL(())(
"Too many DMI regions requested");
169 if (0 == m_dmi_aliases.count(start)) {
170 SCP_INFO(()) <<
"Adding DMI for range [0x" << std::hex << dmi_data.get_start_address() <<
"-0x"
171 << dmi_data.get_end_address() <<
"]";
172 qemu::RcuReadLock rcu_read_lock = m_inst->get().rcu_read_lock_new();
173 DmiRegionAlias::Ptr alias = m_inst->get_dmi_manager().get_new_region_alias(dmi_data, shm_fd);
174 m_dmi_aliases[start] = alias;
175 add_dmi_mr_alias(m_dmi_aliases[start]);
177 SCP_INFO(())(
"Already have DMI for 0x{:x}", start);
180 if (!topology_locked) m_inst->get().unlock_iothread();
184 bool check_qemu_mr_hint(TlmPayload& trans)
186 QemuMrHintTlmExtension* ext =
nullptr;
187 uint64_t mapping_addr;
189 trans.get_extension(ext);
191 if (ext ==
nullptr) {
195 qemu::MemoryRegion target_mr(ext->get_mr());
197 if (m_r ==
nullptr || !target_mr.same_inst_as(*m_r->m_root)) {
201 if (ext->get_offset() > trans.get_address())
return false;
202 mapping_addr = trans.get_address() - ext->get_offset();
204 const uint64_t target_size = target_mr.get_size();
206 const uintptr_t target_mr_id =
reinterpret_cast<uintptr_t
>(target_mr.get_qemu_obj());
207 const bool topology_locked = m_inst->get().is_iothread_locked();
208 if (!topology_locked) m_inst->get().lock_iothread();
210 std::lock_guard<std::mutex> lock(m_mutex);
211 if (m_qemu_mr_aliases.emplace(mapping_addr, target_mr_id).second) {
212 auto mr = m_inst->get().template object_new<qemu::MemoryRegion>();
213 mr.init_alias(*m_r->m_root,
"mr-alias", target_mr, 0, target_size);
214 m_r->m_root->add_subregion(mr, mapping_addr);
217 if (!topology_locked) m_inst->get().unlock_iothread();
221 void do_regular_access(TlmPayload& trans)
223 using sc_core::sc_time;
225 uint64_t addr = trans.get_address();
226 sc_time now = m_initiator !=
nullptr ? m_initiator->initiator_get_local_time() : sc_core::sc_time_stamp();
228 m_inst->get().unlock_iothread();
229 m_on_sysc.run_on_sysc([
this, &trans, &now] { (*this)->b_transport(trans, now); });
230 m_inst->get().lock_iothread();
235 trans.set_address(addr);
236 if (!check_qemu_mr_hint(trans)) {
237 if (trans.is_dmi_allowed()) check_dmi_hint_locked(trans);
240 if (m_initiator !=
nullptr) m_initiator->initiator_set_local_time(now);
243 void do_debug_access(TlmPayload& trans)
245 m_inst->get().unlock_iothread();
246 m_on_sysc.run_on_sysc([
this, &trans] { (*this)->transport_dbg(trans); });
247 m_inst->get().lock_iothread();
250 void do_direct_access(TlmPayload& trans)
252 sc_core::sc_time now = m_initiator !=
nullptr ? m_initiator->initiator_get_local_time()
253 : sc_core::sc_time_stamp();
254 (*this)->b_transport(trans, now);
257 MemTxResult qemu_io_access(tlm::tlm_command command, uint64_t addr, uint64_t* val,
unsigned int size,
261 if (m_finished || m_inst ==
nullptr)
return qemu::MemoryRegionOps::MemTxError;
263 init_payload(trans, command, addr, val, size);
264 QemuMemTxAttrExtensions attr_exts(trans, attrs);
266 if (trans.get_extension<ExclusiveAccessTlmExtension>()) {
270 do_direct_access(trans);
272 std::unique_lock<std::recursive_mutex> io_lock(m_inst->g_rec_qemu_io_lock, std::try_to_lock);
273 if (!io_lock.owns_lock() && !is_on_sysc()) {
279 m_inst->get().unlock_iothread();
281 m_inst->get().lock_iothread();
286 if (reentrancy > 1) {
287 do_direct_access(trans);
288 }
else if (attrs.debug) {
289 do_debug_access(trans);
291 do_regular_access(trans);
296 trans.clear_extension(m_instance_hint.get());
297 if (m_initiator !=
nullptr) m_initiator->initiator_tidy_tlm_payload(trans);
299 switch (trans.get_response_status()) {
300 case tlm::TLM_OK_RESPONSE:
301 return qemu::MemoryRegionOps::MemTxOK;
302 case tlm::TLM_ADDRESS_ERROR_RESPONSE:
303 return qemu::MemoryRegionOps::MemTxDecodeError;
305 return qemu::MemoryRegionOps::MemTxError;
310 MemTxResult qemu_io_read(uint64_t addr, uint64_t* val,
unsigned int size, MemTxAttrs attrs)
312 return qemu_io_access(tlm::TLM_READ_COMMAND, addr, val, size, attrs);
315 MemTxResult qemu_io_write(uint64_t addr, uint64_t val,
unsigned int size, MemTxAttrs attrs)
317 return qemu_io_access(tlm::TLM_WRITE_COMMAND, addr, &val, size, attrs);
320 qemu::MemoryRegionOpsPtr create_io_ops()
322 using namespace std::placeholders;
325 qemu::LibQemu& inst = m_inst->get();
326 qemu::MemoryRegionOpsPtr ops = inst.memory_region_ops_new();
327 ops->set_read_callback(std::bind(&QemuInitiatorSocket::qemu_io_read,
this, _1, _2, _3, _4));
328 ops->set_write_callback(std::bind(&QemuInitiatorSocket::qemu_io_write,
this, _1, _2, _3, _4));
329 ops->set_max_access_size(8);
333 bool is_on_sysc()
const {
return std::this_thread::get_id() == m_thread_id; }
335 QemuInitiatorSocket(
const char* name, QemuInstance* inst =
nullptr, QemuInitiatorIface* initiator =
nullptr)
336 : TlmInitiatorSocket(name)
337 , m_thread_id(std::this_thread::get_id())
339 , m_initiator(initiator)
340 , m_on_sysc(sc_core::sc_gen_unique_name(
"initiator_run_on_sysc"))
342 SCP_DEBUG(()) <<
"QemuInitiatorSocket constructor";
343 TlmInitiatorSocket::bind(*
static_cast<tlm::tlm_bw_transport_if<>*
>(
this));
344 if (inst !=
nullptr) m_instance_hint = std::make_unique<QemuInstanceHintTlmExtension>(*inst);
347 QemuInitiatorSocket(
const char* name, QemuInitiatorIface& initiator, QemuInstance& inst)
348 : QemuInitiatorSocket(name, &inst, &initiator)
352 void set_qemu_inst(QemuInstance& inst)
354 m_instance_hint = std::make_unique<QemuInstanceHintTlmExtension>(inst);
358 void set_cpu(qemu::Cpu cpu)
360 if (!m_cpu.valid()) m_cpu = cpu;
363 void set_invalidate_observer(std::function<
void(sc_dt::uint64, sc_dt::uint64)> observer)
365 m_invalidate_observer = std::move(observer);
368 void init(qemu::Device& dev,
const char* prop)
370 qemu::LibQemu& inst = m_inst->get();
372 qemu::MemoryRegionOpsPtr ops = create_io_ops();
373 m_r->m_root->init_io(dev, TlmInitiatorSocket::name(), std::numeric_limits<uint64_t>::max(), ops);
374 dev.set_prop_link(prop, *m_r->m_root);
377 void init(qemu::IOMMUMemoryRegion& region)
380 assert(m_r ==
nullptr);
382 m_r =
new m_mem_obj(std::make_shared<qemu::MemoryRegion>(region.m_root_io));
383 m_as = region.m_as_io;
386 qemu::AddressSpace& get_address_space()
392 void end_of_simulation() { m_finished =
true; }
394 ~QemuInitiatorSocket()
401 m_r->m_root->removeSubRegions();
410 void qemu_map(qemu::MemoryListener& listener, uint64_t addr, uint64_t len)
413 if (m_finished)
return;
415 SCP_DEBUG(()) <<
"Mapping request for address [0x" << std::hex << addr <<
"-0x" << addr + len - 1 <<
"]";
418 uint64_t current_addr = addr;
420 init_payload(trans, tlm::TLM_IGNORE_COMMAND, current_addr, &temp, 0);
421 trans.set_dmi_allowed(
true);
423 while (current_addr < addr + len) {
424 tlm::tlm_dmi dmi_data = check_dmi_hint_locked(trans);
429 SCP_INFO(()) <<
"0x" << std::hex << current_addr <<
" mapped [0x" << dmi_data.get_start_address() <<
"-0x"
430 << dmi_data.get_end_address() <<
"]";
433 assert(dmi_data.get_end_address() >= current_addr);
434 current_addr = dmi_data.get_end_address();
435 if (current_addr >= addr + len)
break;
437 trans.set_address(current_addr);
440 trans.clear_extension(m_instance_hint.get());
441 if (m_initiator !=
nullptr) m_initiator->initiator_tidy_tlm_payload(trans);
444 void init_global(qemu::Device& dev)
446 using namespace std::placeholders;
448 qemu::LibQemu& inst = m_inst->get();
449 qemu::MemoryRegionOpsPtr ops = create_io_ops();
450 auto system_memory = inst.get_system_memory();
451 system_memory->init_io(dev, TlmInitiatorSocket::name(), std::numeric_limits<uint64_t>::max() - 1, ops);
452 m_r =
new m_mem_obj(std::move(system_memory));
454 m_as = inst.address_space_get_system_memory();
457 m_as->update_topology();
463 m_listener = inst.memory_listener_new();
464 m_listener->set_map_callback(std::bind(&QemuInitiatorSocket::qemu_map,
this, _1, _2, _3));
465 m_listener->register_as(m_as);
469 virtual tlm::tlm_sync_enum nb_transport_bw(tlm::tlm_generic_payload& trans, tlm::tlm_phase& phase,
474 return tlm::TLM_COMPLETED;
477 virtual AliasesIterator remove_alias(AliasesIterator it)
479 DmiRegionAlias::Ptr r = it->second;
481 return m_dmi_aliases.erase(it);
485 void invalidate_single_range(sc_dt::uint64 start_range, sc_dt::uint64 end_range)
487 auto it = m_dmi_aliases.upper_bound(start_range);
489 if (it != m_dmi_aliases.begin()) {
496 while (it != m_dmi_aliases.end()) {
497 DmiRegionAlias::Ptr r = it->second;
499 if (r->get_start() > end_range) {
504 if (r->get_end() < start_range) {
510 it = remove_alias(it);
512 SCP_DEBUG(()) <<
"Invalidated region [0x" << std::hex << r->get_start() <<
", 0x" << std::hex
513 << r->get_end() <<
"]";
517 void invalidate_ranges_safe_cb()
519 if (m_finished)
return;
520 std::lock_guard<std::mutex> lock(m_mutex);
522 SCP_DEBUG(()) <<
"Invalidating " << m_ranges.size() <<
" ranges";
523 auto rit = m_ranges.begin();
524 while (rit != m_ranges.end()) {
525 invalidate_single_range(rit->first, rit->second);
526 rit = m_ranges.erase(rit);
531 virtual void invalidate_direct_mem_ptr(sc_dt::uint64 start_range, sc_dt::uint64 end_range)
533 if (m_finished)
return;
534 if (m_invalidate_observer) m_invalidate_observer(start_range, end_range);
535 SCP_DEBUG(()) <<
"DMI invalidate [0x" << std::hex << start_range <<
", 0x" << std::hex << end_range <<
"]";
536 if (!m_inst || !m_cpu.valid())
return;
537 std::lock_guard<std::mutex> lock(m_mutex);
538 m_ranges.push_back(std::make_pair(start_range, end_range));
539 m_cpu.async_run([
this] { invalidate_ranges_safe_cb(); });
544 std::lock_guard<std::mutex> lock(m_mutex);
545 auto it = m_dmi_aliases.begin();
546 while (it != m_dmi_aliases.end()) {
547 it = remove_alias(it);