qualcomm/qbox
Loading...
Searching...
No Matches
initiator.h
1/*
2 * This file is part of libqbox
3 * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All Rights Reserved.
4 * Author: GreenSocs 2021
5 *
6 * SPDX-License-Identifier: BSD-3-Clause
7 */
8
9#ifndef _LIBQBOX_PORTS_INITIATOR_H
10#define _LIBQBOX_PORTS_INITIATOR_H
11
12#include <functional>
13#include <limits>
14#include <cassert>
15#include <cinttypes>
16#include <set>
17#include <utility>
18
19#include <tlm>
20
21#include <libqemu-cxx/libqemu-cxx.h>
22
23#include <libgssync.h>
24
25#include <scp/report.h>
26
27#include <qemu-instance.h>
28#include <tlm-extensions/qemu-cpu-hint.h>
29#include <tlm-extensions/qemu-instance-hint.h>
30#include <tlm-extensions/qemu-mr-hint.h>
31#include <tlm-extensions/exclusive-access.h>
32#include <tlm-extensions/shmem_extension.h>
33#include <tlm-extensions/qemu-memtx-attrs.h>
34#include <tlm-extensions/qemu-memtx-attr-extensions.h>
35#include <tlm_sockets_buswidth.h>
36
38{
39public:
40 using TlmPayload = tlm::tlm_generic_payload;
41
42 virtual void initiator_customize_tlm_payload(TlmPayload& payload) = 0;
43 virtual void initiator_tidy_tlm_payload(TlmPayload& payload) = 0;
44 virtual sc_core::sc_time initiator_get_local_time() = 0;
45 virtual void initiator_set_local_time(const sc_core::sc_time&) = 0;
46 virtual void initiator_async_run(qemu::Cpu::AsyncJobFn job) = 0;
47};
48
59template <unsigned int BUSWIDTH = DEFAULT_TLM_BUSWIDTH>
60class QemuInitiatorSocket
61 : public tlm::tlm_initiator_socket<BUSWIDTH, tlm::tlm_base_protocol_types, 1, sc_core::SC_ZERO_OR_MORE_BOUND>,
62 public tlm::tlm_bw_transport_if<>
63{
64private:
65 std::mutex m_mutex;
66 std::vector<std::pair<sc_dt::uint64, sc_dt::uint64>> m_ranges;
67 std::thread::id m_thread_id;
68
69public:
70 SCP_LOGGER(());
71
72 using TlmInitiatorSocket = tlm::tlm_initiator_socket<BUSWIDTH, tlm::tlm_base_protocol_types, 1,
73 sc_core::SC_ZERO_OR_MORE_BOUND>;
74 using TlmPayload = tlm::tlm_generic_payload;
75 using MemTxResult = qemu::MemoryRegionOps::MemTxResult;
76 using MemTxAttrs = qemu::MemoryRegionOps::MemTxAttrs;
77 using DmiRegionAlias = QemuInstanceDmiManager::DmiRegionAlias;
78 using DmiRegionAliasKey = uint64_t;
79
80protected:
81 QemuInstance* m_inst = nullptr;
82 std::unique_ptr<QemuInstanceHintTlmExtension> m_instance_hint;
83 QemuInitiatorIface* m_initiator = nullptr;
84 gs::runonsysc m_on_sysc;
85 int reentrancy = 0;
86
87 std::atomic<bool> m_finished = false;
88
89 qemu::Cpu m_cpu;
90 std::shared_ptr<qemu::AddressSpace> m_as;
91 std::shared_ptr<qemu::MemoryListener> m_listener;
92
93 class m_mem_obj
94 {
95 public:
96 std::shared_ptr<qemu::MemoryRegion> m_root;
97 m_mem_obj(qemu::LibQemu& inst) { m_root.reset(new qemu::MemoryRegion(inst.object_new<qemu::MemoryRegion>())); }
98 m_mem_obj(std::shared_ptr<qemu::MemoryRegion> memory): m_root(std::move(memory)) {}
99 };
100 m_mem_obj* m_r = nullptr;
101
102 std::map<DmiRegionAliasKey, DmiRegionAlias::Ptr> m_dmi_aliases;
103 using AliasesIterator = std::map<DmiRegionAliasKey, DmiRegionAlias::Ptr>::iterator;
104
105 // Do not add the same target MR twice at one address.
106 std::set<std::pair<uint64_t, uintptr_t>> m_qemu_mr_aliases;
107 std::function<void(sc_dt::uint64, sc_dt::uint64)> m_invalidate_observer;
108
109 void init_payload(TlmPayload& trans, tlm::tlm_command command, uint64_t addr, uint64_t* val, unsigned int size)
110 {
111 trans.set_command(command);
112 trans.set_address(addr);
113 trans.set_data_ptr(reinterpret_cast<unsigned char*>(val));
114 trans.set_data_length(size);
115 trans.set_streaming_width(size);
116 trans.set_byte_enable_length(0);
117 trans.set_dmi_allowed(false);
118 trans.set_response_status(tlm::TLM_INCOMPLETE_RESPONSE);
119 trans.set_extension(m_instance_hint.get());
120 if (m_initiator != nullptr) m_initiator->initiator_customize_tlm_payload(trans);
121 if (const auto* cpu_hint = trans.get_extension<QemuCpuHintTlmExtension>()) set_cpu(cpu_hint->get_cpu());
122 }
123
124 void add_dmi_mr_alias(DmiRegionAlias::Ptr alias)
125 {
126 SCP_INFO(()) << "Adding " << *alias;
127 qemu::MemoryRegion alias_mr = alias->get_alias_mr();
128 m_r->m_root->add_subregion(alias_mr, alias->get_start());
129 alias->set_installed();
130 }
131
132 void del_dmi_mr_alias(const DmiRegionAlias::Ptr alias)
133 {
134 if (!alias->is_installed()) {
135 return;
136 }
137 SCP_INFO(()) << "Removing " << *alias;
138 m_r->m_root->del_subregion(alias->get_alias_mr());
139 }
140
141 tlm::tlm_dmi check_dmi_hint_locked(TlmPayload& trans)
142 {
143 assert(trans.is_dmi_allowed());
144 tlm::tlm_dmi dmi_data;
145 bool dmi_valid = false;
146 const bool iothread_locked = m_inst->get().is_iothread_locked();
147 if (iothread_locked) m_inst->get().unlock_iothread();
148 const bool completed = m_on_sysc.run_on_sysc([&] {
149 assert(m_on_sysc.is_on_sysc());
150 dmi_valid = (*this)->get_direct_mem_ptr(trans, dmi_data);
151 });
152 if (iothread_locked) m_inst->get().lock_iothread();
153
154 if (!completed || !dmi_valid) return dmi_data;
155
156 int shm_fd = -1;
157 if (const auto* shm_ext = trans.get_extension<gs::ShmemIDExtension>()) shm_fd = shm_ext->m_fd;
158
159 const uint64_t start = dmi_data.get_start_address();
160 const uint64_t end = dmi_data.get_end_address();
161 if (start > end) return tlm::tlm_dmi{};
162
163 const bool topology_locked = m_inst->get().is_iothread_locked();
164 if (!topology_locked) m_inst->get().lock_iothread();
165 {
166 std::lock_guard<std::mutex> lock(m_mutex);
167 constexpr size_t max_map = 250;
168 if (m_dmi_aliases.size() > max_map) SCP_FATAL(())("Too many DMI regions requested");
169 if (0 == m_dmi_aliases.count(start)) {
170 SCP_INFO(()) << "Adding DMI for range [0x" << std::hex << dmi_data.get_start_address() << "-0x"
171 << dmi_data.get_end_address() << "]";
172 qemu::RcuReadLock rcu_read_lock = m_inst->get().rcu_read_lock_new();
173 DmiRegionAlias::Ptr alias = m_inst->get_dmi_manager().get_new_region_alias(dmi_data, shm_fd);
174 m_dmi_aliases[start] = alias;
175 add_dmi_mr_alias(m_dmi_aliases[start]);
176 } else {
177 SCP_INFO(())("Already have DMI for 0x{:x}", start);
178 }
179 }
180 if (!topology_locked) m_inst->get().unlock_iothread();
181 return dmi_data;
182 }
183
184 bool check_qemu_mr_hint(TlmPayload& trans)
185 {
186 QemuMrHintTlmExtension* ext = nullptr;
187 uint64_t mapping_addr;
188
189 trans.get_extension(ext);
190
191 if (ext == nullptr) {
192 return false;
193 }
194
195 qemu::MemoryRegion target_mr(ext->get_mr());
196
197 if (m_r == nullptr || !target_mr.same_inst_as(*m_r->m_root)) {
198 return false;
199 }
200
201 if (ext->get_offset() > trans.get_address()) return false;
202 mapping_addr = trans.get_address() - ext->get_offset();
203
204 const uint64_t target_size = target_mr.get_size();
205
206 const uintptr_t target_mr_id = reinterpret_cast<uintptr_t>(target_mr.get_qemu_obj());
207 const bool topology_locked = m_inst->get().is_iothread_locked();
208 if (!topology_locked) m_inst->get().lock_iothread();
209 {
210 std::lock_guard<std::mutex> lock(m_mutex);
211 if (m_qemu_mr_aliases.emplace(mapping_addr, target_mr_id).second) {
212 auto mr = m_inst->get().template object_new<qemu::MemoryRegion>();
213 mr.init_alias(*m_r->m_root, "mr-alias", target_mr, 0, target_size);
214 m_r->m_root->add_subregion(mr, mapping_addr);
215 }
216 }
217 if (!topology_locked) m_inst->get().unlock_iothread();
218 return true;
219 }
220
221 void do_regular_access(TlmPayload& trans)
222 {
223 using sc_core::sc_time;
224
225 uint64_t addr = trans.get_address();
226 sc_time now = m_initiator != nullptr ? m_initiator->initiator_get_local_time() : sc_core::sc_time_stamp();
227
228 m_inst->get().unlock_iothread();
229 m_on_sysc.run_on_sysc([this, &trans, &now] { (*this)->b_transport(trans, now); });
230 m_inst->get().lock_iothread();
231 /*
232 * Reset transaction address before dmi check (could be altered by
233 * b_transport).
234 */
235 trans.set_address(addr);
236 if (!check_qemu_mr_hint(trans)) {
237 if (trans.is_dmi_allowed()) check_dmi_hint_locked(trans);
238 }
239
240 if (m_initiator != nullptr) m_initiator->initiator_set_local_time(now);
241 }
242
243 void do_debug_access(TlmPayload& trans)
244 {
245 m_inst->get().unlock_iothread();
246 m_on_sysc.run_on_sysc([this, &trans] { (*this)->transport_dbg(trans); });
247 m_inst->get().lock_iothread();
248 }
249
250 void do_direct_access(TlmPayload& trans)
251 {
252 sc_core::sc_time now = m_initiator != nullptr ? m_initiator->initiator_get_local_time()
253 : sc_core::sc_time_stamp();
254 (*this)->b_transport(trans, now);
255 }
256
257 MemTxResult qemu_io_access(tlm::tlm_command command, uint64_t addr, uint64_t* val, unsigned int size,
258 MemTxAttrs attrs)
259 {
260 TlmPayload trans;
261 if (m_finished || m_inst == nullptr) return qemu::MemoryRegionOps::MemTxError;
262
263 init_payload(trans, command, addr, val, size);
264 QemuMemTxAttrExtensions attr_exts(trans, attrs);
265
266 if (trans.get_extension<ExclusiveAccessTlmExtension>()) {
267 /* in the case of an exclusive access keep the iolock (and assume NO side-effects)
268 * clearly dangerous, but exclusives are not guaranteed to work on IO space anyway
269 */
270 do_direct_access(trans);
271 } else {
272 std::unique_lock<std::recursive_mutex> io_lock(m_inst->g_rec_qemu_io_lock, std::try_to_lock);
273 if (!io_lock.owns_lock() && !is_on_sysc()) {
274 /* Allow only a single access, but handle re-entrant code,
275 * while allowing side-effects in SystemC (e.g. calling wait)
276 * [NB re-entrant code caused via memory listeners to
277 * creation of memory regions (due to DMI) in some models]
278 */
279 m_inst->get().unlock_iothread();
280 io_lock.lock();
281 m_inst->get().lock_iothread();
282 }
283 reentrancy++;
284
285 /* Force re-entrant code to use a direct access (safe for reentrancy with no side effects) */
286 if (reentrancy > 1) {
287 do_direct_access(trans);
288 } else if (attrs.debug) {
289 do_debug_access(trans);
290 } else {
291 do_regular_access(trans);
292 }
293
294 reentrancy--;
295 }
296 trans.clear_extension(m_instance_hint.get());
297 if (m_initiator != nullptr) m_initiator->initiator_tidy_tlm_payload(trans);
298
299 switch (trans.get_response_status()) {
300 case tlm::TLM_OK_RESPONSE:
301 return qemu::MemoryRegionOps::MemTxOK;
302 case tlm::TLM_ADDRESS_ERROR_RESPONSE:
303 return qemu::MemoryRegionOps::MemTxDecodeError;
304 default:
305 return qemu::MemoryRegionOps::MemTxError;
306 }
307 }
308
309public:
310 MemTxResult qemu_io_read(uint64_t addr, uint64_t* val, unsigned int size, MemTxAttrs attrs)
311 {
312 return qemu_io_access(tlm::TLM_READ_COMMAND, addr, val, size, attrs);
313 }
314
315 MemTxResult qemu_io_write(uint64_t addr, uint64_t val, unsigned int size, MemTxAttrs attrs)
316 {
317 return qemu_io_access(tlm::TLM_WRITE_COMMAND, addr, &val, size, attrs);
318 }
319
320 qemu::MemoryRegionOpsPtr create_io_ops()
321 {
322 using namespace std::placeholders;
323
324 assert(m_inst);
325 qemu::LibQemu& inst = m_inst->get();
326 qemu::MemoryRegionOpsPtr ops = inst.memory_region_ops_new();
327 ops->set_read_callback(std::bind(&QemuInitiatorSocket::qemu_io_read, this, _1, _2, _3, _4));
328 ops->set_write_callback(std::bind(&QemuInitiatorSocket::qemu_io_write, this, _1, _2, _3, _4));
329 ops->set_max_access_size(8);
330 return ops;
331 }
332
333 bool is_on_sysc() const { return std::this_thread::get_id() == m_thread_id; }
334
335 QemuInitiatorSocket(const char* name, QemuInstance* inst = nullptr, QemuInitiatorIface* initiator = nullptr)
336 : TlmInitiatorSocket(name)
337 , m_thread_id(std::this_thread::get_id())
338 , m_inst(inst)
339 , m_initiator(initiator)
340 , m_on_sysc(sc_core::sc_gen_unique_name("initiator_run_on_sysc"))
341 {
342 SCP_DEBUG(()) << "QemuInitiatorSocket constructor";
343 TlmInitiatorSocket::bind(*static_cast<tlm::tlm_bw_transport_if<>*>(this));
344 if (inst != nullptr) m_instance_hint = std::make_unique<QemuInstanceHintTlmExtension>(*inst);
345 }
346
347 QemuInitiatorSocket(const char* name, QemuInitiatorIface& initiator, QemuInstance& inst)
348 : QemuInitiatorSocket(name, &inst, &initiator)
349 {
350 }
351
352 void set_qemu_inst(QemuInstance& inst)
353 {
354 m_instance_hint = std::make_unique<QemuInstanceHintTlmExtension>(inst);
355 m_inst = &inst;
356 }
357
358 void set_cpu(qemu::Cpu cpu)
359 {
360 if (!m_cpu.valid()) m_cpu = cpu;
361 }
362
363 void set_invalidate_observer(std::function<void(sc_dt::uint64, sc_dt::uint64)> observer)
364 {
365 m_invalidate_observer = std::move(observer);
366 }
367
368 void init(qemu::Device& dev, const char* prop)
369 {
370 qemu::LibQemu& inst = m_inst->get();
371 m_r = new m_mem_obj(inst);
372 qemu::MemoryRegionOpsPtr ops = create_io_ops();
373 m_r->m_root->init_io(dev, TlmInitiatorSocket::name(), std::numeric_limits<uint64_t>::max(), ops);
374 dev.set_prop_link(prop, *m_r->m_root);
375 }
376
377 void init(qemu::IOMMUMemoryRegion& region)
378 {
379 assert(m_inst);
380 assert(m_r == nullptr);
381 assert(!m_as);
382 m_r = new m_mem_obj(std::make_shared<qemu::MemoryRegion>(region.m_root_io));
383 m_as = region.m_as_io;
384 }
385
386 qemu::AddressSpace& get_address_space()
387 {
388 assert(m_as);
389 return *m_as;
390 }
391
392 void end_of_simulation() { m_finished = true; }
393
394 ~QemuInitiatorSocket()
395 {
396#if 0
397 // This could happen during void end_of_simulation() but there is a race with other units trying
398 // to pull down their DMI's
399 if (m_r) {
400 if (m_r->m_root) {
401 m_r->m_root->removeSubRegions();
402 }
403 delete m_r;
404 m_r = nullptr;
405 }
406#endif
407 // dmimgr_unlock();
408 }
409
410 void qemu_map(qemu::MemoryListener& listener, uint64_t addr, uint64_t len)
411 {
412 // this function is relatively expensive, and called a lot, it should be done a different way and removed.
413 if (m_finished) return;
414
415 SCP_DEBUG(()) << "Mapping request for address [0x" << std::hex << addr << "-0x" << addr + len - 1 << "]";
416
417 TlmPayload trans;
418 uint64_t current_addr = addr;
419 uint64_t temp;
420 init_payload(trans, tlm::TLM_IGNORE_COMMAND, current_addr, &temp, 0);
421 trans.set_dmi_allowed(true);
422
423 while (current_addr < addr + len) {
424 tlm::tlm_dmi dmi_data = check_dmi_hint_locked(trans);
425
426 // Current addr is an absolute address while the dmi range might be relative
427 // hence not necesseraly current_addr falls withing dmi_range address boundaries
428 // TODO: is there a way to retrieve the dmi range block offset?
429 SCP_INFO(()) << "0x" << std::hex << current_addr << " mapped [0x" << dmi_data.get_start_address() << "-0x"
430 << dmi_data.get_end_address() << "]";
431
432 // The allocated range may not span the whole length required for mapping
433 assert(dmi_data.get_end_address() >= current_addr);
434 current_addr = dmi_data.get_end_address();
435 if (current_addr >= addr + len) break; // Catch potential loop-rounds
436 current_addr += 1;
437 trans.set_address(current_addr);
438 }
439
440 trans.clear_extension(m_instance_hint.get());
441 if (m_initiator != nullptr) m_initiator->initiator_tidy_tlm_payload(trans);
442 }
443
444 void init_global(qemu::Device& dev)
445 {
446 using namespace std::placeholders;
447
448 qemu::LibQemu& inst = m_inst->get();
449 qemu::MemoryRegionOpsPtr ops = create_io_ops();
450 auto system_memory = inst.get_system_memory();
451 system_memory->init_io(dev, TlmInitiatorSocket::name(), std::numeric_limits<uint64_t>::max() - 1, ops);
452 m_r = new m_mem_obj(std::move(system_memory));
453
454 m_as = inst.address_space_get_system_memory();
455 // System memory has been changed from container to "io", this is relevant
456 // for flatview, and to reflect that we can just update the topology
457 m_as->update_topology();
458
459 /* Sometimes memory regions are added to the global address space by (for instance) virt devices rempaiing
460 * system memory such that they can directly access it. The mapped memory region needs it's DMI pointers
461 * adjusted. qemu_map will do that remapping of the DMI.
462 */
463 m_listener = inst.memory_listener_new();
464 m_listener->set_map_callback(std::bind(&QemuInitiatorSocket::qemu_map, this, _1, _2, _3));
465 m_listener->register_as(m_as);
466 }
467
468 /* tlm::tlm_bw_transport_if<> */
469 virtual tlm::tlm_sync_enum nb_transport_bw(tlm::tlm_generic_payload& trans, tlm::tlm_phase& phase,
470 sc_core::sc_time& t)
471 {
472 /* Should not be reached */
473 assert(false);
474 return tlm::TLM_COMPLETED;
475 }
476
477 virtual AliasesIterator remove_alias(AliasesIterator it)
478 {
479 DmiRegionAlias::Ptr r = it->second;
480 del_dmi_mr_alias(r);
481 return m_dmi_aliases.erase(it);
482 }
483
484private:
485 void invalidate_single_range(sc_dt::uint64 start_range, sc_dt::uint64 end_range)
486 {
487 auto it = m_dmi_aliases.upper_bound(start_range);
488
489 if (it != m_dmi_aliases.begin()) {
490 /*
491 * Start with the preceding region, as it may already cross the
492 * range we must invalidate.
493 */
494 it--;
495 }
496 while (it != m_dmi_aliases.end()) {
497 DmiRegionAlias::Ptr r = it->second;
498
499 if (r->get_start() > end_range) {
500 /* We've got out of the invalidation range */
501 break;
502 }
503
504 if (r->get_end() < start_range) {
505 /* We are not in yet */
506 it++;
507 continue;
508 }
509
510 it = remove_alias(it);
511
512 SCP_DEBUG(()) << "Invalidated region [0x" << std::hex << r->get_start() << ", 0x" << std::hex
513 << r->get_end() << "]";
514 }
515 }
516
517 void invalidate_ranges_safe_cb()
518 {
519 if (m_finished) return;
520 std::lock_guard<std::mutex> lock(m_mutex);
521
522 SCP_DEBUG(()) << "Invalidating " << m_ranges.size() << " ranges";
523 auto rit = m_ranges.begin();
524 while (rit != m_ranges.end()) {
525 invalidate_single_range(rit->first, rit->second);
526 rit = m_ranges.erase(rit);
527 }
528 }
529
530public:
531 virtual void invalidate_direct_mem_ptr(sc_dt::uint64 start_range, sc_dt::uint64 end_range)
532 {
533 if (m_finished) return;
534 if (m_invalidate_observer) m_invalidate_observer(start_range, end_range);
535 SCP_DEBUG(()) << "DMI invalidate [0x" << std::hex << start_range << ", 0x" << std::hex << end_range << "]";
536 if (!m_inst || !m_cpu.valid()) return;
537 std::lock_guard<std::mutex> lock(m_mutex);
538 m_ranges.push_back(std::make_pair(start_range, end_range));
539 m_cpu.async_run([this] { invalidate_ranges_safe_cb(); });
540 }
541
542 virtual void reset()
543 {
544 std::lock_guard<std::mutex> lock(m_mutex);
545 auto it = m_dmi_aliases.begin();
546 while (it != m_dmi_aliases.end()) {
547 it = remove_alias(it);
548 }
549 }
550};
551
552#endif
Definition qemu-cpu-hint.h:17
Definition initiator.h:38
Definition initiator.h:94
An alias to a DMI region.
Definition dmi-manager.h:140
This class encapsulates a libqemu-cxx qemu::LibQemu instance. It handles QEMU parameters and instance...
Definition qemu-instance.h:89
Definition runonsysc.h:23
Definition libqemu-cxx.h:725
Definition libqemu-cxx.h:95
Definition libqemu-cxx.h:396
Definition libqemu-cxx.h:352