// SPDX-FileCopyrightText: 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later #include #include #include #include #include #include #include #include "common/assert.h" #include "common/fiber.h" #include "video_core/control/scheduler.h" #include "video_core/dma_pusher.h" #include "video_core/gpu.h" namespace Tegra::Control { struct GPFifoContext { bool is_active; bool is_running; std::shared_ptr context; std::deque pending_work; std::mutex guard; s32 bind_id; std::shared_ptr info; size_t yield_count; size_t scheduled_count; }; struct Scheduler::SchedulerImpl { // Fifos std::map, std::greater> schedule_priority_queue; std::unordered_map channel_gpfifo_ids; std::deque gpfifos; std::deque free_fifos; // Scheduling std::mutex scheduling_guard; std::shared_ptr master_control; bool must_reschedule{}; GPFifoContext* current_fifo{}; }; Scheduler::Scheduler(GPU& gpu_) : gpu{gpu_} { impl = std::make_unique(); } Scheduler::~Scheduler() = default; void Scheduler::Init() { impl->master_control = Common::Fiber::ThreadToFiber(); } void Scheduler::Resume() { while (UpdateHighestPriorityChannel()) { impl->current_fifo->scheduled_count++; Common::Fiber::YieldTo(impl->master_control, *impl->current_fifo->context); } } bool Scheduler::UpdateHighestPriorityChannel() { std::scoped_lock lk(impl->scheduling_guard); // Clear needs to schedule state. impl->must_reschedule = false; // By default, we don't have a channel to schedule. impl->current_fifo = nullptr; // Check each level to see if we can schedule. for (auto& level : impl->schedule_priority_queue) { if (ScheduleLevel(level.second)) { return true; } } // Nothing to schedule. return false; } bool Scheduler::ScheduleLevel(std::list& queue) { bool found_anything = false; size_t min_schedule_count = std::numeric_limits::max(); for (auto id : queue) { auto& fifo = impl->gpfifos[id]; std::scoped_lock lk(fifo.guard); // With no pending work and nothing running, this channel can't be scheduled. if (fifo.pending_work.empty() && !fifo.is_running) { continue; } // Prioritize channels at current priority which have been run the least. if (fifo.scheduled_count > min_schedule_count) { continue; } // Try not to select the same channel we just yielded from. if (fifo.scheduled_count < fifo.yield_count) { fifo.scheduled_count++; continue; } // Update best selection. min_schedule_count = fifo.scheduled_count; impl->current_fifo = &fifo; found_anything = true; } return found_anything; } void Scheduler::ChangePriority(s32 channel_id, u32 new_priority) { std::scoped_lock lk(impl->scheduling_guard); // Ensure we are tracking this channel. auto fifo_it = impl->channel_gpfifo_ids.find(channel_id); if (fifo_it == impl->channel_gpfifo_ids.end()) { return; } // Get the fifo and update its priority. const size_t fifo_id = fifo_it->second; auto& fifo = impl->gpfifos[fifo_id]; const auto old_priority = std::exchange(fifo.info->priority, new_priority); // Create the new level if needed. impl->schedule_priority_queue.try_emplace(new_priority); // Remove the old level and add to the new level. impl->schedule_priority_queue[new_priority].push_back(fifo_id); impl->schedule_priority_queue[old_priority].remove_if( [fifo_id](size_t id) { return id == fifo_id; }); } void Scheduler::Yield() { ASSERT(impl->current_fifo != nullptr); // Set yield count higher impl->current_fifo->yield_count = impl->current_fifo->scheduled_count + 1; Common::Fiber::YieldTo(impl->current_fifo->context, *impl->master_control); gpu.BindChannel(impl->current_fifo->bind_id); } void Scheduler::CheckStatus() { { std::unique_lock lk(impl->scheduling_guard); // If no reschedule is needed, don't transfer control if (!impl->must_reschedule) { return; } } // Transfer control to the scheduler Common::Fiber::YieldTo(impl->current_fifo->context, *impl->master_control); gpu.BindChannel(impl->current_fifo->bind_id); } void Scheduler::Push(s32 channel, CommandList&& entries) { std::scoped_lock lk(impl->scheduling_guard); // Get and ensure we have this channel. auto it = impl->channel_gpfifo_ids.find(channel); ASSERT(it != impl->channel_gpfifo_ids.end()); auto gpfifo_id = it->second; auto& fifo = impl->gpfifos[gpfifo_id]; // Add the new new work to the channel. { std::scoped_lock lk2(fifo.guard); fifo.pending_work.emplace_back(std::move(entries)); } // If the current running FIFO is null or the one being pushed to then // just return if (impl->current_fifo == nullptr || impl->current_fifo == &fifo) { return; } // If the current fifo has higher or equal priority to the current fifo then return if (impl->current_fifo->info->priority >= fifo.info->priority) { return; } // Mark scheduler update as required. impl->must_reschedule = true; } void Scheduler::ChannelLoop(size_t gpfifo_id, s32 channel_id) { auto& fifo = impl->gpfifos[gpfifo_id]; auto* channel_state = fifo.info.get(); const auto SendToPuller = [&] { std::scoped_lock lk(fifo.guard); if (fifo.pending_work.empty()) { // Stop if no work available. fifo.is_running = false; return false; } // Otherwise, send work to puller and mark as running. CommandList&& entries = std::move(fifo.pending_work.front()); channel_state->dma_pusher->Push(std::move(entries)); fifo.pending_work.pop_front(); fifo.is_running = true; // Succeed. return true; }; // Inform the GPU about the current channel. gpu.BindChannel(channel_id); while (true) { while (SendToPuller()) { // Execute. channel_state->dma_pusher->DispatchCalls(); // Reschedule. CheckStatus(); } // Return to host execution when all work is completed. Common::Fiber::YieldTo(fifo.context, *impl->master_control); // Inform the GPU about the current channel. gpu.BindChannel(channel_id); } } void Scheduler::DeclareChannel(std::shared_ptr new_channel) { s32 channel = new_channel->bind_id; std::unique_lock lk(impl->scheduling_guard); size_t new_fifo_id; if (!impl->free_fifos.empty()) { new_fifo_id = impl->free_fifos.front(); impl->free_fifos.pop_front(); } else { new_fifo_id = impl->gpfifos.size(); impl->gpfifos.emplace_back(); } auto& new_fifo = impl->gpfifos[new_fifo_id]; impl->channel_gpfifo_ids[channel] = new_fifo_id; new_fifo.is_active = true; new_fifo.bind_id = channel; new_fifo.pending_work.clear(); new_fifo.info = new_channel; new_fifo.scheduled_count = 0; new_fifo.yield_count = 0; new_fifo.is_running = false; impl->schedule_priority_queue.try_emplace(new_channel->priority); impl->schedule_priority_queue[new_channel->priority].push_back(new_fifo_id); std::function callback = std::bind(&Scheduler::ChannelLoop, this, new_fifo_id, channel); new_fifo.context = std::make_shared(std::move(callback)); } } // namespace Tegra::Control