fix: reduce idle CPU wakeups and deferred-work polling
This commit is contained in:
@@ -440,6 +440,29 @@ namespace Drivers::Net::Wifi {
|
|||||||
static void ServiceAsync();
|
static void ServiceAsync();
|
||||||
static void ServiceRecovery();
|
static void ServiceRecovery();
|
||||||
|
|
||||||
|
bool HasDeferredWork() {
|
||||||
|
if (g_initPending.load(std::memory_order_acquire) &&
|
||||||
|
!g_initialized && Fs::Vfs::IsDriveRegistered(0)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (!g_iwx.Mmio) return false;
|
||||||
|
if (g_iwx.WorkPending) return true;
|
||||||
|
|
||||||
|
uint64_t now = Timekeeping::GetMilliseconds();
|
||||||
|
if (g_scanDeadline != 0 && now >= g_scanDeadline) return true;
|
||||||
|
// The MLME/WPA state machine owns sub-second retransmission timers in
|
||||||
|
// addition to the overall async deadline. Service it until ServiceAsync
|
||||||
|
// observes Connected/Failed/Idle and clears this flag.
|
||||||
|
if (g_asyncConnect) return true;
|
||||||
|
if (g_iwx.State == IwxFwState::Error && g_initialized &&
|
||||||
|
!g_recoveryGaveUp &&
|
||||||
|
(g_lastRecoveryMs == 0 ||
|
||||||
|
now - g_lastRecoveryMs >= RECOVERY_BACKOFF_MS)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
void ServiceEvents() {
|
void ServiceEvents() {
|
||||||
if (!g_iwx.Mmio) return;
|
if (!g_iwx.Mmio) return;
|
||||||
if (g_iwx.WorkPending) IwxProcessEvents();
|
if (g_iwx.WorkPending) IwxProcessEvents();
|
||||||
|
|||||||
@@ -21,6 +21,10 @@ namespace Drivers::Net::Wifi {
|
|||||||
// Steady-state event pump (RX ring, notifications). Idle-loop callback.
|
// Steady-state event pump (RX ring, notifications). Idle-loop callback.
|
||||||
void ServiceEvents();
|
void ServiceEvents();
|
||||||
|
|
||||||
|
// True when firmware initialization, an RX notification, an expired async
|
||||||
|
// deadline, or a due recovery attempt needs idle-context servicing.
|
||||||
|
bool HasDeferredWork();
|
||||||
|
|
||||||
bool IsInitialized();
|
bool IsInitialized();
|
||||||
bool IsPresent();
|
bool IsPresent();
|
||||||
|
|
||||||
|
|||||||
@@ -419,6 +419,14 @@ namespace Drivers::USB::Bluetooth {
|
|||||||
// ServiceEvents — steady-state event pump (idle loop)
|
// ServiceEvents — steady-state event pump (idle loop)
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
|
|
||||||
|
bool HasDeferredWork() {
|
||||||
|
if (g_initPending.load(std::memory_order_acquire) &&
|
||||||
|
!g_initialized && Fs::Vfs::IsDriveRegistered(0)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return g_initialized && Hci::HasPendingCommands();
|
||||||
|
}
|
||||||
|
|
||||||
void ServiceEvents() {
|
void ServiceEvents() {
|
||||||
if (!g_initialized) return;
|
if (!g_initialized) return;
|
||||||
if (Xhci::InPollContext()) return; // never nest under PollEvents
|
if (Xhci::InPollContext()) return; // never nest under PollEvents
|
||||||
|
|||||||
@@ -27,6 +27,11 @@ namespace Drivers::USB::Bluetooth {
|
|||||||
// (PollEvents/DrainEvents/ProcessPendingCommands all self-serialize).
|
// (PollEvents/DrainEvents/ProcessPendingCommands all self-serialize).
|
||||||
void ServiceEvents();
|
void ServiceEvents();
|
||||||
|
|
||||||
|
// True when boot-deferred initialization or queued HCI control work needs
|
||||||
|
// an idle-context service pass. USB receive events are signaled separately
|
||||||
|
// by xHCI and cause the dispatcher to service Bluetooth in the same pass.
|
||||||
|
bool HasDeferredWork();
|
||||||
|
|
||||||
// Query adapter state
|
// Query adapter state
|
||||||
bool IsInitialized();
|
bool IsInitialized();
|
||||||
uint8_t GetSlotId();
|
uint8_t GetSlotId();
|
||||||
|
|||||||
@@ -1719,6 +1719,11 @@ namespace Drivers::USB::Bluetooth::Hci {
|
|||||||
s_active.store(false, std::memory_order_release);
|
s_active.store(false, std::memory_order_release);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool HasPendingCommands() {
|
||||||
|
return g_pendingTail.load(std::memory_order_acquire) !=
|
||||||
|
g_pendingHead.load(std::memory_order_acquire);
|
||||||
|
}
|
||||||
|
|
||||||
bool WaitSecureSendResult(uint32_t timeoutMs, uint8_t* outResult, uint8_t* outStatus) {
|
bool WaitSecureSendResult(uint32_t timeoutMs, uint8_t* outResult, uint8_t* outStatus) {
|
||||||
uint64_t start = Timekeeping::GetMilliseconds();
|
uint64_t start = Timekeeping::GetMilliseconds();
|
||||||
while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
|
while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
|
||||||
|
|||||||
@@ -331,6 +331,7 @@ namespace Drivers::USB::Bluetooth::Hci {
|
|||||||
// real confirmed transfers. Call from top-level (e.g. the connect loop),
|
// real confirmed transfers. Call from top-level (e.g. the connect loop),
|
||||||
// NOT from an event handler -- event handlers only enqueue.
|
// NOT from an event handler -- event handlers only enqueue.
|
||||||
void ProcessPendingCommands();
|
void ProcessPendingCommands();
|
||||||
|
bool HasPendingCommands();
|
||||||
|
|
||||||
// ACL TX flow control: outstanding (un-acked) ACL packets, and the
|
// ACL TX flow control: outstanding (un-acked) ACL packets, and the
|
||||||
// controller's ACL buffer count (Number-Of-Completed-Packets credits). The
|
// controller's ACL buffer count (Number-Of-Completed-Packets credits). The
|
||||||
|
|||||||
@@ -218,6 +218,10 @@ namespace Smp {
|
|||||||
for (;;) {
|
for (;;) {
|
||||||
// Pick up thermal-governor frequency changes decided by the BSP.
|
// Pick up thermal-governor frequency changes decided by the BSP.
|
||||||
Hal::CpuPower::ApplyPolicyIfChanged();
|
Hal::CpuPower::ApplyPolicyIfChanged();
|
||||||
|
// Runnable work sends this AP a reschedule IPI. Keep its periodic
|
||||||
|
// scheduler tick masked for the entire idle-context pass so long
|
||||||
|
// firmware waits do not keep generating useless timer interrupts.
|
||||||
|
Timekeeping::ApicTimerEnterApIdle();
|
||||||
// Any idle core may run bounded USB/NIC bottom halves. Preserve
|
// Any idle core may run bounded USB/NIC bottom halves. Preserve
|
||||||
// the AP's ACPI/MWAIT idle selection after servicing them.
|
// the AP's ACPI/MWAIT idle selection after servicing them.
|
||||||
Timekeeping::ServiceDeferredWork();
|
Timekeeping::ServiceDeferredWork();
|
||||||
|
|||||||
@@ -1284,6 +1284,15 @@ namespace Sched {
|
|||||||
uint8_t* oldFpu = (oldSlot >= 0) ? processTable[oldSlot].fpuState : nullptr;
|
uint8_t* oldFpu = (oldSlot >= 0) ? processTable[oldSlot].fpuState : nullptr;
|
||||||
uint8_t* newFpu = processTable[next].fpuState;
|
uint8_t* newFpu = processTable[next].fpuState;
|
||||||
|
|
||||||
|
if (oldSlot < 0) {
|
||||||
|
// AP idle loops mask their local periodic timer. Rearm it before
|
||||||
|
// dispatching user work so preemption resumes with the process.
|
||||||
|
// Also pick up a thermal-governor policy epoch that may have
|
||||||
|
// changed while this CPU remained asleep without timer ticks.
|
||||||
|
Hal::CpuPower::ApplyPolicyIfChanged();
|
||||||
|
Timekeeping::ApicTimerLeaveApIdle();
|
||||||
|
}
|
||||||
|
|
||||||
LoadUserFsBase(cpu, processTable[next].fsBase);
|
LoadUserFsBase(cpu, processTable[next].fsBase);
|
||||||
|
|
||||||
// DO NOT release schedLock here! It is held across the context
|
// DO NOT release schedLock here! It is held across the context
|
||||||
|
|||||||
@@ -41,8 +41,8 @@ namespace Timekeeping {
|
|||||||
static constexpr uint32_t DIVIDE_BY_16 = 0x03;
|
static constexpr uint32_t DIVIDE_BY_16 = 0x03;
|
||||||
|
|
||||||
// The BSP keeps a 1 ms tick for timekeeping and sleep deadlines.
|
// The BSP keeps a 1 ms tick for timekeeping and sleep deadlines.
|
||||||
// APs use a coarser 10 ms scheduler tick to avoid waking idle cores
|
// Running APs use a 10 ms scheduler tick. Idle APs mask it entirely and
|
||||||
// 1000 times per second with no useful work to do.
|
// rely on reschedule IPIs, avoiding periodic package wakeups.
|
||||||
static constexpr uint32_t BSP_TICK_INTERVAL_MS = 1;
|
static constexpr uint32_t BSP_TICK_INTERVAL_MS = 1;
|
||||||
static constexpr uint32_t BSP_TIMER_HZ = 1000 / BSP_TICK_INTERVAL_MS;
|
static constexpr uint32_t BSP_TIMER_HZ = 1000 / BSP_TICK_INTERVAL_MS;
|
||||||
static constexpr uint32_t AP_TICK_INTERVAL_MS = 10;
|
static constexpr uint32_t AP_TICK_INTERVAL_MS = 10;
|
||||||
@@ -220,8 +220,28 @@ namespace Timekeeping {
|
|||||||
// identical. This avoids PIT contention during AP boot.
|
// identical. This avoids PIT contention during AP boot.
|
||||||
if (g_ticksPerMs == 0) return;
|
if (g_ticksPerMs == 0) return;
|
||||||
|
|
||||||
// Configure a coarser periodic timer on APs. The scheduler still gets
|
// Configure the 10 ms scheduler timer for running APs. Their idle loop
|
||||||
// a 10 ms time slice, but idle APs stop taking 1000 timer interrupts/sec.
|
// masks it after initialization and rearms it when dispatching work.
|
||||||
|
ProgramTimer(true, AP_TICK_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ApicTimerEnterApIdle() {
|
||||||
|
auto* cpu = Smp::GetCurrentCpuData();
|
||||||
|
if (cpu == nullptr || cpu->cpuIndex == 0 || g_ticksPerMs == 0) return;
|
||||||
|
|
||||||
|
uint32_t lvt = Hal::LocalApic::ReadRegister(Hal::LocalApic::REG_TIMER_LVT);
|
||||||
|
if ((lvt & LVT_MASKED) == 0) {
|
||||||
|
Hal::LocalApic::WriteRegister(Hal::LocalApic::REG_TIMER_LVT,
|
||||||
|
lvt | LVT_MASKED);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void ApicTimerLeaveApIdle() {
|
||||||
|
auto* cpu = Smp::GetCurrentCpuData();
|
||||||
|
if (cpu == nullptr || cpu->cpuIndex == 0 || g_ticksPerMs == 0) return;
|
||||||
|
|
||||||
|
// Reprogram the initial count as well as unmasking. A deep idle state
|
||||||
|
// may have stopped the local timer at an arbitrary point in its period.
|
||||||
ProgramTimer(true, AP_TICK_INTERVAL_MS);
|
ProgramTimer(true, AP_TICK_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -239,19 +259,24 @@ namespace Timekeeping {
|
|||||||
bool wasReserved = cpu->reservedForKernelWork;
|
bool wasReserved = cpu->reservedForKernelWork;
|
||||||
cpu->reservedForKernelWork = true;
|
cpu->reservedForKernelWork = true;
|
||||||
|
|
||||||
// Drain USB hot-plug deferred work from any idle core, not just the BSP.
|
// Drain USB work only when the MSI path has actually queued something.
|
||||||
if (Drivers::USB::Xhci::HasDeferredWork()) {
|
// Bluetooth shares this controller, so service its protocol queues in
|
||||||
|
// the same pass after xHCI has delivered completion callbacks.
|
||||||
|
bool usbWork = Drivers::USB::Xhci::HasDeferredWork();
|
||||||
|
if (usbWork) {
|
||||||
Drivers::USB::Xhci::ProcessDeferredWork();
|
Drivers::USB::Xhci::ProcessDeferredWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
// NIC hard IRQs only acknowledge/mask and queue RX work. Dispatching
|
// NIC hard IRQs only acknowledge/mask and queue RX work. Dispatching
|
||||||
// Ethernet/TCP/UDP here keeps process-context IPC mutexes out of IRQs.
|
// Ethernet/TCP/UDP here keeps process-context IPC mutexes out of IRQs.
|
||||||
Drivers::Net::E1000::ProcessDeferredWork();
|
if (Drivers::Net::E1000::HasDeferredWork())
|
||||||
Drivers::Net::E1000E::ProcessDeferredWork();
|
Drivers::Net::E1000::ProcessDeferredWork();
|
||||||
|
if (Drivers::Net::E1000E::HasDeferredWork())
|
||||||
|
Drivers::Net::E1000E::ProcessDeferredWork();
|
||||||
|
|
||||||
// HDA completion IRQs only acknowledge/mask. Resampling and DMA-ring
|
// HDA completion IRQs only acknowledge/mask. Resampling and DMA-ring
|
||||||
// refill are far too expensive for hard interrupt context.
|
// refill are far too expensive for hard interrupt context.
|
||||||
if (cpu->cpuIndex == 0) {
|
if (cpu->cpuIndex == 0 && Drivers::Audio::IntelHda::HasDeferredWork()) {
|
||||||
Drivers::Audio::IntelHda::ProcessDeferredWork();
|
Drivers::Audio::IntelHda::ProcessDeferredWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -260,7 +285,10 @@ namespace Timekeeping {
|
|||||||
// seconds and used to stall kmain before the first process spawned.
|
// seconds and used to stall kmain before the first process spawned.
|
||||||
// Cheap no-op unless an adapter is waiting; self-claiming, and safe
|
// Cheap no-op unless an adapter is waiting; self-claiming, and safe
|
||||||
// to preempt (the scheduler saves/resumes the idle context).
|
// to preempt (the scheduler saves/resumes the idle context).
|
||||||
Drivers::USB::Bluetooth::ServiceDeferredInit();
|
bool bluetoothWork = usbWork ||
|
||||||
|
Drivers::USB::Bluetooth::HasDeferredWork();
|
||||||
|
if (bluetoothWork)
|
||||||
|
Drivers::USB::Bluetooth::ServiceDeferredInit();
|
||||||
|
|
||||||
// Service Bluetooth inbound traffic (the headset's SDP/AVRCP queries,
|
// Service Bluetooth inbound traffic (the headset's SDP/AVRCP queries,
|
||||||
// AVDTP commands, ACL flow-control credits) whenever a core idles.
|
// AVDTP commands, ACL flow-control credits) whenever a core idles.
|
||||||
@@ -269,13 +297,16 @@ namespace Timekeeping {
|
|||||||
// writes got silence (observed: Bose re-dialing SDP during playback,
|
// writes got silence (observed: Bose re-dialing SDP during playback,
|
||||||
// queries never answered). Self-serializing and a cheap no-op when
|
// queries never answered). Self-serializing and a cheap no-op when
|
||||||
// the adapter is down.
|
// the adapter is down.
|
||||||
Drivers::USB::Bluetooth::ServiceEvents();
|
if (bluetoothWork)
|
||||||
|
Drivers::USB::Bluetooth::ServiceEvents();
|
||||||
|
|
||||||
// Wi-Fi mirrors the Bluetooth split: the firmware load needs the
|
// Wi-Fi mirrors the Bluetooth split: the firmware load needs the
|
||||||
// ramdisk, and the RX/notification ring must be drained outside hard
|
// ramdisk, and the RX/notification ring must be drained outside hard
|
||||||
// interrupt context (the MSI handler only latches a flag).
|
// interrupt context (the MSI handler only latches a flag).
|
||||||
Drivers::Net::Wifi::ServiceDeferredInit();
|
if (Drivers::Net::Wifi::HasDeferredWork()) {
|
||||||
Drivers::Net::Wifi::ServiceEvents();
|
Drivers::Net::Wifi::ServiceDeferredInit();
|
||||||
|
Drivers::Net::Wifi::ServiceEvents();
|
||||||
|
}
|
||||||
|
|
||||||
// Thermal policy records transitions during BSP maintenance; print
|
// Thermal policy records transitions during BSP maintenance; print
|
||||||
// them from this explicitly non-interrupt idle path.
|
// them from this explicitly non-interrupt idle path.
|
||||||
|
|||||||
@@ -11,9 +11,14 @@ namespace Timekeeping {
|
|||||||
// Initialize the APIC timer: calibrate against PIT, start periodic interrupts
|
// Initialize the APIC timer: calibrate against PIT, start periodic interrupts
|
||||||
void ApicTimerInitialize();
|
void ApicTimerInitialize();
|
||||||
|
|
||||||
// Initialize the APIC timer on an AP (calibrate + start, no IRQ handler registration)
|
// Initialize the scheduler timer on an AP using the BSP calibration.
|
||||||
void ApicTimerInitializeAP();
|
void ApicTimerInitializeAP();
|
||||||
|
|
||||||
|
// Idle APs are woken for runnable work by the reschedule IPI, so their
|
||||||
|
// periodic scheduler timer can remain masked until a process is dispatched.
|
||||||
|
void ApicTimerEnterApIdle();
|
||||||
|
void ApicTimerLeaveApIdle();
|
||||||
|
|
||||||
// Reinitialize the APIC timer after S3 resume using the previously
|
// Reinitialize the APIC timer after S3 resume using the previously
|
||||||
// calibrated tick rate. Skips PIT calibration and IRQ registration
|
// calibrated tick rate. Skips PIT calibration and IRQ registration
|
||||||
// (both survive in RAM). Only reprograms the timer hardware registers.
|
// (both survive in RAM). Only reprograms the timer hardware registers.
|
||||||
|
|||||||
Reference in New Issue
Block a user