fix(nvdrv): prevent infinite loop and improve error handling when SMMU address space is exhausted

Fix crash in PinHandle when SMMU allocation fails and unmap queue is empty.
Previously, the code would log an error and continue looping indefinitely,
causing log spam and eventual crash.

- Free multiple handles from unmap queue (up to 100) before giving up
- Add maximum attempt counter to prevent infinite loops
- Return 0 gracefully when no more handles can be freed
- Add error handling in callers (Remap, MapBufferEx, MapBuffer) to check for
  PinHandle failure and return NvResult::InsufficientMemory instead of using
  invalid addresses

Signed-off-by: Zephyron <zephyron@citron-emu.org>

src/core/hle/service/nvdrv/core/nvmap.cpp

@@ -210,17 +210,32 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, bool low_area_pin) {
             handle_description->in_heap = true;
         } else {
             size_t aligned_up = Common::AlignUp(map_size, BIG_PAGE_SIZE);
+            constexpr size_t MAX_FREE_ATTEMPTS = 100; // Prevent infinite loop
+            size_t free_attempts = 0;
             while ((address = smmu.Allocate(aligned_up)) == 0) {
                 // Free handles until the allocation succeeds
                 std::scoped_lock queueLock(unmap_queue_lock);
-                if (auto freeHandleDesc{unmap_queue.front()}) {
-                    // Handles in the unmap queue are guaranteed not to be pinned so don't bother
-                    // checking if they are before unmapping
-                    std::scoped_lock freeLock(freeHandleDesc->mutex);
-                    if (freeHandleDesc->d_address)
-                        UnmapHandle(*freeHandleDesc);
-                } else {
-                    LOG_CRITICAL(Service_NVDRV, "Ran out of SMMU address space!");
+                bool freed_any = false;
+                // Try to free multiple handles from the queue
+                while (!unmap_queue.empty() && free_attempts < MAX_FREE_ATTEMPTS) {
+                    if (auto freeHandleDesc{unmap_queue.front()}) {
+                        // Handles in the unmap queue are guaranteed not to be pinned so don't bother
+                        // checking if they are before unmapping
+                        std::scoped_lock freeLock(freeHandleDesc->mutex);
+                        if (freeHandleDesc->d_address) {
+                            UnmapHandle(*freeHandleDesc);
+                            freed_any = true;
+                        }
+                        // Remove from queue even if d_address was 0 (already unmapped)
+                        unmap_queue.pop_front();
+                    } else {
+                        unmap_queue.pop_front();
+                    }
+                    free_attempts++;
+                }
+
+                if (!freed_any || unmap_queue.empty()) {
+                    LOG_CRITICAL(Service_NVDRV, "Ran out of SMMU address space! No more handles to free.");
                     // Break out of the loop to prevent infinite spinning when no handles can be freed
                     return 0;
                 }

src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp

@@ -1,5 +1,6 @@
 // SPDX-FileCopyrightText: 2021 yuzu Emulator Project
 // SPDX-FileCopyrightText: 2021 Skyline Team and Contributors
+// SPDX-FileCopyrightText: 2025 citron Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include <cstring>
@@ -298,6 +299,10 @@ NvResult nvhost_as_gpu::Remap(std::span<IoctlRemapEntry> entries) {
             }
 
             DAddr base = nvmap.PinHandle(entry.handle, false);
+            if (!base) {
+                LOG_ERROR(Service_NVDRV, "Failed to pin handle {}: SMMU address space exhausted", entry.handle);
+                return NvResult::InsufficientMemory;
+            }
             DAddr device_address{static_cast<DAddr>(
                 base + (static_cast<u64>(entry.handle_offset_big_pages) << vm.big_page_size_bits))};
 
@@ -353,8 +358,12 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
         return NvResult::BadValue;
     }
 
-    DAddr device_address{
-        static_cast<DAddr>(nvmap.PinHandle(params.handle, false) + params.buffer_offset)};
+    DAddr base = nvmap.PinHandle(params.handle, false);
+    if (!base) {
+        LOG_ERROR(Service_NVDRV, "Failed to pin handle {}: SMMU address space exhausted", params.handle);
+        return NvResult::InsufficientMemory;
+    }
+    DAddr device_address{static_cast<DAddr>(base + params.buffer_offset)};
     u64 size{params.mapping_size ? params.mapping_size : handle->orig_size};
 
     bool big_page{[&]() {

src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp

@@ -1,4 +1,5 @@
 // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
+// SPDX-FileCopyrightText: Copyright 2025 citron Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include <algorithm>
@@ -141,6 +142,10 @@ NvResult nvhost_nvdec_common::MapBuffer(IoctlMapBuffer& params, std::span<MapBuf
     const size_t num_entries = std::min(params.num_entries, static_cast<u32>(entries.size()));
     for (size_t i = 0; i < num_entries; i++) {
         DAddr pin_address = nvmap.PinHandle(entries[i].map_handle, true);
+        if (!pin_address) {
+            LOG_ERROR(Service_NVDRV, "Failed to pin handle {}: SMMU address space exhausted", entries[i].map_handle);
+            return NvResult::InsufficientMemory;
+        }
         entries[i].map_address = static_cast<u32>(pin_address);
     }