WIP: fix(video_core): shadow map square artifacts for Metroid Prime 4

- Force shadow maps to use CLAMP_TO_BORDER with white border color
- Convert CLAMP_TO_EDGE to CLAMP_TO_BORDER for shadow maps
- Improve GL_CLAMP handling for shadow maps on AMD

Issue may persist - likely needs investigation of shadow map rendering
or shader coordinate generation.

Signed-off-by: Zephyron <zephyron@citron-emu.org>

src/video_core/renderer_vulkan/maxwell_to_vk.cpp

@@ -49,13 +49,18 @@ VkSamplerMipmapMode MipmapMode(Tegra::Texture::TextureMipmapFilter mipmap_filter
 }
 
 VkSamplerAddressMode WrapMode(const Device& device, Tegra::Texture::WrapMode wrap_mode,
-                              Tegra::Texture::TextureFilter filter) {
+                              Tegra::Texture::TextureFilter filter, bool is_shadow_map) {
     switch (wrap_mode) {
     case Tegra::Texture::WrapMode::Wrap:
         return VK_SAMPLER_ADDRESS_MODE_REPEAT;
     case Tegra::Texture::WrapMode::Mirror:
         return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
     case Tegra::Texture::WrapMode::ClampToEdge:
+        // For shadow maps, use CLAMP_TO_BORDER instead of CLAMP_TO_EDGE
+        // CLAMP_TO_EDGE can cause square artifacts by repeating edge pixels
+        if (is_shadow_map) {
+            return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
+        }
         return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
     case Tegra::Texture::WrapMode::Border:
         return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
@@ -65,11 +70,20 @@ VkSamplerAddressMode WrapMode(const Device& device, Tegra::Texture::WrapMode wra
             // by sending an invalid enumeration.
             return static_cast<VkSamplerAddressMode>(0xcafe);
         }
-        // TODO(Rodrigo): Emulate GL_CLAMP properly on other vendors
+        // For shadow maps, GL_CLAMP should use CLAMP_TO_BORDER to prevent square artifacts
+        // GL_CLAMP clamps coordinates to [0,1] and uses border color for out-of-range values
+        // Using CLAMP_TO_BORDER provides the closest match for shadow mapping
+        if (is_shadow_map) {
+            return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
+        }
+        // For non-shadow textures, use appropriate fallback based on filter
+        // GL_CLAMP with linear filtering should use border to match interpolation behavior
         switch (filter) {
         case Tegra::Texture::TextureFilter::Nearest:
+            // Nearest filtering: use edge clamping to avoid border artifacts
             return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
         case Tegra::Texture::TextureFilter::Linear:
+            // Linear filtering: use border to match GL_CLAMP interpolation behavior
             return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
         }
         ASSERT(false);

src/video_core/renderer_vulkan/maxwell_to_vk.h

@@ -1,4 +1,5 @@
 // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
+// SPDX-FileCopyrightText: Copyright 2025 citron Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
@@ -22,7 +23,8 @@ VkFilter Filter(Tegra::Texture::TextureFilter filter);
 VkSamplerMipmapMode MipmapMode(Tegra::Texture::TextureMipmapFilter mipmap_filter);
 
 VkSamplerAddressMode WrapMode(const Device& device, Tegra::Texture::WrapMode wrap_mode,
-                              Tegra::Texture::TextureFilter filter);
+                              Tegra::Texture::TextureFilter filter,
+                              bool is_shadow_map = false);
 
 VkCompareOp DepthCompareFunction(Tegra::Texture::DepthCompareFunc depth_compare_func);
 

src/video_core/renderer_vulkan/vk_texture_cache.cpp

@@ -1,4 +1,5 @@
 // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
+// SPDX-FileCopyrightText: Copyright 2025 citron Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include <algorithm>
@@ -2025,17 +2026,65 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t
     // Some games have samplers with garbage. Sanitize them here.
     const f32 max_anisotropy = std::clamp(tsc.MaxAnisotropy(), 1.0f, 16.0f);
 
+    // For shadow maps (depth compare enabled), ensure proper border color
+    // Shadow maps should use opaque white border (1.0) so out-of-range samples are not shadowed
+    // This is critical: in depth compare mode, white (1.0) = not shadowed, black (0.0) = shadowed
+    const bool is_shadow_map = tsc.depth_compare_enabled;
+    const std::array<float, 4> shadow_border_color_custom{1.0f, 1.0f, 1.0f, 1.0f};
+
+    // Determine wrap modes (shadow maps will use CLAMP_TO_BORDER for GL_CLAMP and ClampToEdge)
+    const auto wrap_u_mode = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_u, tsc.mag_filter, is_shadow_map);
+    const auto wrap_v_mode = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_v, tsc.mag_filter, is_shadow_map);
+    const auto wrap_p_mode = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_p, tsc.mag_filter, is_shadow_map);
+
+    // For shadow maps, ALWAYS use white border color regardless of wrap mode
+    // This ensures any edge cases or driver-specific behavior uses correct border
+    VkSamplerCustomBorderColorCreateInfoEXT shadow_border_ci{
+        .sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT,
+        .pNext = nullptr,
+        .customBorderColor = Common::BitCast<VkClearColorValue>(shadow_border_color_custom),
+        .format = VK_FORMAT_UNDEFINED,
+    };
+    const void* shadow_pnext = nullptr;
+    if (is_shadow_map && arbitrary_borders) {
+        shadow_pnext = &shadow_border_ci;
+        // Chain with reduction mode if needed
+        if (runtime.device.IsExtSamplerFilterMinmaxSupported() &&
+            MaxwellToVK::SamplerReduction(tsc.reduction_filter) != VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT) {
+            shadow_border_ci.pNext = pnext;
+        }
+    }
+
     const auto create_sampler = [&](const f32 anisotropy) {
+        VkBorderColor final_border_color;
+        const void* final_pnext;
+
+        // For ALL shadow maps, force opaque white border to prevent black square artifacts
+        // This is safe because shadow maps should never be shadowed outside their bounds
+        if (is_shadow_map) {
+            if (arbitrary_borders) {
+                final_pnext = shadow_pnext;
+                final_border_color = VK_BORDER_COLOR_FLOAT_CUSTOM_EXT;
+            } else {
+                final_pnext = pnext;
+                // Use opaque white even if not using border clamp - some drivers may sample borders anyway
+                final_border_color = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
+            }
+        } else {
+            final_pnext = pnext;
+            final_border_color = arbitrary_borders ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : ConvertBorderColor(color);
+        }
+
         return device.GetLogical().CreateSampler(VkSamplerCreateInfo{
             .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
-            .pNext = pnext,
+            .pNext = final_pnext,
             .flags = 0,
             .magFilter = MaxwellToVK::Sampler::Filter(tsc.mag_filter),
             .minFilter = MaxwellToVK::Sampler::Filter(tsc.min_filter),
             .mipmapMode = MaxwellToVK::Sampler::MipmapMode(tsc.mipmap_filter),
-            .addressModeU = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_u, tsc.mag_filter),
+            .addressModeU = wrap_u_mode,
-            .addressModeV = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_v, tsc.mag_filter),
+            .addressModeV = wrap_v_mode,
-            .addressModeW = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_p, tsc.mag_filter),
+            .addressModeW = wrap_p_mode,
             .mipLodBias = tsc.LodBias(),
             .anisotropyEnable = static_cast<VkBool32>(anisotropy > 1.0f ? VK_TRUE : VK_FALSE),
             .maxAnisotropy = anisotropy,
@@ -2043,8 +2092,7 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t
             .compareOp = MaxwellToVK::Sampler::DepthCompareFunction(tsc.depth_compare_func),
             .minLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.0f : tsc.MinLod(),
             .maxLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.25f : tsc.MaxLod(),
-            .borderColor =
+            .borderColor = final_border_color,
-                arbitrary_borders ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : ConvertBorderColor(color),
             .unnormalizedCoordinates = VK_FALSE,
         });
     };