LibGfx: Add debug macro to enable Vulkan validation layers

Vulkan Validation Layers provides diagnostic feedback about Vulkan API
usage while live testing Ladybird. It is possible to enable this
diagnostic output without changing the code (using the Vulkan SDK),
but having it built into the code makes it very easy to enable whenever
required.

Co-authored-by: Rocco Corsi <5201151+rcorsi@users.noreply.github.com>

AK/Debug.h.in

@@ -298,6 +298,10 @@
 #    cmakedefine01 VPX_DEBUG
 #endif
 
+#ifndef VULKAN_VALIDATION_LAYERS_DEBUG
+#    cmakedefine01 VULKAN_VALIDATION_LAYERS_DEBUG
+#endif
+
 #ifndef WASI_DEBUG
 #    cmakedefine01 WASI_DEBUG
 #endif

Documentation/AdvancedBuildInstructions.md

@@ -187,3 +187,21 @@ After you’ve finished debugging your code changes with that build, you can rev
 
 That will restore your git environment to the state it was in before you patched the build file.
 
+## Debugging with Vulkan Validation Layers
+
+To turn on Vulkan validation layers for the release build, for example, use:
+
+```
+cmake -B Build/release -DVULKAN_VALIDATION_LAYERS_DEBUG=ON
+```
+
+and then build normally.
+
+When running Ladybird the message `Vulkan validation layers: active` will be visible during the creation of the VulkanContext if the Vulkan validation layers are properly setup.
+
+If instead `Vulkan validation layers: not available` appears then the system is most likely missing the validation layers. On Ubuntu adding the package `vulkan-validationlayers` will add them in, other distros may also have a package but the name could be different (for example, on Arch, Fedora and NixOS the name is `vulkan-validation-layers`.
+
+Another way to get the validation layers is to use the Vulkan SDK, for more info see:
+https://github.com/KhronosGroup/Vulkan-ValidationLayers
+
+If absolutely no output related to the Vulkan validation layers is received, then the initial `cmake` command in this section was not performed or the VulkanContext was not created successfully or was not created at all.

Libraries/LibGfx/VulkanContext.cpp

@@ -4,12 +4,18 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <AK/Debug.h>
 #include <AK/Format.h>
 #include <AK/Vector.h>
 #include <LibGfx/VulkanContext.h>
 
 namespace Gfx {
 
+#if VULKAN_VALIDATION_LAYERS_DEBUG
+static void setup_vulkan_validation_layers_callback(VkInstanceCreateInfo& create_info, VkDebugUtilsMessengerCreateInfoEXT& debug_messenger_create_info);
+static void enable_vulkan_validation_layers_callback(VkInstance const& instance, VkDebugUtilsMessengerCreateInfoEXT const& debug_messenger_create_info);
+#endif
+
 static ErrorOr<VkInstance> create_instance(uint32_t api_version)
 {
     VkInstance instance;
@@ -26,12 +32,21 @@ static ErrorOr<VkInstance> create_instance(uint32_t api_version)
     create_info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
     create_info.pApplicationInfo = &app_info;
 
+#if VULKAN_VALIDATION_LAYERS_DEBUG
+    VkDebugUtilsMessengerCreateInfoEXT debug_messenger_create_info {};
+    setup_vulkan_validation_layers_callback(create_info, debug_messenger_create_info);
+#endif
+
     auto result = vkCreateInstance(&create_info, nullptr, &instance);
     if (result != VK_SUCCESS) {
         dbgln("vkCreateInstance returned {}", to_underlying(result));
         return Error::from_string_literal("Application instance creation failed");
     }
 
+#if VULKAN_VALIDATION_LAYERS_DEBUG
+    enable_vulkan_validation_layers_callback(instance, debug_messenger_create_info);
+#endif
+
     return instance;
 }
 
@@ -421,4 +436,64 @@ ErrorOr<NonnullRefPtr<VulkanImage>> create_shared_vulkan_image(VulkanContext con
 }
 #endif
 
+#if VULKAN_VALIDATION_LAYERS_DEBUG
+static bool check_layer_support(StringView layer_name)
+{
+    uint32_t layer_count = 0;
+    vkEnumerateInstanceLayerProperties(&layer_count, nullptr);
+
+    Vector<VkLayerProperties> layers;
+    layers.resize(layer_count);
+    vkEnumerateInstanceLayerProperties(&layer_count, layers.data());
+
+    return layers.contains([&layer_name](auto const& layer_properties) {
+        return layer_properties.layerName == layer_name;
+    });
+}
+
+static constexpr Array<char const*, 1> vvl_layers = { "VK_LAYER_KHRONOS_validation" };
+static constexpr Array<char const*, 1> vvl_extensions = { VK_EXT_DEBUG_UTILS_EXTENSION_NAME };
+
+static void setup_vulkan_validation_layers_callback(VkInstanceCreateInfo& create_info, VkDebugUtilsMessengerCreateInfoEXT& debug_messenger_create_info)
+{
+    if (!check_layer_support("VK_LAYER_KHRONOS_validation"sv))
+        return;
+
+    debug_messenger_create_info.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT;
+    debug_messenger_create_info.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT;
+    debug_messenger_create_info.messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT;
+    debug_messenger_create_info.pfnUserCallback = [](VkDebugUtilsMessageSeverityFlagBitsEXT,
+                                                      VkDebugUtilsMessageTypeFlagsEXT,
+                                                      VkDebugUtilsMessengerCallbackDataEXT const* pCallbackData,
+                                                      void*) {
+        dbgln("Vulkan validation layers: {}", pCallbackData->pMessage);
+        dump_backtrace(3);
+        return VK_FALSE;
+    };
+
+    create_info.enabledLayerCount = vvl_layers.size();
+    create_info.ppEnabledLayerNames = vvl_layers.data();
+    create_info.enabledExtensionCount = vvl_extensions.size();
+    create_info.ppEnabledExtensionNames = vvl_extensions.data();
+    create_info.pNext = &debug_messenger_create_info;
+}
+
+static void enable_vulkan_validation_layers_callback(VkInstance const& instance, VkDebugUtilsMessengerCreateInfoEXT const& debug_messenger_create_info)
+{
+    // Vulkan validation layers not available if struct not setup properly
+    if (debug_messenger_create_info.sType != VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT) {
+        dbgln("Vulkan validation layers: not available");
+        return;
+    }
+
+    auto pfn_create_debug_messenger = reinterpret_cast<PFN_vkCreateDebugUtilsMessengerEXT>(vkGetInstanceProcAddr(instance, "vkCreateDebugUtilsMessengerEXT"));
+    VkDebugUtilsMessengerEXT debug_messenger { VK_NULL_HANDLE };
+    auto result = pfn_create_debug_messenger(instance, &debug_messenger_create_info, nullptr, &debug_messenger);
+    if (result != VK_SUCCESS)
+        dbgln("vkCreateDebugUtilsMessengerEXT returned {}", to_underlying(result));
+    else
+        dbgln("Vulkan validation layers: active");
+}
+#endif
+
 }

Meta/CMake/all_the_debug_macros.cmake

@@ -70,6 +70,7 @@ set(URL_PARSER_DEBUG ON)
 set(URL_PATTERN_DEBUG ON)
 set(UTF8_DEBUG ON)
 set(VPX_DEBUG ON)
+set(VULKAN_VALIDATION_LAYERS_DEBUG ON)
 set(WASI_DEBUG ON)
 set(WASI_FINE_GRAINED_DEBUG ON)
 set(WASM_BINPARSER_DEBUG ON)
@@ -93,3 +94,8 @@ set(XML_PARSER_DEBUG ON)
 # set(gn_include_dirs_DEBUG ON)
 # set(gn_ldflags_DEBUG ON)
 # set(gn_lib_dirs_DEBUG ON)
+# False positive: Vulkan Validation Layers enums and defines
+# set(VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT ON)
+# set(VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT ON)
+# set(VK_EXT_DEBUG_UTILS_EXTENSION_NAME ON)
+# set(VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT ON)