tdf#156881 Disable Metal with AMD Radeon Pro 5XXX GPUs on macOS Catalina

When running macOS Catalina on a 2019 MacBook Pro, unexpected drawing
artifacts are drawn so disable Metal for the AMD Radeon Pro GPUs listed
for that model in https://support.apple.com/kb/SP809.

Change-Id: Iffe44da1f07af2f3bbc367051b5ea4d522216eb7
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/157443
Tested-by: Jenkins
Reviewed-by: Patrick Luby <plubius@neooffice.org>
(cherry picked from commit fe3fa1699f12eb007d26bbb19f5ff80826bc384f)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/157460
diff --git a/vcl/Library_vcl.mk b/vcl/Library_vcl.mk
index a7c1fbc..af8e311 100644
--- a/vcl/Library_vcl.mk
+++ b/vcl/Library_vcl.mk
@@ -721,6 +721,7 @@ $(eval $(call gb_Library_add_objcxxobjects,vcl,\
$(eval $(call gb_Library_use_system_darwin_frameworks,vcl,\
    Cocoa \
    CoreFoundation \
    Metal \
))
endif

diff --git a/vcl/inc/quartz/cgutils.h b/vcl/inc/quartz/cgutils.h
index 6c49944..786b214 100644
--- a/vcl/inc/quartz/cgutils.h
+++ b/vcl/inc/quartz/cgutils.h
@@ -31,4 +31,8 @@ CGImageRef VCL_DLLPUBLIC CreateWithSalBitmapAndMask(const SalBitmap& rBitmap,
                                                    const SalBitmap& rMask, int nX, int nY,
                                                    int nWidth, int nHeight);

#ifdef MACOSX
bool VCL_DLLPUBLIC DefaultMTLDeviceIsSupported();
#endif

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/quartz/cgutils.mm b/vcl/quartz/cgutils.mm
index c28391c..6ae1576f 100644
--- a/vcl/quartz/cgutils.mm
+++ b/vcl/quartz/cgutils.mm
@@ -26,6 +26,12 @@
#include <ios/iosinst.hxx>
#endif

#ifdef MACOSX
#include <premac.h>
#include <Metal/Metal.h>
#include <postmac.h>
#endif

static void CFRTLFree(void* /*info*/, const void* data, size_t /*size*/)
{
    std::free( const_cast<void*>(data) );
@@ -74,4 +80,41 @@ CGImageRef CreateWithSalBitmapAndMask( const SalBitmap& rBitmap, const SalBitmap
    return xMaskedImage;
}

#ifdef MACOSX

bool DefaultMTLDeviceIsSupported()
{
    id<MTLDevice> pMetalDevice = MTLCreateSystemDefaultDevice();
    if (!pMetalDevice || !pMetalDevice.name)
    {
        SAL_WARN("vcl.skia", "MTLCreateSystemDefaultDevice() returned nil");
        return false;
    }

    SAL_WARN("vcl.skia", "Default MTLDevice is \"" << [pMetalDevice.name UTF8String] << "\"");

    bool bRet = true;

    // tdf#156881 Disable Metal with AMD Radeon Pro 5XXX GPUs on macOS Catalina
    // When running macOS Catalina on a 2019 MacBook Pro, unexpected drawing
    // artifacts are drawn so disable Metal for the AMD Radeon Pro GPUs listed
    // for that model in https://support.apple.com/kb/SP809.
    if (@available(macOS 11, *))
    {
        // No known problems with macOS Big Sur or later
    }
    else
    {
       static NSString* pAMDRadeonPro5300Prefix = @"AMD Radeon Pro 5300M";
       static NSString* pAMDRadeonPro5500Prefix = @"AMD Radeon Pro 5500M";
       if ([pMetalDevice.name hasPrefix:pAMDRadeonPro5300Prefix] || [pMetalDevice.name hasPrefix:pAMDRadeonPro5500Prefix])
           bRet = false;
    }

    [pMetalDevice release];
    return bRet;
}

#endif

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/skia/SkiaHelper.cxx b/vcl/skia/SkiaHelper.cxx
index 9875b78..4a83aa1 100644
--- a/vcl/skia/SkiaHelper.cxx
+++ b/vcl/skia/SkiaHelper.cxx
@@ -54,6 +54,12 @@ bool isAlphaMaskBlendingEnabled() { return false; }
#include <tools/sk_app/MetalWindowContext.h>
#include <fstream>

#ifdef SK_METAL
#ifdef MACOSX
#include <quartz/cgutils.h>
#endif
#endif

namespace SkiaHelper
{
static OUString getCacheFolder()
@@ -245,10 +251,19 @@ static void checkDeviceDenylisted(bool blockDisable = false)
            }
            if (grDirectContext) // Metal was initialized properly
            {
                // Try to assume Metal always works, given that Mac doesn't have such as wide range of HW vendors as PC.
                // If there turns out to be problems, handle it similarly to Vulkan.
                SAL_INFO("vcl.skia", "Using Skia Metal mode");
                writeSkiaMetalInfo();
#ifdef MACOSX
                if (!blockDisable && !DefaultMTLDeviceIsSupported())
                {
                    SAL_INFO("vcl.skia", "Metal default device not supported");
                    disableRenderMethod(RenderMetal);
                    useRaster = true;
                }
                else
#endif
                {
                    SAL_INFO("vcl.skia", "Using Skia Metal mode");
                    writeSkiaMetalInfo();
                }
            }
            else
            {