tdf#156881 Disable Metal with AMD Radeon Pro 5XXX GPUs on macOS Catalina

When running macOS Catalina on a 2019 MacBook Pro, unexpected drawing
artifacts are drawn so disable Metal for the AMD Radeon Pro GPUs listed
for that model in https://support.apple.com/kb/SP809.

Change-Id: Iffe44da1f07af2f3bbc367051b5ea4d522216eb7
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/157443
Tested-by: Jenkins
Reviewed-by: Patrick Luby <plubius@neooffice.org>
diff --git a/vcl/Library_vcl.mk b/vcl/Library_vcl.mk
index 6eb5043..d6a3707 100644
--- a/vcl/Library_vcl.mk
+++ b/vcl/Library_vcl.mk
@@ -714,6 +714,7 @@ $(eval $(call gb_Library_add_objcxxobjects,vcl,\
$(eval $(call gb_Library_use_system_darwin_frameworks,vcl,\
    Cocoa \
    CoreFoundation \
    Metal \
))
endif

diff --git a/vcl/inc/quartz/cgutils.h b/vcl/inc/quartz/cgutils.h
index 6c49944..786b214 100644
--- a/vcl/inc/quartz/cgutils.h
+++ b/vcl/inc/quartz/cgutils.h
@@ -31,4 +31,8 @@ CGImageRef VCL_DLLPUBLIC CreateWithSalBitmapAndMask(const SalBitmap& rBitmap,
                                                    const SalBitmap& rMask, int nX, int nY,
                                                    int nWidth, int nHeight);

#ifdef MACOSX
bool VCL_DLLPUBLIC DefaultMTLDeviceIsSupported();
#endif

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/quartz/cgutils.mm b/vcl/quartz/cgutils.mm
index c6a490d..0d611be 100644
--- a/vcl/quartz/cgutils.mm
+++ b/vcl/quartz/cgutils.mm
@@ -26,6 +26,12 @@
#include <ios/iosinst.hxx>
#endif

#ifdef MACOSX
#include <premac.h>
#include <Metal/Metal.h>
#include <postmac.h>
#endif

static void CFRTLFree(void* /*info*/, const void* data, size_t /*size*/)
{
    std::free( const_cast<void*>(data) );
@@ -85,4 +91,41 @@ CGImageRef CreateWithSalBitmapAndMask( const SalBitmap& rBitmap, const SalBitmap
    return xMaskedImage;
}

#ifdef MACOSX

bool DefaultMTLDeviceIsSupported()
{
    id<MTLDevice> pMetalDevice = MTLCreateSystemDefaultDevice();
    if (!pMetalDevice || !pMetalDevice.name)
    {
        SAL_WARN("vcl.skia", "MTLCreateSystemDefaultDevice() returned nil");
        return false;
    }

    SAL_WARN("vcl.skia", "Default MTLDevice is \"" << [pMetalDevice.name UTF8String] << "\"");

    bool bRet = true;

    // tdf#156881 Disable Metal with AMD Radeon Pro 5XXX GPUs on macOS Catalina
    // When running macOS Catalina on a 2019 MacBook Pro, unexpected drawing
    // artifacts are drawn so disable Metal for the AMD Radeon Pro GPUs listed
    // for that model in https://support.apple.com/kb/SP809.
    if (@available(macOS 11, *))
    {
        // No known problems with macOS Big Sur or later
    }
    else
    {
       static NSString* pAMDRadeonPro5300Prefix = @"AMD Radeon Pro 5300M";
       static NSString* pAMDRadeonPro5500Prefix = @"AMD Radeon Pro 5500M";
       if ([pMetalDevice.name hasPrefix:pAMDRadeonPro5300Prefix] || [pMetalDevice.name hasPrefix:pAMDRadeonPro5500Prefix])
           bRet = false;
    }

    [pMetalDevice release];
    return bRet;
}

#endif

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/vcl/skia/SkiaHelper.cxx b/vcl/skia/SkiaHelper.cxx
index 41ce551..712ba39 100644
--- a/vcl/skia/SkiaHelper.cxx
+++ b/vcl/skia/SkiaHelper.cxx
@@ -85,6 +85,12 @@ bool isAlphaMaskBlendingEnabled() { return false; }

#include <fstream>

#ifdef SK_METAL
#ifdef MACOSX
#include <quartz/cgutils.h>
#endif
#endif

namespace SkiaHelper
{
static OUString getCacheFolder()
@@ -276,10 +282,19 @@ static void checkDeviceDenylisted(bool blockDisable = false)
            }
            if (grDirectContext) // Metal was initialized properly
            {
                // Try to assume Metal always works, given that Mac doesn't have such as wide range of HW vendors as PC.
                // If there turns out to be problems, handle it similarly to Vulkan.
                SAL_INFO("vcl.skia", "Using Skia Metal mode");
                writeSkiaMetalInfo();
#ifdef MACOSX
                if (!blockDisable && !DefaultMTLDeviceIsSupported())
                {
                    SAL_INFO("vcl.skia", "Metal default device not supported");
                    disableRenderMethod(RenderMetal);
                    useRaster = true;
                }
                else
#endif
                {
                    SAL_INFO("vcl.skia", "Using Skia Metal mode");
                    writeSkiaMetalInfo();
                }
            }
            else
            {