make CalcMaskShift a verifiable operation
and check it on untrusted data
Change-Id: I7c97a27d70f91b9686adf9dcb8b68c5aa25c2b4c
Reviewed-on: https://gerrit.libreoffice.org/18637
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>
diff --git a/include/vcl/salbtype.hxx b/include/vcl/salbtype.hxx
index eb3b6d3..82ae036 100644
--- a/include/vcl/salbtype.hxx
+++ b/include/vcl/salbtype.hxx
@@ -187,29 +187,29 @@ struct VCL_DLLPUBLIC ColorMaskElement
int mnShift;
int mnOrShift;
sal_uInt8 mnOr;
ColorMaskElement(sal_uInt32 nMask)
explicit ColorMaskElement(sal_uInt32 nMask = 0)
: mnMask(nMask)
, mnShift(0)
, mnOrShift(0)
, mnOr(0)
{
}
static bool CalcMaskShift(ColorMaskElement &rElem)
bool CalcMaskShift()
{
if (rElem.mnMask == 0)
if (mnMask == 0)
return true;
// from which bit starts the mask?
int nShift = 31;
while( nShift >= 0 && !( rElem.mnMask & ( 1 << nShift ) ) )
while( nShift >= 0 && !( mnMask & ( 1 << nShift ) ) )
--nShift;
rElem.mnShift = nShift - 7;
mnShift = nShift - 7;
int nLen = 0;
// XXX determine number of bits set => walk right until null
while( nShift >= 0 && ( rElem.mnMask & ( 1 << nShift ) ) )
while( nShift >= 0 && ( mnMask & ( 1 << nShift ) ) )
{
nShift--;
nLen++;
@@ -218,8 +218,8 @@ struct VCL_DLLPUBLIC ColorMaskElement
if (nLen > 8) // mask length must be 8 bits or less
return false;
rElem.mnOrShift = 8 - nLen;
rElem.mnOr = static_cast<sal_uInt8>( ( 0xFF >> nLen ) << rElem.mnOrShift );
mnOrShift = 8 - nLen;
mnOr = static_cast<sal_uInt8>( ( 0xFF >> nLen ) << mnOrShift );
return true;
}
@@ -235,11 +235,18 @@ class VCL_DLLPUBLIC ColorMask
public:
inline ColorMask( sal_uInt32 nRedMask = 0,
sal_uInt32 nGreenMask = 0,
sal_uInt32 nBlueMask = 0,
sal_uInt32 nAlphaChannel = 0 );
inline ~ColorMask() {}
ColorMask(const ColorMaskElement& rRedMask = ColorMaskElement(),
const ColorMaskElement& rGreenMask = ColorMaskElement(),
const ColorMaskElement& rBlueMask = ColorMaskElement(),
sal_uInt32 nAlphaChannel = 0)
: maR(rRedMask)
, maG(rGreenMask)
, maB(rBlueMask)
, mnAlphaChannel(nAlphaChannel)
{
}
~ColorMask() {}
inline sal_uInt32 GetRedMask() const;
inline sal_uInt32 GetGreenMask() const;
@@ -599,20 +606,6 @@ inline sal_uInt16 BitmapPalette::GetBestIndex( const BitmapColor& rCol ) const
return nRetIndex;
}
inline ColorMask::ColorMask( sal_uInt32 nRedMask,
sal_uInt32 nGreenMask,
sal_uInt32 nBlueMask,
sal_uInt32 nAlphaChannel )
: maR(nRedMask)
, maG(nGreenMask)
, maB(nBlueMask)
, mnAlphaChannel(nAlphaChannel)
{
ColorMaskElement::CalcMaskShift(maR);
ColorMaskElement::CalcMaskShift(maG);
ColorMaskElement::CalcMaskShift(maB);
}
inline sal_uInt32 ColorMask::GetRedMask() const
{
return maR.mnMask;
diff --git a/vcl/headless/svpbmp.cxx b/vcl/headless/svpbmp.cxx
index 3231043..30f6a56 100644
--- a/vcl/headless/svpbmp.cxx
+++ b/vcl/headless/svpbmp.cxx
@@ -172,65 +172,130 @@ BitmapBuffer* SvpSalBitmap::AcquireBuffer( BitmapAccessMode )
pBuf->mnFormat = BMP_FORMAT_8BIT_PAL;
break;
case Format::SixteenBitLsbTcMask:
{
nBitCount = 16;
pBuf->mnFormat = BMP_FORMAT_16BIT_TC_LSB_MASK;
pBuf->maColorMask = ColorMask( 0xf800, 0x07e0, 0x001f );
ColorMaskElement aRedMask(0xf800);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x07e0);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x001f);
aBlueMask.CalcMaskShift();
pBuf->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
case Format::SixteenBitMsbTcMask:
{
nBitCount = 16;
pBuf->mnFormat = BMP_FORMAT_16BIT_TC_MSB_MASK;
pBuf->maColorMask = ColorMask( 0xf800, 0x07e0, 0x001f );
ColorMaskElement aRedMask(0xf800);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x07e0);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x001f);
aBlueMask.CalcMaskShift();
pBuf->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
case Format::TwentyFourBitTcMask:
nBitCount = 24;
pBuf->mnFormat = BMP_FORMAT_24BIT_TC_BGR;
break;
case Format::ThirtyTwoBitTcMaskBGRX:
{
nBitCount = 32;
pBuf->mnFormat = BMP_FORMAT_32BIT_TC_MASK;
#ifdef OSL_BIGENDIAN
pBuf->maColorMask = ColorMask( 0x0000ff00, 0x00ff0000, 0xff000000 );
ColorMaskElement aRedMask(0x0000ff00);
ColorMaskElement aGreenMask(0x00ff0000);
ColorMaskElement aBlueMask(0xff000000);
#else
pBuf->maColorMask = ColorMask( 0x00ff0000, 0x0000ff00, 0x000000ff );
ColorMaskElement aRedMask(0x00ff0000);
ColorMaskElement aGreenMask(0x0000ff00);
ColorMaskElement aBlueMask(0x000000ff);
#endif
aBlueMask.CalcMaskShift();
aRedMask.CalcMaskShift();
aGreenMask.CalcMaskShift();
pBuf->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
case Format::ThirtyTwoBitTcMaskBGRA:
{
nBitCount = 32;
pBuf->mnFormat = BMP_FORMAT_32BIT_TC_MASK;
#ifdef OSL_BIGENDIAN
pBuf->maColorMask = ColorMask( 0x0000ff00, 0x00ff0000, 0xff000000, 0x000000ff );
ColorMaskElement aRedMask(0x0000ff00);
ColorMaskElement aGreenMask(0x00ff0000);
ColorMaskElement aBlueMask(0xff000000);
#else
pBuf->maColorMask = ColorMask( 0x00ff0000, 0x0000ff00, 0x000000ff, 0xff000000 );
ColorMaskElement aRedMask(0x00ff0000);
ColorMaskElement aGreenMask(0x0000ff00);
ColorMaskElement aBlueMask(0x000000ff);
#endif
aBlueMask.CalcMaskShift();
aRedMask.CalcMaskShift();
aGreenMask.CalcMaskShift();
pBuf->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask, 0xff000000);
break;
}
case Format::ThirtyTwoBitTcMaskARGB:
{
nBitCount = 32;
pBuf->mnFormat = BMP_FORMAT_32BIT_TC_MASK;
#ifdef OSL_BIGENDIAN
pBuf->maColorMask = ColorMask( 0x00ff0000, 0x0000ff00, 0x000000ff, 0xff000000 );
ColorMaskElement aRedMask(0x00ff0000);
ColorMaskElement aGreenMask(0x0000ff00);
ColorMaskElement aBlueMask(0x000000ff);
#else
pBuf->maColorMask = ColorMask( 0x0000ff00, 0x00ff0000, 0xff000000, 0x000000ff );
ColorMaskElement aRedMask(0x0000ff00);
ColorMaskElement aGreenMask(0x00ff0000);
ColorMaskElement aBlueMask(0xff000000);
#endif
aBlueMask.CalcMaskShift();
aRedMask.CalcMaskShift();
aGreenMask.CalcMaskShift();
pBuf->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask, 0xff000000 );
break;
}
case Format::ThirtyTwoBitTcMaskABGR:
{
nBitCount = 32;
pBuf->mnFormat = BMP_FORMAT_32BIT_TC_MASK;
#ifdef OSL_BIGENDIAN
pBuf->maColorMask = ColorMask( 0x000000ff, 0x0000ff00, 0x00ff0000, 0xff000000 );
ColorMaskElement aRedMask(0x000000ff);
ColorMaskElement aGreenMask(0x0000ff00);
ColorMaskElement aBlueMask(0x00ff0000);
#else
pBuf->maColorMask = ColorMask( 0xff000000, 0x00ff0000, 0x0000ff00, 0x000000ff );
ColorMaskElement aRedMask(0xff000000);
ColorMaskElement aGreenMask(0x00ff0000);
ColorMaskElement aBlueMask(0x0000ff00);
#endif
aBlueMask.CalcMaskShift();
aRedMask.CalcMaskShift();
aGreenMask.CalcMaskShift();
pBuf->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask, 0xff000000);
break;
}
case Format::ThirtyTwoBitTcMaskRGBA:
{
nBitCount = 32;
pBuf->mnFormat = BMP_FORMAT_32BIT_TC_MASK;
#ifdef OSL_BIGENDIAN
pBuf->maColorMask = ColorMask( 0xff000000, 0x00ff0000, 0x0000ff00, 0x000000ff );
ColorMaskElement aRedMask(0xff000000);
ColorMaskElement aGreenMask(0x00ff0000);
ColorMaskElement aBlueMask(0x0000ff00);
#else
pBuf->maColorMask = ColorMask( 0x000000ff, 0x0000ff00, 0x00ff0000, 0xff000000 );
ColorMaskElement aRedMask(0x000000ff);
ColorMaskElement aGreenMask(0x0000ff00);
ColorMaskElement aBlueMask(0x00ff0000);
#endif
aBlueMask.CalcMaskShift();
aRedMask.CalcMaskShift();
aGreenMask.CalcMaskShift();
pBuf->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask, 0x000000ff);
break;
}
default:
// this is an error case !!!!!
nBitCount = 1;
diff --git a/vcl/opengl/salbmp.cxx b/vcl/opengl/salbmp.cxx
index 286c463..6144d9a 100644
--- a/vcl/opengl/salbmp.cxx
+++ b/vcl/opengl/salbmp.cxx
@@ -709,18 +709,44 @@ BitmapBuffer* OpenGLSalBitmap::AcquireBuffer( BitmapAccessMode nMode )
pBuffer->mnScanlineSize = mnBytesPerRow;
pBuffer->mpBits = maUserBuffer.get();
pBuffer->mnBitCount = mnBits;
switch( mnBits )
switch (mnBits)
{
case 1: pBuffer->mnFormat = BMP_FORMAT_1BIT_MSB_PAL; break;
case 4: pBuffer->mnFormat = BMP_FORMAT_4BIT_MSN_PAL; break;
case 8: pBuffer->mnFormat = BMP_FORMAT_8BIT_PAL; break;
case 16: pBuffer->mnFormat = BMP_FORMAT_16BIT_TC_MSB_MASK;
pBuffer->maColorMask = ColorMask( 0xf800, 0x07e0, 0x001f );
break;
case 24: pBuffer->mnFormat = BMP_FORMAT_24BIT_TC_RGB; break;
case 32: pBuffer->mnFormat = BMP_FORMAT_32BIT_TC_RGBA;
pBuffer->maColorMask = ColorMask( 0xff000000, 0x00ff0000, 0x0000ff00 );
break;
case 1:
pBuffer->mnFormat = BMP_FORMAT_1BIT_MSB_PAL;
break;
case 4:
pBuffer->mnFormat = BMP_FORMAT_4BIT_MSN_PAL;
break;
case 8:
pBuffer->mnFormat = BMP_FORMAT_8BIT_PAL;
break;
case 16:
{
pBuffer->mnFormat = BMP_FORMAT_16BIT_TC_MSB_MASK;
ColorMaskElement aRedMask(0xf800);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x07e0);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x001f);
aBlueMask.CalcMaskShift();
pBuffer->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
case 24:
pBuffer->mnFormat = BMP_FORMAT_24BIT_TC_RGB;
break;
case 32:
{
pBuffer->mnFormat = BMP_FORMAT_32BIT_TC_RGBA;
ColorMaskElement aRedMask(0xff000000);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x00ff0000);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x0000ff00);
aBlueMask.CalcMaskShift();
pBuffer->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
}
return pBuffer;
diff --git a/vcl/qa/cppunit/graphicfilter/data/bmp/pass/crash-1.bmp b/vcl/qa/cppunit/graphicfilter/data/bmp/pass/crash-1.bmp
new file mode 100644
index 0000000..84b6c35
--- /dev/null
+++ b/vcl/qa/cppunit/graphicfilter/data/bmp/pass/crash-1.bmp
Binary files differ
diff --git a/vcl/quartz/salbmp.cxx b/vcl/quartz/salbmp.cxx
index 8463d06..6699e80 100644
--- a/vcl/quartz/salbmp.cxx
+++ b/vcl/quartz/salbmp.cxx
@@ -759,16 +759,42 @@ BitmapBuffer* QuartzSalBitmap::AcquireBuffer( BitmapAccessMode /*nMode*/ )
pBuffer->mnBitCount = mnBits;
switch( mnBits )
{
case 1: pBuffer->mnFormat = BMP_FORMAT_1BIT_MSB_PAL; break;
case 4: pBuffer->mnFormat = BMP_FORMAT_4BIT_MSN_PAL; break;
case 8: pBuffer->mnFormat = BMP_FORMAT_8BIT_PAL; break;
case 16: pBuffer->mnFormat = BMP_FORMAT_16BIT_TC_MSB_MASK;
pBuffer->maColorMask = ColorMask( k16BitRedColorMask, k16BitGreenColorMask, k16BitBlueColorMask );
break;
case 24: pBuffer->mnFormat = BMP_FORMAT_24BIT_TC_BGR; break;
case 32: pBuffer->mnFormat = BMP_FORMAT_32BIT_TC_ARGB;
pBuffer->maColorMask = ColorMask( k32BitRedColorMask, k32BitGreenColorMask, k32BitBlueColorMask );
break;
case 1:
pBuffer->mnFormat = BMP_FORMAT_1BIT_MSB_PAL;
break;
case 4:
pBuffer->mnFormat = BMP_FORMAT_4BIT_MSN_PAL;
break;
case 8:
pBuffer->mnFormat = BMP_FORMAT_8BIT_PAL;
break;
case 16:
{
pBuffer->mnFormat = BMP_FORMAT_16BIT_TC_MSB_MASK;
ColorMaskElement aRedMask(k16BitRedColorMask);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(k16BitGreenColorMask);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(k16BitBlueColorMask);
aBlueMask.CalcMaskShift();
pBuffer->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
case 24:
pBuffer->mnFormat = BMP_FORMAT_24BIT_TC_BGR;
break;
case 32:
{
pBuffer->mnFormat = BMP_FORMAT_32BIT_TC_ARGB;
ColorMaskElement aRedMask(k32BitRedColorMask);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(k32BitGreenColorMask);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(k32BitBlueColorMask);
aBlueMask.CalcMaskShift();
pBuffer->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
}
pBuffer->mnFormat |= BMP_FORMAT_BOTTOM_UP;
diff --git a/vcl/source/gdi/dibtools.cxx b/vcl/source/gdi/dibtools.cxx
index d597ee1..bbd2658 100644
--- a/vcl/source/gdi/dibtools.cxx
+++ b/vcl/source/gdi/dibtools.cxx
@@ -596,7 +596,17 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
case( 16 ):
{
ColorMask aMask( nRMask, nGMask, nBMask );
ColorMaskElement aRedMask(nRMask);
if (!aRedMask.CalcMaskShift())
return false;
ColorMaskElement aGreenMask(nGMask);
if (!aGreenMask.CalcMaskShift())
return false;
ColorMaskElement aBlueMask(nBMask);
if (!aBlueMask.CalcMaskShift())
return false;
ColorMask aMask(aRedMask, aGreenMask, aBlueMask);
BitmapColor aColor;
sal_uInt16* pTmp16;
@@ -643,7 +653,17 @@ bool ImplReadDIBBits(SvStream& rIStm, DIBV5Header& rHeader, BitmapWriteAccess& r
case( 32 ):
{
ColorMask aMask(nRMask, nGMask, nBMask);
ColorMaskElement aRedMask(nRMask);
if (!aRedMask.CalcMaskShift())
return false;
ColorMaskElement aGreenMask(nGMask);
if (!aGreenMask.CalcMaskShift())
return false;
ColorMaskElement aBlueMask(nBMask);
if (!aBlueMask.CalcMaskShift())
return false;
ColorMask aMask(aRedMask, aGreenMask, aBlueMask);
BitmapColor aColor;
sal_uInt32* pTmp32;
diff --git a/vcl/unx/generic/gdi/salbmp.cxx b/vcl/unx/generic/gdi/salbmp.cxx
index c6c156b..8204449 100644
--- a/vcl/unx/generic/gdi/salbmp.cxx
+++ b/vcl/unx/generic/gdi/salbmp.cxx
@@ -154,14 +154,30 @@ BitmapBuffer* X11SalBitmap::ImplCreateDIB(
case( 8 ): pDIB->mnFormat |= BMP_FORMAT_8BIT_PAL; break;
#ifdef OSL_BIGENDIAN
case(16 ):
{
pDIB->mnFormat|= BMP_FORMAT_16BIT_TC_MSB_MASK;
pDIB->maColorMask = ColorMask( 0xf800, 0x07e0, 0x001f );
ColorMaskElement aRedMask(0xf800);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x07e0);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x001f);
aBlueMask.CalcMaskShift();
pDIB->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
#else
case(16 ):
{
pDIB->mnFormat|= BMP_FORMAT_16BIT_TC_LSB_MASK;
pDIB->maColorMask = ColorMask( 0xf800, 0x07e0, 0x001f );
ColorMaskElement aRedMask(0xf800);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x07e0);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x001f);
aBlueMask.CalcMaskShift();
pDIB->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
break;
}
#endif
default:
nBitCount = 24;
@@ -274,7 +290,13 @@ BitmapBuffer* X11SalBitmap::ImplCreateDIB(
case( 16 ):
{
aSrcBuf.maColorMask = ColorMask( pImage->red_mask, pImage->green_mask, pImage->blue_mask );
ColorMaskElement aRedMask(pImage->red_mask);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(pImage->green_mask);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(pImage->blue_mask);
aBlueMask.CalcMaskShift();
aSrcBuf.maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
if( LSBFirst == pImage->byte_order )
{
@@ -444,7 +466,13 @@ XImage* X11SalBitmap::ImplCreateXImage(
#endif
xMask.reset(new ColorMask( pImage->red_mask, pImage->green_mask, pImage->blue_mask ));
ColorMaskElement aRedMask(pImage->red_mask);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(pImage->green_mask);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(pImage->blue_mask);
aBlueMask.CalcMaskShift();
xMask.reset(new ColorMask(aRedMask, aGreenMask, aBlueMask));
}
break;
diff --git a/vcl/win/source/gdi/salbmp.cxx b/vcl/win/source/gdi/salbmp.cxx
index 0147005..eeaa1cd 100644
--- a/vcl/win/source/gdi/salbmp.cxx
+++ b/vcl/win/source/gdi/salbmp.cxx
@@ -906,14 +906,34 @@ BitmapBuffer* WinSalBitmap::AcquireBuffer( BitmapAccessMode /*nMode*/ )
if( pBIH->biCompression == BI_BITFIELDS )
{
nOffset = 3 * sizeof( RGBQUAD );
pBuffer->maColorMask = ColorMask( *(UINT32*) &pBI->bmiColors[ 0 ],
*(UINT32*) &pBI->bmiColors[ 1 ],
*(UINT32*) &pBI->bmiColors[ 2 ] );
ColorMaskElement aRedMask(*(UINT32*) &pBI->bmiColors[ 0 ]);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(*(UINT32*) &pBI->bmiColors[ 1 ]);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(*(UINT32*) &pBI->bmiColors[ 2 ]);
aBlueMask.CalcMaskShift();
pBuffer->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
}
else if( pBIH->biBitCount == 16 )
pBuffer->maColorMask = ColorMask( 0x00007c00UL, 0x000003e0UL, 0x0000001fUL );
{
ColorMaskElement aRedMask(0x00007c00UL);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x000003e0UL);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x0000001fUL);
aBlueMask.CalcMaskShift();
pBuffer->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
}
else
pBuffer->maColorMask = ColorMask( 0x00ff0000UL, 0x0000ff00UL, 0x000000ffUL );
{
ColorMaskElement aRedMask(0x00ff0000UL);
aRedMask.CalcMaskShift();
ColorMaskElement aGreenMask(0x0000ff00UL);
aGreenMask.CalcMaskShift();
ColorMaskElement aBlueMask(0x000000ffUL);
aBlueMask.CalcMaskShift();
pBuffer->maColorMask = ColorMask(aRedMask, aGreenMask, aBlueMask);
}
pBuffer->mpBits = (PBYTE) pBI + *(DWORD*) pBI + nOffset;
}
