Assert flags passed to rtl_convertTextToUnicode/UnicodeToText are valid

...so that at least some typos of using OUSTRING_TO_OSTRING_CVTFLAGS (0x566)
instead of OSTRING_TO_OUSTRING_CVTFLAGS (0x333) can be found.  (Unfortunately,
in the other direction, 0x333 is a valid combination of
RTL_UNICODETOTEXT_FLAGS_*.)

Change-Id: I7cfb3677b103ae90de88833cc93b0a5384607e15
Reviewed-on: https://gerrit.libreoffice.org/42288
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>
diff --git a/sal/osl/unx/security.cxx b/sal/osl/unx/security.cxx
index d1d39c3..dafa69f 100644
--- a/sal/osl/unx/security.cxx
+++ b/sal/osl/unx/security.cxx
@@ -163,7 +163,7 @@ sal_Bool SAL_CALL osl_getUserIdent(oslSecurity Security, rtl_uString **ustrIdent

    bRet = osl_psz_getUserIdent(Security,pszIdent,sizeof(pszIdent));

    rtl_string2UString( ustrIdent, pszIdent, rtl_str_getLength( pszIdent ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
    rtl_string2UString( ustrIdent, pszIdent, rtl_str_getLength( pszIdent ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
    SAL_WARN_IF(*ustrIdent == nullptr, "sal.osl", "*ustrIdent == NULL");

    return bRet;
@@ -197,7 +197,7 @@ sal_Bool SAL_CALL osl_getUserName(oslSecurity Security, rtl_uString **ustrName)

    bRet = osl_psz_getUserName(Security,pszName,sizeof(pszName));

    rtl_string2UString( ustrName, pszName, rtl_str_getLength( pszName ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
    rtl_string2UString( ustrName, pszName, rtl_str_getLength( pszName ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
    SAL_WARN_IF(*ustrName == nullptr, "sal.osl", "ustrName == NULL");

    return bRet;
@@ -231,7 +231,7 @@ sal_Bool SAL_CALL osl_getHomeDir(oslSecurity Security, rtl_uString **pustrDirect

    if ( bRet )
    {
        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
        SAL_WARN_IF(*pustrDirectory == nullptr, "sal.osl", "*pustrDirectory == NULL");
        osl_getFileURLFromSystemPath( *pustrDirectory, pustrDirectory );
    }
@@ -339,7 +339,7 @@ sal_Bool SAL_CALL osl_getConfigDir(oslSecurity Security, rtl_uString **pustrDire

    if ( bRet )
    {
        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OUSTRING_TO_OSTRING_CVTFLAGS );
        rtl_string2UString( pustrDirectory, pszDirectory, rtl_str_getLength( pszDirectory ), osl_getThreadTextEncoding(), OSTRING_TO_OUSTRING_CVTFLAGS );
        SAL_WARN_IF(*pustrDirectory == nullptr, "sal.osl", "*pustrDirectory == NULL");
        osl_getFileURLFromSystemPath( *pustrDirectory, pustrDirectory );
    }
diff --git a/sal/osl/w32/file_url.cxx b/sal/osl/w32/file_url.cxx
index 5b0b545..bb6cd00 100644
--- a/sal/osl/w32/file_url.cxx
+++ b/sal/osl/w32/file_url.cxx
@@ -539,7 +539,7 @@ static bool osl_decodeURL_( rtl_String* strUTF8, rtl_uString** pstrDecodedURL )

    if ( bValidEncoded )
    {
        rtl_string2UString( pstrDecodedURL, pBuffer, rtl_str_getLength(pBuffer), RTL_TEXTENCODING_UTF8, OUSTRING_TO_OSTRING_CVTFLAGS );
        rtl_string2UString( pstrDecodedURL, pBuffer, rtl_str_getLength(pBuffer), RTL_TEXTENCODING_UTF8, OSTRING_TO_OUSTRING_CVTFLAGS );
        OSL_ASSERT(*pstrDecodedURL != nullptr);
    }

diff --git a/sal/textenc/textcvt.cxx b/sal/textenc/textcvt.cxx
index b102497..0391483 100644
--- a/sal/textenc/textcvt.cxx
+++ b/sal/textenc/textcvt.cxx
@@ -19,6 +19,8 @@

#include "sal/config.h"

#include <cassert>

#include "rtl/textcvt.h"

#include "gettextencodingdata.hxx"
@@ -157,6 +159,24 @@ sal_Size SAL_CALL rtl_convertTextToUnicode( rtl_TextToUnicodeConverter hConverte
                                            sal_uInt32 nFlags, sal_uInt32* pInfo,
                                            sal_Size* pSrcCvtBytes )
{
    assert(
        (nFlags & RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_MASK)
        <= RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_DEFAULT);
    assert(
        (nFlags & RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_MASK)
        <= RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_DEFAULT);
    assert(
        (nFlags & RTL_TEXTTOUNICODE_FLAGS_INVALID_MASK)
        <= RTL_TEXTTOUNICODE_FLAGS_INVALID_DEFAULT);
    assert(
        (nFlags
         & ~(RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_MASK
             | RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_MASK
             | RTL_TEXTTOUNICODE_FLAGS_INVALID_MASK
             | RTL_TEXTTOUNICODE_FLAGS_FLUSH
             | RTL_TEXTTOUNICODE_FLAGS_GLOBAL_SIGNATURE))
        == 0);

    const ImplTextConverter* pConverter = static_cast<const ImplTextConverter*>(hConverter);

    /* Only temporary, because we don't want die, if we don't have a
@@ -233,6 +253,27 @@ sal_Size SAL_CALL rtl_convertUnicodeToText( rtl_UnicodeToTextConverter hConverte
                                            sal_uInt32 nFlags, sal_uInt32* pInfo,
                                            sal_Size* pSrcCvtChars )
{
    assert(
        (nFlags & RTL_UNICODETOTEXT_FLAGS_UNDEFINED_MASK)
        <= RTL_UNICODETOTEXT_FLAGS_UNDEFINED_DEFAULT);
    assert(
        (nFlags & RTL_UNICODETOTEXT_FLAGS_INVALID_MASK)
        <= RTL_UNICODETOTEXT_FLAGS_INVALID_DEFAULT);
    assert(
        (nFlags
         & ~(RTL_UNICODETOTEXT_FLAGS_UNDEFINED_MASK
             | RTL_UNICODETOTEXT_FLAGS_INVALID_MASK
             | RTL_UNICODETOTEXT_FLAGS_UNDEFINED_REPLACE
             | RTL_UNICODETOTEXT_FLAGS_UNDEFINED_REPLACESTR
             | RTL_UNICODETOTEXT_FLAGS_PRIVATE_MAPTO0
             | RTL_UNICODETOTEXT_FLAGS_NONSPACING_IGNORE
             | RTL_UNICODETOTEXT_FLAGS_CONTROL_IGNORE
             | RTL_UNICODETOTEXT_FLAGS_PRIVATE_IGNORE
             | RTL_UNICODETOTEXT_FLAGS_NOCOMPOSITE
             | RTL_UNICODETOTEXT_FLAGS_FLUSH
             | RTL_UNICODETOTEXT_FLAGS_GLOBAL_SIGNATURE))
        == 0);

    const ImplTextConverter* pConverter = static_cast<const ImplTextConverter*>(hConverter);

    /* Only temporary, because we don't want die, if we don't have a
diff --git a/tools/qa/cppunit/test_reversemap.cxx b/tools/qa/cppunit/test_reversemap.cxx
index d6da18a5..e9b7adc 100644
--- a/tools/qa/cppunit/test_reversemap.cxx
+++ b/tools/qa/cppunit/test_reversemap.cxx
@@ -60,7 +60,7 @@ namespace
            aAllChars[i-1] = static_cast<sal_Char>(i);

        //Some slots are unused, so don't map to private, just set them to 'X'
        sal_uInt32 const convertFlags = OUSTRING_TO_OSTRING_CVTFLAGS ^ RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_MAPTOPRIVATE;
        sal_uInt32 const convertFlags = OSTRING_TO_OUSTRING_CVTFLAGS ^ RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_MAPTOPRIVATE;
        OUString sOrigText(&aAllChars[0], aAllChars.size(), eEncoding, convertFlags);
        sOrigText = sOrigText.replace( 0xfffd, 'X' );