tdf#156835 - FILEOPEN XLSX: add SHA-384 encryption support for ooxml import

Password protected file with SHA-384 encryption does not open before this
patch.

Change-Id: I482233f788b8e9da210ad6d2a6c4ece18d05d248
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/156282
Tested-by: Jenkins
Reviewed-by: Samuel Mehrbrodt <samuel.mehrbrodt@allotropia.de>
diff --git a/comphelper/source/misc/docpasswordhelper.cxx b/comphelper/source/misc/docpasswordhelper.cxx
index 5489690..0adb6ef 100644
--- a/comphelper/source/misc/docpasswordhelper.cxx
+++ b/comphelper/source/misc/docpasswordhelper.cxx
@@ -373,6 +373,8 @@ std::vector<unsigned char> DocPasswordHelper::GetOoxHashAsVector(
        eType = comphelper::HashType::SHA512;
    else if (rAlgorithmName == u"SHA-256" || rAlgorithmName == u"SHA256")
        eType = comphelper::HashType::SHA256;
    else if (rAlgorithmName == u"SHA-384" || rAlgorithmName == u"SHA384")
        eType = comphelper::HashType::SHA384;
    else if (rAlgorithmName == u"SHA-1" || rAlgorithmName == u"SHA1") // "SHA1" might be in the wild
        eType = comphelper::HashType::SHA1;
    else if (rAlgorithmName == u"MD5")
diff --git a/comphelper/source/misc/hash.cxx b/comphelper/source/misc/hash.cxx
index d537b9e..25b93ad 100644
--- a/comphelper/source/misc/hash.cxx
+++ b/comphelper/source/misc/hash.cxx
@@ -43,6 +43,8 @@ struct HashImpl
                return HASH_AlgSHA1;
            case HashType::SHA256:
                return HASH_AlgSHA256;
            case HashType::SHA384:
                return HASH_AlgSHA384;
            case HashType::SHA512:
                return HASH_AlgSHA512;
        }
@@ -62,6 +64,8 @@ struct HashImpl
                return EVP_sha1();
            case HashType::SHA256:
                return EVP_sha256();
            case HashType::SHA384:
                return EVP_sha384();
            case HashType::SHA512:
                return EVP_sha512();
        }
@@ -151,6 +155,8 @@ size_t Hash::getLength() const
            return SHA1_HASH_LENGTH;
        case HashType::SHA256:
            return SHA256_HASH_LENGTH;
        case HashType::SHA384:
            return SHA384_HASH_LENGTH;
        case HashType::SHA512:
            return SHA512_HASH_LENGTH;
    }
diff --git a/include/comphelper/hash.hxx b/include/comphelper/hash.hxx
index 54ab3a2..a3ad468 100644
--- a/include/comphelper/hash.hxx
+++ b/include/comphelper/hash.hxx
@@ -27,12 +27,14 @@ enum class HashType
    MD5,
    SHA1,
    SHA256,
    SHA384,
    SHA512
};

const sal_uInt32 MD5_HASH_LENGTH = RTL_DIGEST_LENGTH_MD5;
const sal_uInt32 SHA1_HASH_LENGTH = RTL_DIGEST_LENGTH_SHA1;
const sal_uInt32 SHA256_HASH_LENGTH = 32;
const sal_uInt32 SHA384_HASH_LENGTH = 48;
const sal_uInt32 SHA512_HASH_LENGTH = 64;

struct HashImpl;
diff --git a/include/oox/crypto/AgileEngine.hxx b/include/oox/crypto/AgileEngine.hxx
index ece4928..07ce3cb 100644
--- a/include/oox/crypto/AgileEngine.hxx
+++ b/include/oox/crypto/AgileEngine.hxx
@@ -70,6 +70,7 @@ struct OOX_DLLPUBLIC AgileEncryptionParameters
enum class AgileEncryptionPreset
{
    AES_128_SHA1,
    AES_128_SHA384,
    AES_256_SHA512,
};

diff --git a/include/oox/crypto/CryptTools.hxx b/include/oox/crypto/CryptTools.hxx
index c8c142007..10382b97 100644
--- a/include/oox/crypto/CryptTools.hxx
+++ b/include/oox/crypto/CryptTools.hxx
@@ -48,6 +48,7 @@ enum class CryptoHashType
{
    SHA1,
    SHA256,
    SHA384,
    SHA512
};

diff --git a/oox/qa/unit/CryptoTest.cxx b/oox/qa/unit/CryptoTest.cxx
index cbdf99b..290fce3 100644
--- a/oox/qa/unit/CryptoTest.cxx
+++ b/oox/qa/unit/CryptoTest.cxx
@@ -96,6 +96,15 @@ void CryptoTest::testCryptoHash()
    }

    {
        oox::crypto::CryptoHash aCryptoHash(aKey, oox::crypto::CryptoHashType::SHA384);
        aCryptoHash.update(aContent);
        std::vector<sal_uInt8> aHash = aCryptoHash.finalize();
        CPPUNIT_ASSERT_EQUAL(std::string("d7f4727e2c0b39ae0f1e40cc96f60242d5b7801841cea6fc592c5d3e1"
                                         "ae50700582a96cf35e1e554995fe4e03381c237"),
                             toString(aHash));
    }

    {
        oox::crypto::CryptoHash aCryptoHash(aKey, oox::crypto::CryptoHashType::SHA512);
        aCryptoHash.update(aContent);
        std::vector<sal_uInt8> aHash = aCryptoHash.finalize();
@@ -197,6 +206,13 @@ void CryptoTest::testAgileEncryptionVerifier()
    CPPUNIT_ASSERT_EQUAL(false, aEngine.decryptAndCheckVerifierHash("Wrong"));
    CPPUNIT_ASSERT_EQUAL(true, aEngine.decryptAndCheckVerifierHash(aPassword));

    aEngine.setupEncryptionParameters({ 100000, 16, 128, 48, 16, OUString("AES"),
                                        OUString("ChainingModeCBC"), OUString("SHA384") });

    CPPUNIT_ASSERT_EQUAL(true, aEngine.generateAndEncryptVerifierHash(aPassword));
    CPPUNIT_ASSERT_EQUAL(false, aEngine.decryptAndCheckVerifierHash("Wrong"));
    CPPUNIT_ASSERT_EQUAL(true, aEngine.decryptAndCheckVerifierHash(aPassword));

    aEngine.setupEncryptionParameters({ 100000, 16, 256, 64, 16, OUString("AES"),
                                        OUString("ChainingModeCBC"), OUString("SHA512") });

@@ -256,6 +272,52 @@ void CryptoTest::testAgileEncryptionInfoWritingAndParsing()
        }
    }

    { // Preset AES128 - SHA384
        SvMemoryStream aEncryptionInfo;
        {
            oox::crypto::AgileEngine aEngine;

            aEngine.setPreset(oox::crypto::AgileEncryptionPreset::AES_128_SHA384);
            aEngine.setupEncryption(aPassword);
            aKeyDataSalt = aEngine.getInfo().keyDataSalt;

            oox::BinaryXOutputStream aBinaryEncryptionInfoOutputStream(
                new utl::OSeekableOutputStreamWrapper(aEncryptionInfo), true);

            aEngine.writeEncryptionInfo(aBinaryEncryptionInfoOutputStream);
            aBinaryEncryptionInfoOutputStream.close();

            CPPUNIT_ASSERT_EQUAL(sal_uInt64(1040), aEncryptionInfo.GetSize());
        }

        aEncryptionInfo.Seek(STREAM_SEEK_TO_BEGIN);

        {
            oox::crypto::AgileEngine aEngine;

            uno::Reference<io::XInputStream> xInputStream(
                new utl::OSeekableInputStreamWrapper(aEncryptionInfo));

            xInputStream->skipBytes(4); // Encryption type -> Agile

            CPPUNIT_ASSERT(aEngine.readEncryptionInfo(xInputStream));

            oox::crypto::AgileEncryptionInfo& rInfo = aEngine.getInfo();
            CPPUNIT_ASSERT_EQUAL(sal_Int32(100000), rInfo.spinCount);
            CPPUNIT_ASSERT_EQUAL(sal_Int32(16), rInfo.saltSize);
            CPPUNIT_ASSERT_EQUAL(sal_Int32(128), rInfo.keyBits);
            CPPUNIT_ASSERT_EQUAL(sal_Int32(48), rInfo.hashSize);
            CPPUNIT_ASSERT_EQUAL(sal_Int32(16), rInfo.blockSize);
            CPPUNIT_ASSERT_EQUAL(OUString("AES"), rInfo.cipherAlgorithm);
            CPPUNIT_ASSERT_EQUAL(OUString("ChainingModeCBC"), rInfo.cipherChaining);
            CPPUNIT_ASSERT_EQUAL(OUString("SHA384"), rInfo.hashAlgorithm);
            CPPUNIT_ASSERT_EQUAL(toString(aKeyDataSalt), toString(rInfo.keyDataSalt));

            CPPUNIT_ASSERT_EQUAL(false, aEngine.decryptAndCheckVerifierHash("Wrong"));
            CPPUNIT_ASSERT_EQUAL(true, aEngine.decryptAndCheckVerifierHash(aPassword));
        }
    }

    { // Preset AES256 - SHA512
        SvMemoryStream aEncryptionInfo;
        {
diff --git a/oox/source/crypto/AgileEngine.cxx b/oox/source/crypto/AgileEngine.cxx
index f751849..ae2568d 100644
--- a/oox/source/crypto/AgileEngine.cxx
+++ b/oox/source/crypto/AgileEngine.cxx
@@ -206,6 +206,12 @@ bool hashCalc(std::vector<sal_uInt8>& output,
        output = out;
        return true;
    }
    else if (sAlgorithm == u"SHA384")
    {
        std::vector<unsigned char> out = comphelper::Hash::calculateHash(input.data(), input.size(), comphelper::HashType::SHA384);
        output = out;
        return true;
    }
    else if (sAlgorithm == u"SHA512")
    {
        std::vector<unsigned char> out = comphelper::Hash::calculateHash(input.data(), input.size(), comphelper::HashType::SHA512);
@@ -219,7 +225,10 @@ CryptoHashType cryptoHashTypeFromString(std::u16string_view sAlgorithm)
{
    if (sAlgorithm == u"SHA512")
        return CryptoHashType::SHA512;
    return CryptoHashType::SHA1;
    else if (sAlgorithm == u"SHA384")
        return CryptoHashType::SHA384;
    else
        return CryptoHashType::SHA1;
}

} // namespace
@@ -384,6 +393,8 @@ bool AgileEngine::decryptHmacKey()
    comphelper::HashType eType;
    if (mInfo.hashAlgorithm == "SHA1")
        eType = comphelper::HashType::SHA1;
    else if (mInfo.hashAlgorithm == "SHA384")
        eType = comphelper::HashType::SHA384;
    else if (mInfo.hashAlgorithm == "SHA512")
        eType = comphelper::HashType::SHA512;
    else
@@ -410,6 +421,8 @@ bool AgileEngine::decryptHmacValue()
    comphelper::HashType eType;
    if (mInfo.hashAlgorithm == "SHA1")
        eType = comphelper::HashType::SHA1;
    else if (mInfo.hashAlgorithm == "SHA384")
        eType = comphelper::HashType::SHA384;
    else if (mInfo.hashAlgorithm == "SHA512")
        eType = comphelper::HashType::SHA512;
    else
@@ -550,6 +563,16 @@ bool AgileEngine::readEncryptionInfo(uno::Reference<io::XInputStream> & rxInputS
        return true;
    }

    // AES 128 CBC with SHA384
    if (mInfo.keyBits         == 128 &&
        mInfo.cipherAlgorithm == "AES" &&
        mInfo.cipherChaining  == "ChainingModeCBC" &&
        mInfo.hashAlgorithm   == "SHA384" &&
        mInfo.hashSize        == comphelper::SHA384_HASH_LENGTH)
    {
        return true;
    }

    // AES 256 CBC with SHA512
    if (mInfo.keyBits         == 256 &&
        mInfo.cipherAlgorithm == "AES" &&
@@ -613,6 +636,8 @@ bool AgileEngine::encryptHmacKey()
    comphelper::HashType eType;
    if (mInfo.hashAlgorithm == "SHA1")
        eType = comphelper::HashType::SHA1;
    else if (mInfo.hashAlgorithm == "SHA384")
        eType = comphelper::HashType::SHA384;
    else if (mInfo.hashAlgorithm == "SHA512")
        eType = comphelper::HashType::SHA512;
    else
@@ -640,6 +665,8 @@ bool AgileEngine::encryptHmacValue()
    comphelper::HashType eType;
    if (mInfo.hashAlgorithm == "SHA1")
        eType = comphelper::HashType::SHA1;
    else if (mInfo.hashAlgorithm == "SHA384")
        eType = comphelper::HashType::SHA384;
    else if (mInfo.hashAlgorithm == "SHA512")
        eType = comphelper::HashType::SHA512;
    else
@@ -679,6 +706,8 @@ bool AgileEngine::setupEncryption(OUString const & rPassword)
{
    if (meEncryptionPreset == AgileEncryptionPreset::AES_128_SHA1)
        setupEncryptionParameters({ 100000, 16, 128, 20, 16, OUString("AES"), OUString("ChainingModeCBC"), OUString("SHA1") });
    else if (meEncryptionPreset == AgileEncryptionPreset::AES_128_SHA384)
        setupEncryptionParameters({ 100000, 16, 128, 48, 16, OUString("AES"), OUString("ChainingModeCBC"), OUString("SHA384") });
    else
        setupEncryptionParameters({ 100000, 16, 256, 64, 16, OUString("AES"), OUString("ChainingModeCBC"), OUString("SHA512") });

diff --git a/oox/source/crypto/CryptTools.cxx b/oox/source/crypto/CryptTools.cxx
index e0a4b9d..86d8ab2 100644
--- a/oox/source/crypto/CryptTools.cxx
+++ b/oox/source/crypto/CryptTools.cxx
@@ -117,6 +117,8 @@ struct CryptoImpl
                aEvpMd = EVP_sha1(); break;
            case CryptoHashType::SHA256:
                aEvpMd = EVP_sha256(); break;
            case CryptoHashType::SHA384:
                aEvpMd = EVP_sha384(); break;
            case CryptoHashType::SHA512:
                aEvpMd = EVP_sha512(); break;
        }
@@ -318,6 +320,9 @@ struct CryptoImpl
            case CryptoHashType::SHA256:
                aMechanism = CKM_SHA256_HMAC;
                break;
            case CryptoHashType::SHA384:
                aMechanism = CKM_SHA384_HMAC;
                break;
            case CryptoHashType::SHA512:
                aMechanism = CKM_SHA512_HMAC;
                break;
@@ -460,6 +465,7 @@ sal_Int32 getSizeForHashType(CryptoHashType eType)
    {
        case CryptoHashType::SHA1: return 20;
        case CryptoHashType::SHA256: return 32;
        case CryptoHashType::SHA384: return 48;
        case CryptoHashType::SHA512: return 64;
    }
    return 0;