commit 49af7e22e61c2e5d440ad55cd362388983e128ae
author Eike Rathke <erack@redhat.com> Sat May 22 20:46:35 2021 +0200
committer Eike Rathke <erack@redhat.com> Sun May 23 03:04:56 2021 +0200
tree fb81babe1c80900f9bd4f6a1e13daac6e60bb61c
parent 60cba23bc0f5e8eafecc03c437f1133b62569fa6

Related: tdf#136794 tdf#137063 Unlimit decimals in rtl_math_round()

Change-Id: I7f9fb45824458d578ee302a668386fe0c3c80974
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/116010
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Jenkins

sal/rtl/math.cxx

diff --git a/sal/rtl/math.cxx b/sal/rtl/math.cxx
index a85c8ac..698e158 100644
--- a/sal/rtl/math.cxx
+++ b/sal/rtl/math.cxx
@@ -1131,8 +1131,6 @@ double SAL_CALL rtl_math_round(double fValue, int nDecPlaces,
                               enum rtl_math_RoundingMode eMode)
    SAL_THROW_EXTERN_C()
{
    OSL_ASSERT(nDecPlaces >= -20 && nDecPlaces <= 20);

    if (!std::isfinite(fValue))
        return fValue;

@@ -1142,6 +1140,8 @@ double SAL_CALL rtl_math_round(double fValue, int nDecPlaces,
    if ( nDecPlaces == 0 && eMode == rtl_math_RoundingMode_Corrected )
        return std::round( fValue );

    const double fOrigValue = fValue;

    // sign adjustment
    bool bSign = std::signbit( fValue );
    if (bSign)
@@ -1153,42 +1153,50 @@ double SAL_CALL rtl_math_round(double fValue, int nDecPlaces,
    if (nDecPlaces >= 0
            && (fValue >= (static_cast<sal_Int64>(1) << 52)
                || isRepresentableInteger(fValue)))
        return bSign ? -fValue : fValue;
        return fOrigValue;

    double fFac = 0;
    if (nDecPlaces != 0)
    {
        if (nDecPlaces > 1 && fValue > 4294967296.0)
        if (nDecPlaces > 0)
        {
            // 4294967296 is 2^32 with room for at least 20 decimals, checking
            // smaller values is not necessary. Lower the limit if more than 20
            // decimals were to be allowed.

            // Determine how many decimals are representable in the precision.
            // Anything greater 2^52 and 0.0 was already ruled out above.
            // Theoretically 0.5, 0.25, 0.125, 0.0625, 0.03125, ...
            const sal_math_Double* pd = reinterpret_cast<const sal_math_Double*>(&fValue);
            const sal_Int32 nDec = 52 - (pd->parts.exponent - 1023);

            if (nDec <= 0)
            {
                assert(!"Shouldn't this had been caught already as large number?");
                return fOrigValue;
            }

            if (nDec < nDecPlaces)
                nDecPlaces = nDec;
        }

        /* TODO: this was without the inverse factor and determining max
         * possible decimals, it could now be adjusted to be more lenient. */
        // max 20 decimals, we don't have unlimited precision
        // #38810# and no overflow on fValue*=fFac
        if (nDecPlaces < -20 || 20 < nDecPlaces || fValue > (DBL_MAX / 1e20))
            return bSign ? -fValue : fValue;

        // Avoid 1e-5 (1.0000000000000001e-05) and such inaccurate fractional
        // factors that later when dividing back spoil things. For negative
        // decimals divide first with the inverse, then multiply the rounded
        // value back.
        fFac = getN10Exp(abs(nDecPlaces));

        if (fFac == 0.0 || (nDecPlaces < 0 && !std::isfinite(fFac)))
            // Underflow, rounding to that many integer positions would be 0.
            return 0.0;

        if (!std::isfinite(fFac))
            // Overflow with very small values and high number of decimals.
            return fOrigValue;

        if (nDecPlaces < 0)
            fValue /= fFac;
        else
            fValue *= fFac;

        if (!std::isfinite(fValue))
            return fOrigValue;
    }

    // Round only if not already in distance precision gaps of integers, where
@@ -1276,6 +1284,9 @@ double SAL_CALL rtl_math_round(double fValue, int nDecPlaces,
            fValue /= fFac;
    }

    if (!std::isfinite(fValue))
        return fOrigValue;

    return bSign ? -fValue : fValue;
}