Resolves: tdf#131442 Sort must not contain matrix formula except 1x1 array

Change-Id: Idc7a9646a70c59fceee0b36426f38a938cf073ce
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/91858
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Jenkins
(cherry picked from commit b1da67699bd05b26ee11460347ca7077d366c2fc)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/91723
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
diff --git a/sc/inc/column.hxx b/sc/inc/column.hxx
index 74fa4e8..4f839d0 100644
--- a/sc/inc/column.hxx
+++ b/sc/inc/column.hxx
@@ -234,7 +234,7 @@ public:
    void        FindUsed( SCROW nStartRow, SCROW nEndRow, mdds::flat_segment_tree<SCROW, bool>& rUsed ) const;

    SCSIZE             VisibleCount( SCROW nStartRow, SCROW nEndRow ) const;
    sc::MatrixEdge     GetBlockMatrixEdges(SCROW nRow1, SCROW nRow2, sc::MatrixEdge nMask ) const;
    sc::MatrixEdge     GetBlockMatrixEdges(SCROW nRow1, SCROW nRow2, sc::MatrixEdge nMask, bool bNoMatrixAtAll ) const;
    bool    HasSelectionMatrixFragment(const ScMarkData& rMark) const;

    bool    GetFirstVisibleAttr( SCROW& rFirstRow ) const;
diff --git a/sc/inc/document.hxx b/sc/inc/document.hxx
index 758bd4c..15fdc50 100644
--- a/sc/inc/document.hxx
+++ b/sc/inc/document.hxx
@@ -890,7 +890,8 @@ public:

    bool            IsBlockEditable( SCTAB nTab, SCCOL nStartCol, SCROW nStartRow,
                                     SCCOL nEndCol, SCROW nEndRow,
                                     bool* pOnlyNotBecauseOfMatrix = nullptr ) const;
                                     bool* pOnlyNotBecauseOfMatrix = nullptr,
                                     bool bNoMatrixAtAll = false ) const;
    bool            IsSelectionEditable( const ScMarkData& rMark,
                                         bool* pOnlyNotBecauseOfMatrix = nullptr ) const;
    bool            HasSelectedBlockMatrixFragment( SCCOL nStartCol, SCROW nStartRow,
diff --git a/sc/inc/table.hxx b/sc/inc/table.hxx
index bf97930..b2fca01 100644
--- a/sc/inc/table.hxx
+++ b/sc/inc/table.hxx
@@ -389,11 +389,13 @@ public:
    void        UnlockTable();

    bool        IsBlockEditable( SCCOL nCol1, SCROW nRow1, SCCOL nCol2,
                        SCROW nRow2, bool* pOnlyNotBecauseOfMatrix = nullptr ) const;
                        SCROW nRow2, bool* pOnlyNotBecauseOfMatrix = nullptr,
                        bool bNoMatrixAtAll = false ) const;
    bool        IsSelectionEditable( const ScMarkData& rMark,
                        bool* pOnlyNotBecauseOfMatrix = nullptr ) const;

    bool        HasBlockMatrixFragment( const SCCOL nCol1, SCROW nRow1, const SCCOL nCol2, SCROW nRow2 ) const;
    bool        HasBlockMatrixFragment( const SCCOL nCol1, SCROW nRow1, const SCCOL nCol2, SCROW nRow2,
                                        bool bNoMatrixAtAll = false ) const;
    bool        HasSelectionMatrixFragment( const ScMarkData& rMark ) const;

    bool        IsBlockEmpty( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, bool bIgnoreNotes ) const;
diff --git a/sc/source/core/data/column.cxx b/sc/source/core/data/column.cxx
index ebcc7df..5dd5420 100644
--- a/sc/source/core/data/column.cxx
+++ b/sc/source/core/data/column.cxx
@@ -109,7 +109,8 @@ SCROW ScColumn::GetNextUnprotected( SCROW nRow, bool bUp ) const
    return pAttrArray->GetNextUnprotected(nRow, bUp);
}

sc::MatrixEdge ScColumn::GetBlockMatrixEdges( SCROW nRow1, SCROW nRow2, sc::MatrixEdge nMask ) const
sc::MatrixEdge ScColumn::GetBlockMatrixEdges( SCROW nRow1, SCROW nRow2, sc::MatrixEdge nMask,
        bool bNoMatrixAtAll ) const
{
    using namespace sc;

@@ -162,6 +163,11 @@ sc::MatrixEdge ScColumn::GetBlockMatrixEdges( SCROW nRow1, SCROW nRow2, sc::Matr
            if (nEdges == MatrixEdge::Nothing)
                continue;

            // A 1x1 matrix array formula is OK even for no matrix at all.
            if (bNoMatrixAtAll
                    && (nEdges != (MatrixEdge::Top | MatrixEdge::Left | MatrixEdge::Bottom | MatrixEdge::Right)))
                return MatrixEdge::Inside;  // per convention Inside

            if (nEdges & MatrixEdge::Top)
                bOpen = true;       // top edge opens, keep on looking
            else if (!bOpen)
diff --git a/sc/source/core/data/document.cxx b/sc/source/core/data/document.cxx
index ec4a0e3..106360e 100644
--- a/sc/source/core/data/document.cxx
+++ b/sc/source/core/data/document.cxx
@@ -5308,7 +5308,8 @@ void ScDocument::UnlockTable(SCTAB nTab)

bool ScDocument::IsBlockEditable( SCTAB nTab, SCCOL nStartCol, SCROW nStartRow,
                                        SCCOL nEndCol, SCROW nEndRow,
                                        bool* pOnlyNotBecauseOfMatrix /* = NULL */ ) const
                                        bool* pOnlyNotBecauseOfMatrix /* = NULL */,
                                        bool bNoMatrixAtAll ) const
{
    // import into read-only document is possible
    if (!bImportingXML && !mbChangeReadOnlyEnabled && mpShell && mpShell->IsReadOnly())
@@ -5321,7 +5322,7 @@ bool ScDocument::IsBlockEditable( SCTAB nTab, SCCOL nStartCol, SCROW nStartRow,
    if (ValidTab(nTab) && nTab < static_cast<SCTAB>(maTabs.size()))
        if (maTabs[nTab])
            return maTabs[nTab]->IsBlockEditable( nStartCol, nStartRow, nEndCol,
                nEndRow, pOnlyNotBecauseOfMatrix );
                nEndRow, pOnlyNotBecauseOfMatrix, bNoMatrixAtAll );

    OSL_FAIL("wrong table number");
    if ( pOnlyNotBecauseOfMatrix )
diff --git a/sc/source/core/data/table2.cxx b/sc/source/core/data/table2.cxx
index c35c92a..1eb549b 100644
--- a/sc/source/core/data/table2.cxx
+++ b/sc/source/core/data/table2.cxx
@@ -2272,7 +2272,8 @@ void ScTable::FindMaxRotCol( RowInfo* pRowInfo, SCSIZE nArrCount, SCCOL nX1, SCC
    }
}

bool ScTable::HasBlockMatrixFragment( const SCCOL nCol1, SCROW nRow1, const SCCOL nCol2, SCROW nRow2 ) const
bool ScTable::HasBlockMatrixFragment( const SCCOL nCol1, SCROW nRow1, const SCCOL nCol2, SCROW nRow2,
        bool bNoMatrixAtAll ) const
{
    using namespace sc;

@@ -2286,28 +2287,38 @@ bool ScTable::HasBlockMatrixFragment( const SCCOL nCol1, SCROW nRow1, const SCCO
    if ( nCol1 == nMaxCol2 )
    {   // left and right column
        const MatrixEdge n = MatrixEdge::Left | MatrixEdge::Right;
        nEdges = aCol[nCol1].GetBlockMatrixEdges( nRow1, nRow2, n );
        nEdges = aCol[nCol1].GetBlockMatrixEdges( nRow1, nRow2, n, bNoMatrixAtAll );
        if ((nEdges != MatrixEdge::Nothing) && (((nEdges & n)!=n) || (nEdges & (MatrixEdge::Inside|MatrixEdge::Open))))
            return true;        // left or right edge is missing or open
    }
    else
    {   // left column
        nEdges = aCol[nCol1].GetBlockMatrixEdges(nRow1, nRow2, MatrixEdge::Left);
        nEdges = aCol[nCol1].GetBlockMatrixEdges(nRow1, nRow2, MatrixEdge::Left, bNoMatrixAtAll);
        if ((nEdges != MatrixEdge::Nothing) && ((!(nEdges & MatrixEdge::Left)) || (nEdges & (MatrixEdge::Inside|MatrixEdge::Open))))
            return true;        // left edge missing or open
        // right column
        nEdges = aCol[nMaxCol2].GetBlockMatrixEdges(nRow1, nRow2, MatrixEdge::Right);
        nEdges = aCol[nMaxCol2].GetBlockMatrixEdges(nRow1, nRow2, MatrixEdge::Right, bNoMatrixAtAll);
        if ((nEdges != MatrixEdge::Nothing) && ((!(nEdges & MatrixEdge::Right)) || (nEdges & (MatrixEdge::Inside|MatrixEdge::Open))))
            return true;        // right edge is missing or open
    }

    if ( nRow1 == nRow2 )
    if (bNoMatrixAtAll)
    {
        for (SCCOL i=nCol1; i<=nMaxCol2; i++)
        {
            nEdges = aCol[i].GetBlockMatrixEdges( nRow1, nRow2, MatrixEdge::Nothing, bNoMatrixAtAll);
            if (nEdges != MatrixEdge::Nothing
                    && (nEdges != (MatrixEdge::Top | MatrixEdge::Left | MatrixEdge::Bottom | MatrixEdge::Right)))
                return true;
        }
    }
    else if ( nRow1 == nRow2 )
    {   // Row on top and on bottom
        bool bOpen = false;
        const MatrixEdge n = MatrixEdge::Bottom | MatrixEdge::Top;
        for ( SCCOL i=nCol1; i<=nMaxCol2; i++)
        {
            nEdges = aCol[i].GetBlockMatrixEdges( nRow1, nRow1, n );
            nEdges = aCol[i].GetBlockMatrixEdges( nRow1, nRow1, n, bNoMatrixAtAll );
            if (nEdges != MatrixEdge::Nothing)
            {
                if ( (nEdges & n) != n )
@@ -2335,7 +2346,7 @@ bool ScTable::HasBlockMatrixFragment( const SCCOL nCol1, SCROW nRow1, const SCCO
            bool bOpen = false;
            for ( SCCOL i=nCol1; i<=nMaxCol2; i++)
            {
                nEdges = aCol[i].GetBlockMatrixEdges( nR, nR, n );
                nEdges = aCol[i].GetBlockMatrixEdges( nR, nR, n, bNoMatrixAtAll );
                if ( nEdges != MatrixEdge::Nothing)
                {
                    // in top row no top edge respectively
@@ -2374,7 +2385,8 @@ bool ScTable::HasSelectionMatrixFragment( const ScMarkData& rMark ) const
}

bool ScTable::IsBlockEditable( SCCOL nCol1, SCROW nRow1, SCCOL nCol2,
            SCROW nRow2, bool* pOnlyNotBecauseOfMatrix /* = NULL */ ) const
            SCROW nRow2, bool* pOnlyNotBecauseOfMatrix /* = NULL */,
            bool bNoMatrixAtAll ) const
{
    if ( !ValidColRow( nCol2, nRow2 ) )
    {
@@ -2442,7 +2454,7 @@ bool ScTable::IsBlockEditable( SCCOL nCol1, SCROW nRow1, SCCOL nCol2,
    }
    if ( bIsEditable )
    {
        if ( HasBlockMatrixFragment( nCol1, nRow1, nCol2, nRow2 ) )
        if (HasBlockMatrixFragment( nCol1, nRow1, nCol2, nRow2, bNoMatrixAtAll))
        {
            bIsEditable = false;
            if ( pOnlyNotBecauseOfMatrix )
diff --git a/sc/source/ui/docshell/dbdocfun.cxx b/sc/source/ui/docshell/dbdocfun.cxx
index b4dff2e..417c10d 100644
--- a/sc/source/ui/docshell/dbdocfun.cxx
+++ b/sc/source/ui/docshell/dbdocfun.cxx
@@ -513,8 +513,8 @@ bool ScDBDocFunc::Sort( SCTAB nTab, const ScSortParam& rSortParam,
        else
            nStartingColToEdit++;
    }
    ScEditableTester aTester( &rDoc, nTab, nStartingColToEdit,nStartingRowToEdit,
                                        aLocalParam.nCol2,aLocalParam.nRow2 );
    ScEditableTester aTester( &rDoc, nTab, nStartingColToEdit, nStartingRowToEdit,
            aLocalParam.nCol2, aLocalParam.nRow2, true /*bNoMatrixAtAll*/ );
    if (!aTester.IsEditable())
    {
        if (!bApi)
diff --git a/sc/source/ui/docshell/editable.cxx b/sc/source/ui/docshell/editable.cxx
index 8d461a0..ec9f071 100644
--- a/sc/source/ui/docshell/editable.cxx
+++ b/sc/source/ui/docshell/editable.cxx
@@ -29,11 +29,11 @@ ScEditableTester::ScEditableTester() :
}

ScEditableTester::ScEditableTester( const ScDocument* pDoc, SCTAB nTab,
                        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow ) :
        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow, bool bNoMatrixAtAll ) :
    mbIsEditable(true),
    mbOnlyMatrix(true)
{
    TestBlock( pDoc, nTab, nStartCol, nStartRow, nEndCol, nEndRow );
    TestBlock( pDoc, nTab, nStartCol, nStartRow, nEndCol, nEndRow, bNoMatrixAtAll );
}

ScEditableTester::ScEditableTester( const ScDocument* pDoc,
@@ -80,12 +80,12 @@ ScEditableTester::ScEditableTester(
}

void ScEditableTester::TestBlock( const ScDocument* pDoc, SCTAB nTab,
                        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow )
        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow, bool bNoMatrixAtAll )
{
    if (mbIsEditable || mbOnlyMatrix)
    {
        bool bThisMatrix;
        if ( !pDoc->IsBlockEditable( nTab, nStartCol, nStartRow, nEndCol, nEndRow, &bThisMatrix ) )
        if (!pDoc->IsBlockEditable( nTab, nStartCol, nStartRow, nEndCol, nEndRow, &bThisMatrix, bNoMatrixAtAll))
        {
            mbIsEditable = false;
            if ( !bThisMatrix )
@@ -104,7 +104,7 @@ void ScEditableTester::TestSelectedBlock( const ScDocument* pDoc,
        if (rTab >= nTabCount)
            break;

        TestBlock( pDoc, rTab, nStartCol, nStartRow, nEndCol, nEndRow );
        TestBlock( pDoc, rTab, nStartCol, nStartRow, nEndCol, nEndRow, false );
    }
}

@@ -117,7 +117,7 @@ void ScEditableTester::TestRange(  const ScDocument* pDoc, const ScRange& rRange
    SCROW nEndRow = rRange.aEnd.Row();
    SCTAB nEndTab = rRange.aEnd.Tab();
    for (SCTAB nTab=nStartTab; nTab<=nEndTab; nTab++)
        TestBlock( pDoc, nTab, nStartCol, nStartRow, nEndCol, nEndRow );
        TestBlock( pDoc, nTab, nStartCol, nStartRow, nEndCol, nEndRow, false );
}

void ScEditableTester::TestSelection( const ScDocument* pDoc, const ScMarkData& rMark )
diff --git a/sc/source/ui/inc/editable.hxx b/sc/source/ui/inc/editable.hxx
index 05eaa98..97d28ed 100644
--- a/sc/source/ui/inc/editable.hxx
+++ b/sc/source/ui/inc/editable.hxx
@@ -41,8 +41,12 @@ public:
            ScEditableTester();

            // calls TestBlock
            /** @param  bNoMatrixAtAll
                        TRUE if there must not be any matrix, not even entirely
                        containted; for example in sorting. */
            ScEditableTester( const ScDocument* pDoc, SCTAB nTab,
                        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow );
                        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow,
                        bool bNoMatrixAtAll = false );

            // calls TestSelectedBlock
            ScEditableTester( const ScDocument* pDoc,
@@ -65,7 +69,8 @@ public:
            // Several calls to the Test... methods check if *all* of the ranges
            // are editable. For several independent checks, Reset() has to be used.
    void    TestBlock( const ScDocument* pDoc, SCTAB nTab,
                        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow );
                        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow,
                        bool bNoMatrixAtAll = false );
    void    TestSelectedBlock( const ScDocument* pDoc,
                        SCCOL nStartCol, SCROW nStartRow, SCCOL nEndCol, SCROW nEndRow,
                        const ScMarkData& rMark );