add initial file format representation for data import feature
Change-Id: I51143ecfe4eb1584f13bd1590f927743de8fa91e
Reviewed-on: https://gerrit.libreoffice.org/40572
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>
diff --git a/include/xmloff/xmltoken.hxx b/include/xmloff/xmltoken.hxx
index 02037f1..e1c0c4d 100644
--- a/include/xmloff/xmltoken.hxx
+++ b/include/xmloff/xmltoken.hxx
@@ -3256,6 +3256,10 @@ namespace xmloff { namespace token {
XML_MARGIN, // #i117001#
XML_PROPERTY_MAPPING,
XML_PROVIDER,
XML_DATA_MAPPINGS,
XML_DATA_MAPPING,
XML_DATA_FREQUENCY,
// regina, ODF1.2 additional symbols in charts
XML_STAR,
diff --git a/sc/Library_sc.mk b/sc/Library_sc.mk
index 87bd70a..e9220df 100644
--- a/sc/Library_sc.mk
+++ b/sc/Library_sc.mk
@@ -325,6 +325,7 @@ $(eval $(call gb_Library_add_exception_objects,sc,\
sc/source/filter/xml/xmlfonte \
sc/source/filter/xml/xmlimprt \
sc/source/filter/xml/xmllabri \
sc/source/filter/xml/xmlmappingi \
sc/source/filter/xml/xmlnexpi \
sc/source/filter/xml/xmlrowi \
sc/source/filter/xml/xmlsceni \
diff --git a/sc/Module_sc.mk b/sc/Module_sc.mk
index 338ad5cd..25c987f 100644
--- a/sc/Module_sc.mk
+++ b/sc/Module_sc.mk
@@ -60,7 +60,6 @@ $(eval $(call gb_Module_add_slowcheck_targets,sc, \
CppunitTest_sc_subsequent_export_test \
CppunitTest_sc_html_export_test \
CppunitTest_sc_copypaste \
CppunitTest_sc_dataproviders_test \
))
# Various function tests fail in 32-bit linux_x86 build due to dreaded floating
diff --git a/sc/inc/datamapper.hxx b/sc/inc/datamapper.hxx
new file mode 100644
index 0000000..28476df
--- /dev/null
+++ b/sc/inc/datamapper.hxx
@@ -0,0 +1,106 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#ifndef INCLUDED_SC_INC_EXTERNALDATAMAPPER_HXX
#define INCLUDED_SC_INC_EXTERNALDATAMAPPER_HXX
#include <memory>
#include <rtl/ustring.hxx>
class ScDocument;
class ScDBData;
namespace sc {
class ScDBDataManager;
class DataProvider;
class ScDBDataManager;
class ExternalDataSource
{
private:
/**
* The URL for the external data provider. The URL
* will be passed to the data provider together with
* the ID.
*
* A data provider may decide to ignore the URL string.
*/
OUString maURL;
/**
* The data provider is a unique identifier that will
* allow to identify and instantiate the required data
* provider.
*
* Examples for the internal data providers are:
*
* org.libreoffice.dataprovider.calc.csv
* org.libreoffice.dataprovider.calc.json
*
* Only internal data providers should use the:
* "org.libreoffice.dataprovider prefix".
*/
OUString maProvider;
/**
* The ID allows the same data provider to support different
* data streams.
*
* A data provider may decide to ignore the ID string.
*/
OUString maID;
double mnUpdateFrequency;
std::shared_ptr<DataProvider> mpDataProvider;
std::shared_ptr<ScDBDataManager> mpDBDataManager;
public:
ExternalDataSource(const OUString& rURL,
const OUString& rProvider);
void setUpdateFrequency(double nUpdateFrequency);
void setID(const OUString& rID);
const OUString& getURL() const;
const OUString& getProvider() const;
const OUString& getID() const;
double getUpdateFrequency() const;
OUString getDBName() const;
void setDBData(ScDBData* pDBData);
void refresh(ScDocument* pDoc);
};
class SC_DLLPUBLIC ExternalDataMapper
{
//ScDocument* mpDoc;
std::vector<ExternalDataSource> maDataSources;
public:
ExternalDataMapper(ScDocument* pDoc);
~ExternalDataMapper();
void insertDataSource(const ExternalDataSource& rSource);
const std::vector<ExternalDataSource>& getDataSources() const;
std::vector<ExternalDataSource>& getDataSources();
};
}
#endif
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/inc/document.hxx b/sc/inc/document.hxx
index bf59343..2cc5f02 100644
--- a/sc/inc/document.hxx
+++ b/sc/inc/document.hxx
@@ -86,6 +86,7 @@ class ColumnSet;
class UpdatedRangeNames;
class TableColumnBlockPositionSet;
class ColumnIterator;
class ExternalDataMapper;
}
@@ -350,6 +351,7 @@ private:
ScRefreshTimerControl* pRefreshTimerControl;
std::shared_ptr<SvxForbiddenCharactersTable> xForbiddenCharacters;
ScDBData* mpAnonymousDBData;
std::unique_ptr<sc::ExternalDataMapper> mpDataMapper;
ScFieldEditEngine* pCacheFieldEditEngine;
@@ -708,6 +710,7 @@ public:
const ScDBData* GetDBAtArea(SCTAB nTab, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2) const;
ScDBData* GetDBAtArea(SCTAB nTab, SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2);
void RefreshDirtyTableColumnNames();
sc::ExternalDataMapper& GetExternalDataMapper();
SC_DLLPUBLIC const ScRangeData* GetRangeAtBlock( const ScRange& rBlock, OUString* pName ) const;
diff --git a/sc/source/core/data/documen2.cxx b/sc/source/core/data/documen2.cxx
index 104cd45..973231d 100644
--- a/sc/source/core/data/documen2.cxx
+++ b/sc/source/core/data/documen2.cxx
@@ -103,6 +103,7 @@
#include "docsh.hxx"
#include "clipoptions.hxx"
#include <listenercontext.hxx>
#include "datamapper.hxx"
using namespace com::sun::star;
diff --git a/sc/source/core/data/document10.cxx b/sc/source/core/data/document10.cxx
index 4768bfe..89c1975 100644
--- a/sc/source/core/data/document10.cxx
+++ b/sc/source/core/data/document10.cxx
@@ -29,6 +29,7 @@
#include <svl/whiter.hxx>
#include <editeng/colritem.hxx>
#include "scitems.hxx"
#include "datamapper.hxx"
// Add totally brand-new methods to this source file.
@@ -930,4 +931,12 @@ void ScDocument::EnsureFormulaCellResults( const ScRange& rRange )
}
}
sc::ExternalDataMapper& ScDocument::GetExternalDataMapper()
{
if (!mpDataMapper)
mpDataMapper.reset(new sc::ExternalDataMapper(this));
return *mpDataMapper;
}
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/filter/xml/xmlbodyi.cxx b/sc/source/filter/xml/xmlbodyi.cxx
index ec36351..159c25a 100644
--- a/sc/source/filter/xml/xmlbodyi.cxx
+++ b/sc/source/filter/xml/xmlbodyi.cxx
@@ -32,6 +32,7 @@
#include "xmlcvali.hxx"
#include "xmlstyli.hxx"
#include "xmllabri.hxx"
#include "xmlmappingi.hxx"
#include "XMLConsolidationContext.hxx"
#include "XMLDDELinksContext.hxx"
#include "XMLCalculationSettingsContext.hxx"
@@ -172,6 +173,10 @@ uno::Reference< xml::sax::XFastContextHandler > SAL_CALL
pContext = new ScXMLDatabaseRangesContext ( GetScImport(), nElement,
xAttrList );
break;
case XML_ELEMENT( CALC_EXT, XML_DATA_MAPPINGS ):
pContext = new ScXMLMappingsContext(GetScImport(), nElement,
xAttrList);
break;
case XML_ELEMENT( TABLE, XML_DATABASE_RANGE ):
pContext = new ScXMLDatabaseRangeContext ( GetScImport(), nElement,
xAttrList );
diff --git a/sc/source/filter/xml/xmlexprt.cxx b/sc/source/filter/xml/xmlexprt.cxx
index 7d79adb..6011398 100644
--- a/sc/source/filter/xml/xmlexprt.cxx
+++ b/sc/source/filter/xml/xmlexprt.cxx
@@ -68,6 +68,7 @@
#include <documentlinkmgr.hxx>
#include <tokenstringcontext.hxx>
#include <cellform.hxx>
#include "datamapper.hxx"
#include <xmloff/xmltoken.hxx>
#include <xmloff/xmlnmspe.hxx>
@@ -1933,6 +1934,7 @@ void ScXMLExport::ExportContent_()
WriteNamedExpressions();
WriteDataStream();
aExportDatabaseRanges.WriteDatabaseRanges();
WriteExternalDataMapping();
ScXMLExportDataPilot aExportDataPilot(*this);
aExportDataPilot.WriteDataPilots(xSpreadDoc);
WriteConsolidation();
@@ -4018,6 +4020,32 @@ void ScXMLExport::WriteNamedExpressions()
WriteNamedRange(pNamedRanges);
}
void ScXMLExport::WriteExternalDataMapping()
{
if (!pDoc)
return;
if (getDefaultVersion() <= SvtSaveOptions::ODFVER_012)
// Export this only for 1.2 extended and above.
return;
sc::ExternalDataMapper& rDataMapper = pDoc->GetExternalDataMapper();
auto& rDataSources = rDataMapper.getDataSources();
if (!rDataSources.empty())
{
SvXMLElementExport aMappings(*this, XML_NAMESPACE_CALC_EXT, XML_DATA_MAPPINGS, true, true);
for (auto& itr : rDataSources)
{
AddAttribute(XML_NAMESPACE_XLINK, XML_HREF, itr.getURL());
AddAttribute(XML_NAMESPACE_CALC_EXT, XML_PROVIDER, itr.getProvider());
AddAttribute(XML_NAMESPACE_CALC_EXT, XML_DATA_FREQUENCY, OUString::number(itr.getUpdateFrequency()));
AddAttribute(XML_NAMESPACE_CALC_EXT, XML_ID, itr.getID());
AddAttribute(XML_NAMESPACE_CALC_EXT, XML_DATABASE_NAME, itr.getDBName());
SvXMLElementExport aMapping(*this, XML_NAMESPACE_CALC_EXT, XML_DATA_MAPPING, true, true);
}
}
}
void ScXMLExport::WriteDataStream()
{
if (!pDoc)
diff --git a/sc/source/filter/xml/xmlexprt.hxx b/sc/source/filter/xml/xmlexprt.hxx
index 675b818..e232415 100644
--- a/sc/source/filter/xml/xmlexprt.hxx
+++ b/sc/source/filter/xml/xmlexprt.hxx
@@ -197,6 +197,7 @@ class ScXMLExport : public SvXMLExport
void WriteTheLabelRanges(const css::uno::Reference< css::sheet::XSpreadsheetDocument >& xSpreadDoc);
void WriteLabelRanges( const css::uno::Reference< css::container::XIndexAccess >& xRangesIAccess, bool bColumn );
void WriteNamedExpressions();
void WriteExternalDataMapping();
void WriteDataStream();
void WriteNamedRange(ScRangeName* pRangeName);
void ExportConditionalFormat(SCTAB nTab);
diff --git a/sc/source/filter/xml/xmlmappingi.cxx b/sc/source/filter/xml/xmlmappingi.cxx
new file mode 100644
index 0000000..f65c929
--- /dev/null
+++ b/sc/source/filter/xml/xmlmappingi.cxx
@@ -0,0 +1,143 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#include "xmlmappingi.hxx"
#include <xmloff/xmltkmap.hxx>
#include <xmloff/nmspmap.hxx>
#include <xmloff/xmltoken.hxx>
#include <xmloff/xmlnmspe.hxx>
#include <xmloff/xmlerror.hxx>
#include "datamapper.hxx"
#include "document.hxx"
#include "dbdata.hxx"
#include <sax/tools/converter.hxx>
using namespace com::sun::star;
using namespace xmloff::token;
ScXMLMappingsContext::ScXMLMappingsContext( ScXMLImport& rImport,
sal_Int32 /*nElement*/,
const css::uno::Reference<css::xml::sax::XFastAttributeList>& /* xAttrList */ ) :
ScXMLImportContext( rImport )
{
// has no attributes
rImport.LockSolarMutex();
}
ScXMLMappingsContext::~ScXMLMappingsContext()
{
GetScImport().UnlockSolarMutex();
}
uno::Reference< xml::sax::XFastContextHandler > SAL_CALL ScXMLMappingsContext::createFastChildContext(
sal_Int32 nElement,
const uno::Reference< xml::sax::XFastAttributeList >& xAttrList )
{
SvXMLImportContext *pContext = nullptr;
switch( nElement )
{
case XML_ELEMENT( CALC_EXT, XML_DATA_MAPPING ):
{
pContext = new ScXMLMappingContext( GetScImport(), nElement, xAttrList );
}
break;
}
if( !pContext )
pContext = new SvXMLImportContext( GetImport() );
return pContext;
}
ScXMLMappingContext::ScXMLMappingContext( ScXMLImport& rImport,
sal_Int32 /*nElement*/,
const css::uno::Reference<css::xml::sax::XFastAttributeList>& xAttrList) :
ScXMLImportContext( rImport )
{
OUString aProvider;
OUString aID;
OUString aURL;
// OUString aFrequency;
OUString aDBName;
if( xAttrList.is() )
{
sax_fastparser::FastAttributeList *pAttribList =
sax_fastparser::FastAttributeList::castToFastAttributeList( xAttrList );
for( auto &aIter : *pAttribList )
{
switch( aIter.getToken() )
{
case XML_ELEMENT( XLINK, XML_HREF ):
{
aURL = aIter.toString();
}
break;
case XML_ELEMENT( CALC_EXT, XML_PROVIDER ):
{
aProvider = aIter.toString();
}
break;
case XML_ELEMENT( CALC_EXT, XML_ID ):
{
aID = aIter.toString();
}
break;
case XML_ELEMENT( CALC_EXT, XML_DATABASE_NAME ):
{
aDBName = aIter.toString();
}
break;
case XML_ELEMENT( CALC_EXT, XML_DATA_FREQUENCY ):
{
}
break;
}
}
}
if (!aProvider.isEmpty())
{
ScDocument* pDoc = GetScImport().GetDocument();
ScDBData* pDBData = pDoc->GetDBCollection()->getNamedDBs().findByUpperName(ScGlobal::pCharClass->uppercase(aDBName));
if (pDBData)
{
auto& rDataMapper = pDoc->GetExternalDataMapper();
sc::ExternalDataSource aSource(aURL, aProvider);
aSource.setID(aID);
aSource.setDBData(pDBData);
rDataMapper.insertDataSource(aSource);
}
}
}
ScXMLMappingContext::~ScXMLMappingContext()
{
}
uno::Reference< xml::sax::XFastContextHandler > SAL_CALL ScXMLMappingContext::createFastChildContext(
sal_Int32 /*nElement*/, const uno::Reference< xml::sax::XFastAttributeList >& /*xAttrList*/ )
{
SvXMLImportContext *pContext = nullptr;
if( !pContext )
pContext = new SvXMLImportContext( GetImport() );
return pContext;
}
void SAL_CALL ScXMLMappingContext::endFastElement( sal_Int32 /*nElement*/ )
{
}
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/filter/xml/xmlmappingi.hxx b/sc/source/filter/xml/xmlmappingi.hxx
new file mode 100644
index 0000000..2fafd98
--- /dev/null
+++ b/sc/source/filter/xml/xmlmappingi.hxx
@@ -0,0 +1,52 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#ifndef INCLUDED_SC_SOURCE_FILTER_XML_XMLMAPPINGI_HXX
#define INCLUDED_SC_SOURCE_FILTER_XML_XMLMAPPINGI_HXX
#include <xmloff/xmlictxt.hxx>
#include <xmloff/xmlimp.hxx>
#include "xmlimprt.hxx"
#include "importcontext.hxx"
class ScXMLMappingsContext : public ScXMLImportContext
{
public:
ScXMLMappingsContext( ScXMLImport& rImport, sal_Int32 nElement,
const css::uno::Reference<css::xml::sax::XFastAttributeList>& xAttrList);
virtual ~ScXMLMappingsContext() override;
virtual css::uno::Reference< css::xml::sax::XFastContextHandler > SAL_CALL createFastChildContext(
sal_Int32 nElement,
const css::uno::Reference< css::xml::sax::XFastAttributeList >& xAttrList ) override;
};
class ScXMLMappingContext : public ScXMLImportContext
{
public:
ScXMLMappingContext( ScXMLImport& rImport, sal_Int32 nElement,
const css::uno::Reference<css::xml::sax::XFastAttributeList>& xAttrList);
virtual ~ScXMLMappingContext() override;
virtual css::uno::Reference< css::xml::sax::XFastContextHandler > SAL_CALL createFastChildContext(
sal_Int32 nElement, const css::uno::Reference< css::xml::sax::XFastAttributeList >& xAttrList ) override;
virtual void SAL_CALL endFastElement( sal_Int32 nElement ) override;
};
#endif
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/sc/source/ui/docshell/dataprovider.cxx b/sc/source/ui/docshell/dataprovider.cxx
index 31b0732d8..0f39adc 100644
--- a/sc/source/ui/docshell/dataprovider.cxx
+++ b/sc/source/ui/docshell/dataprovider.cxx
@@ -6,6 +6,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#include <dataprovider.hxx>
#include <com/sun/star/ucb/XSimpleFileAccess3.hpp>
#include <com/sun/star/ucb/SimpleFileAccess.hpp>
@@ -56,30 +57,101 @@ std::unique_ptr<SvStream> FetchStreamFromURL(const OUString& rURL, OStringBuffer
}
ExternalDataMapper::ExternalDataMapper(ScDocShell* pDocShell, const OUString& rURL, const OUString& rName, SCTAB nTab,
SCCOL nCol1,SCROW nRow1, SCCOL nCol2, SCROW nRow2, bool bAllowResize, bool& bSuccess):
maRange (ScRange(nCol1, nRow1, nTab, nCol2, nRow2, nTab)),
mpDocShell(pDocShell),
mpDBCollection (pDocShell->GetDocument().GetDBCollection())
ExternalDataSource::ExternalDataSource(const OUString& rURL,
const OUString& rProvider):
maURL(rURL),
maProvider(rProvider),
mnUpdateFrequency(0)
{
bSuccess = true;
ScDBCollection::NamedDBs& rNamedDBS = mpDBCollection->getNamedDBs();
ScDBData* aDBData = new ScDBData (rName, nTab, nCol1, nRow1, nCol2, nRow2);
if(!rNamedDBS.insert (aDBData))
bSuccess = false;
mpDBDataManager = std::shared_ptr<ScDBDataManager>(new ScDBDataManager(aDBData, bAllowResize));
mpDBDataManager->SetDestinationRange(maRange);
}
mpDataProvider = std::unique_ptr<DataProvider> (new CSVDataProvider(mpDocShell, rURL, maRange, mpDBDataManager.get()));
void ExternalDataSource::setID(const OUString& rID)
{
maID = rID;
}
const OUString& ExternalDataSource::getURL() const
{
return maURL;
}
const OUString& ExternalDataSource::getProvider() const
{
return maProvider;
}
const OUString& ExternalDataSource::getID() const
{
return maID;
}
OUString ExternalDataSource::getDBName() const
{
if (mpDBDataManager)
{
ScDBData* pDBData = mpDBDataManager->getDBData();
if (pDBData)
return pDBData->GetName();
}
return OUString();
}
void ExternalDataSource::setDBData(ScDBData* pDBData)
{
if (!mpDBDataManager)
{
mpDBDataManager.reset(new ScDBDataManager(pDBData, false));
}
else
{
mpDBDataManager->SetDatabase(pDBData);
}
}
double ExternalDataSource::getUpdateFrequency() const
{
return mnUpdateFrequency;
}
void ExternalDataSource::refresh(ScDocument* pDoc)
{
// no DB data available
if (!mpDBDataManager)
return;
// if no data provider exists, try to create one
if (!mpDataProvider)
mpDataProvider = DataProviderFactory::getDataProvider(pDoc, maProvider, maURL, maID, mpDBDataManager.get());
// if we still have not been able to create one, we can not refresh the data
if (!mpDataProvider)
return;
mpDataProvider->Import();
}
ExternalDataMapper::ExternalDataMapper(ScDocument* /*pDoc*/)
//mpDoc(pDoc)
{
}
ExternalDataMapper::~ExternalDataMapper()
{
}
void ExternalDataMapper::StartImport()
void ExternalDataMapper::insertDataSource(const sc::ExternalDataSource& rSource)
{
mpDataProvider->StartImport();
maDataSources.push_back(rSource);
}
const std::vector<sc::ExternalDataSource>& ExternalDataMapper::getDataSources() const
{
return maDataSources;
}
std::vector<sc::ExternalDataSource>& ExternalDataMapper::getDataSources()
{
return maDataSources;
}
DataProvider::~DataProvider()
@@ -137,18 +209,14 @@ public:
}
};
CSVFetchThread::CSVFetchThread(ScDocument& rDoc, ScDBDataManager* pBDDataManager, const OUString& mrURL, size_t nColCount):
Thread("ReaderThread"),
mpStream(nullptr),
CSVFetchThread::CSVFetchThread(ScDocument& rDoc, const OUString& mrURL):
Thread("CSV Fetch Thread"),
mrDocument(rDoc),
maURL (mrURL),
mnColCount(nColCount),
mpDBDataManager(pBDDataManager),
mbTerminate(false)
{
maConfig.delimiters.push_back(',');
maConfig.text_qualifier = '"';
mrDocument.InsertTab(0, "blah");
}
CSVFetchThread::~CSVFetchThread()
@@ -175,17 +243,20 @@ void CSVFetchThread::EndThread()
void CSVFetchThread::execute()
{
OStringBuffer aBuffer(64000);
mpStream = FetchStreamFromURL(maURL, aBuffer);
if (mpStream->good())
std::unique_ptr<SvStream> pStream = FetchStreamFromURL(maURL, aBuffer);
if (pStream->good())
{
LinesType aLines(10);
SCROW nCurRow = 0;
SCCOL nCol = 0;
for (Line & rLine : aLines)
{
if (mbTerminate)
return;
rLine.maCells.clear();
mpStream->ReadLine(rLine.maLine);
CSVHandler aHdl(rLine, mnColCount);
pStream->ReadLine(rLine.maLine);
CSVHandler aHdl(rLine, MAXCOL);
orcus::csv_parser<CSVHandler> parser(rLine.maLine.getStr(), rLine.maLine.getLength(), aHdl, maConfig);
parser.parse();
@@ -210,8 +281,6 @@ void CSVFetchThread::execute()
}
nCurRow++;
}
mpDBDataManager->SetSourceRange(nCol, nCurRow);
}
}
@@ -243,49 +312,40 @@ void CSVFetchThread::ResumeFetchStream()
maCondReadStream.set();
}
CSVDataProvider::CSVDataProvider(ScDocShell* pDocShell, const OUString& rURL, ScRange& rRange, ScDBDataManager* pBDDataManager):
CSVDataProvider::CSVDataProvider(ScDocument* pDoc, const OUString& rURL, ScDBDataManager* pBDDataManager):
maURL(rURL),
mrRange(rRange),
mpDocShell(pDocShell),
mpDocument(&pDocShell->GetDocument()),
mpDocument(pDoc),
mpDBDataManager(pBDDataManager),
mpLines(nullptr),
mnLineCount(0),
mbImportUnderway(false)
mnLineCount(0)
{
mpDBDataManager->SetDestinationRange(rRange);
}
CSVDataProvider::~CSVDataProvider()
{
}
void CSVDataProvider::StartImport()
void CSVDataProvider::Import()
{
if (mbImportUnderway)
return;
if (!mxCSVFetchThread.is())
ScDocument aDoc(SCDOCMODE_CLIP);
aDoc.ResetClip(mpDocument, (SCTAB)0);
mxCSVFetchThread = new CSVFetchThread(aDoc, maURL);
mxCSVFetchThread->launch();
if (mxCSVFetchThread.is())
{
ScDocument aDoc;
mxCSVFetchThread = new CSVFetchThread(aDoc, mpDBDataManager, maURL, mrRange.aEnd.Col() - mrRange.aStart.Col() + 1);
mxCSVFetchThread->launch();
if (mxCSVFetchThread.is())
{
mxCSVFetchThread->EndThread();
mxCSVFetchThread->join();
}
WriteToDoc(aDoc);
mxCSVFetchThread->join();
}
WriteToDoc(aDoc, mpDBDataManager->getDBData());
Refresh();
}
void CSVDataProvider::Refresh()
{
mpDocShell->DoHardRecalc();
mpDocShell->SetDocumentModified();
ScDocShell* pDocShell = static_cast<ScDocShell*>(mpDocument->GetDocumentShell());
pDocShell->DoHardRecalc();
pDocShell->SetDocumentModified();
}
Line CSVDataProvider::GetLine()
@@ -310,35 +370,42 @@ Line CSVDataProvider::GetLine()
return mpLines->at(mnLineCount++);
}
void CSVDataProvider::WriteToDoc(ScDocument& rDoc)
// TODO: why don't we use existing copy functionality
void CSVDataProvider::WriteToDoc(ScDocument& rDoc, ScDBData* pDBData)
{
if (mpDBDataManager->Resize())
mrRange = mpDBDataManager->GetDestinationRange();
bool bShrunk = false;
SCCOL nStartCol = 0;
SCROW nStartRow = 0;
SCCOL nEndCol = MAXCOL;
SCROW nEndRow = MAXROW;
rDoc.ShrinkToUsedDataArea(bShrunk, 0, nStartCol, nStartRow, nEndCol, nEndRow, false, true, true);
ScRange aDestRange;
pDBData->GetArea(aDestRange);
double* pfValue;
for (int nRow = mrRange.aStart.Row(); nRow < mrRange.aEnd.Row(); ++nRow)
for (int nRow = nStartRow; nRow < nEndRow; ++nRow)
{
for (int nCol = mrRange.aStart.Col(); nCol < mrRange.aEnd.Col(); ++nCol)
for (int nCol = nStartCol; nCol < nEndCol; ++nCol)
{
ScAddress aAddr = ScAddress(nCol, nRow, mrRange.aStart.Tab());
ScAddress aAddr = ScAddress(nCol, nRow, 0);
pfValue = rDoc.GetValueCell(aAddr);
if (pfValue == nullptr)
{
OUString aString = rDoc.GetString(nCol, nRow, mrRange.aStart.Tab());
mpDocument->SetString(nCol, nRow, mrRange.aStart.Tab(), aString);
OUString aString = rDoc.GetString(nCol, nRow, 0);
mpDocument->SetString(aDestRange.aStart.Col() + nCol, aDestRange.aStart.Row() + nRow, aDestRange.aStart.Tab(), aString);
}
else
{
mpDocument->SetValue(nCol, nRow, mrRange.aStart.Tab(), *pfValue);
mpDocument->SetValue(aDestRange.aStart.Col() + nCol, aDestRange.aStart.Row() + nRow, aDestRange.aStart.Tab(), *pfValue);
}
}
}
}
ScDBDataManager::ScDBDataManager(ScDBData* pDBData, bool bAllowResize = false):
mpDBData(pDBData),
mbAllowResize(bAllowResize)
ScDBDataManager::ScDBDataManager(ScDBData* pDBData, bool /*bAllowResize*/):
mpDBData(pDBData)
//mbAllowResize(bAllowResize)
{
}
@@ -351,60 +418,31 @@ void ScDBDataManager::SetDatabase(ScDBData* pDbData)
mpDBData = pDbData;
}
bool ScDBDataManager::IsResizeAllowed()
ScDBData* ScDBDataManager::getDBData()
{
return mbAllowResize;
return mpDBData;
}
bool ScDBDataManager::RequiresResize(SCROW& RowDifference, SCCOL& ColDifference)
bool DataProviderFactory::isInternalDataProvider(const OUString& rProvider)
{
SCROW nTotalSourceRows = maSourceRange.aStart.Row() - maSourceRange.aEnd.Row();
SCCOL nTotalSourceCols = maSourceRange.aStart.Col() - maSourceRange.aEnd.Col();
SCROW nTotalDestinationRows = maDestinationRange.aStart.Row() - maDestinationRange.aEnd.Row();
SCCOL nTotalDestinationCols = maDestinationRange.aStart.Col() - maDestinationRange.aEnd.Col();
RowDifference = nTotalSourceRows - nTotalDestinationRows;
ColDifference = nTotalSourceCols - nTotalDestinationCols;
if (nTotalSourceRows != nTotalDestinationRows || nTotalSourceCols != nTotalDestinationCols)
return true;
return false;
return rProvider.startsWith("org.libreoffice.calc");
}
bool ScDBDataManager::Resize()
std::shared_ptr<DataProvider> DataProviderFactory::getDataProvider(ScDocument* pDoc, const OUString& rProvider, const OUString& rURL, const OUString& /*rID*/, ScDBDataManager* pManager)
{
SCROW RowDifference =0;
SCCOL ColDifference = 0;
if (IsResizeAllowed() && RequiresResize(RowDifference, ColDifference))
bool bInternal = DataProviderFactory::isInternalDataProvider(rProvider);
if (bInternal)
{
maDestinationRange.aEnd = ScAddress(maDestinationRange.aEnd.Row() + RowDifference, maDestinationRange.aEnd.Col() + ColDifference, maDestinationRange.aEnd.Tab());
return true;
if (rProvider == "org.libreoffice.calc.csv")
return std::shared_ptr<DataProvider>(new CSVDataProvider(pDoc, rURL, pManager));
}
return false;
}
else
{
SAL_WARN("sc", "no external data provider supported yet");
return std::shared_ptr<DataProvider>();
}
void ScDBDataManager::SetSourceRange(SCCOL nCol, SCROW nRow)
{
maSourceRange = ScRange(0, 0, 0, nCol, nRow, 0);
}
void ScDBDataManager::SetDestinationRange(ScRange& aRange)
{
maDestinationRange = aRange;
}
ScRange& ScDBDataManager::GetSourceRange()
{
return maSourceRange;
}
ScRange& ScDBDataManager::GetDestinationRange()
{
return maDestinationRange;
return std::shared_ptr<DataProvider>();
}
}
diff --git a/sc/source/ui/inc/dataprovider.hxx b/sc/source/ui/inc/dataprovider.hxx
index 8f943ec..7f50b2d 100644
--- a/sc/source/ui/inc/dataprovider.hxx
+++ b/sc/source/ui/inc/dataprovider.hxx
@@ -23,6 +23,7 @@
#include "docsh.hxx"
#include "scdllapi.h"
#include "datamapper.hxx"
#include <queue>
@@ -39,24 +40,6 @@ class DataProvider;
class CSVDataProvider;
class ScDBDataManager;
class SC_DLLPUBLIC ExternalDataMapper
{
ScRange maRange;
ScDocShell* mpDocShell;
std::unique_ptr<DataProvider> mpDataProvider;
ScDocument maDocument;
ScDBCollection* mpDBCollection;
std::shared_ptr<ScDBDataManager> mpDBDataManager;
public:
ExternalDataMapper(ScDocShell* pDocShell, const OUString& rUrl, const OUString& rName,
SCTAB nTab, SCCOL nCol1,SCROW nRow1, SCCOL nCOL2, SCROW nRow2, bool bAllowResize, bool& bSuccess);
~ExternalDataMapper();
void StartImport();
};
struct Cell
{
struct Str
@@ -87,11 +70,8 @@ typedef std::vector<Line> LinesType;
class CSVFetchThread : public salhelper::Thread
{
std::unique_ptr<SvStream> mpStream;
ScDocument& mrDocument;
OUString maURL;
size_t mnColCount;
ScDBDataManager* mpDBDataManager;
bool mbTerminate;
osl::Mutex maMtxTerminate;
@@ -104,10 +84,9 @@ class CSVFetchThread : public salhelper::Thread
orcus::csv::parser_config maConfig;
virtual void execute() override;
public:
CSVFetchThread(ScDocument& rDoc, ScDBDataManager*, const OUString&, size_t);
CSVFetchThread(ScDocument& rDoc, const OUString&);
virtual ~CSVFetchThread() override;
void RequestTerminate();
@@ -120,72 +99,81 @@ public:
void WaitForNewLines();
LinesType* GetNewLines();
void ResumeFetchStream();
virtual void execute() override;
};
/**
* Abstract class for all data provider.
*
*/
class DataProvider
{
public:
virtual ~DataProvider() = 0;
virtual void StartImport() = 0;
virtual void Refresh() = 0;
virtual void WriteToDoc(ScDocument&) = 0;
virtual void Import() = 0;
virtual void WriteToDoc(ScDocument& rDoc, ScDBData* pDBData) = 0;
virtual ScRange GetRange() const = 0;
virtual const OUString& GetURL() const = 0;
};
class CSVDataProvider : public DataProvider
{
OUString maURL;
ScRange mrRange;
rtl::Reference<CSVFetchThread> mxCSVFetchThread;
ScDocShell* mpDocShell;
ScDocument* mpDocument;
ScDBDataManager* mpDBDataManager;
LinesType* mpLines;
size_t mnLineCount;
bool mbImportUnderway;
public:
CSVDataProvider (ScDocShell* pDocShell, const OUString& rUrl, ScRange& rRange, ScDBDataManager*);
virtual ~CSVDataProvider() override;
virtual void StartImport() override;
virtual void Refresh() override;
virtual void WriteToDoc(ScDocument&) override;
void Refresh();
Line GetLine();
ScRange GetRange() const override
{
return mrRange;
}
public:
CSVDataProvider (ScDocument* pDoc, const OUString& rURL, ScDBDataManager* pDBManager);
virtual ~CSVDataProvider() override;
virtual void Import() override;
// TODO: this method should be moved to the ScDBDataManager
virtual void WriteToDoc(ScDocument& rDoc, ScDBData* pDBData) override;
const OUString& GetURL() const override { return maURL; }
};
/**
* This class handles the copying of the data from the imported
* temporary document to the actual document. Additionally, in the future
* we may decide to store data transformations in this class.
*
* In addition this class also handles how to deal with excess data by for example extending the ScDBData or by only showing the first or last entries.
*
* TODO: move the DataProvider::WriteToDoc here
*
*/
class ScDBDataManager
{
ScDBData* mpDBData;
ScRange maSourceRange;
ScRange maDestinationRange;
bool mbAllowResize;
public:
ScDBDataManager(ScDBData*, bool);
ScDBDataManager(ScDBData* pDBData, bool bAllowResize);
~ScDBDataManager();
bool IsResizeAllowed();
bool Resize();
bool RequiresResize(SCROW&, SCCOL&);
void SetDatabase(ScDBData* pDBData);
void SetDatabase(ScDBData*);
void SetSourceRange(SCCOL, SCROW);
void SetDestinationRange(ScRange&);
ScDBData* getDBData();
};
ScRange& GetDestinationRange();
ScRange& GetSourceRange();
class DataProviderFactory
{
private:
static bool isInternalDataProvider(const OUString& rProvider);
public:
static std::shared_ptr<DataProvider> getDataProvider(ScDocument* pDoc, const OUString& rProvider, const OUString& rURL, const OUString& rID, ScDBDataManager* pManager);
};
}
diff --git a/sc/source/ui/view/gridwin.cxx b/sc/source/ui/view/gridwin.cxx
index b37b6d1..77865b5 100644
--- a/sc/source/ui/view/gridwin.cxx
+++ b/sc/source/ui/view/gridwin.cxx
@@ -133,6 +133,7 @@
#include "uiobject.hxx"
#include "scabstdlg.hxx"
#include "undoblk.hxx"
#include "datamapper.hxx"
#include <svx/sdrpagewindow.hxx>
#include <svx/sdr/overlay/overlaymanager.hxx>
@@ -3087,6 +3088,16 @@ void ScGridWindow::KeyInput(const KeyEvent& rKEvt)
{
dumpColumnCellStorage();
}
else if (rKeyCode.GetCode() == KEY_F7)
{
ScDocument* pDoc = pViewData->GetDocument();
auto& rMapper = pDoc->GetExternalDataMapper();
for (auto& itr : rMapper.getDataSources())
{
itr.refresh(pDoc);
}
return;
}
}
#endif
diff --git a/xmloff/source/core/xmltoken.cxx b/xmloff/source/core/xmltoken.cxx
index 65829c2..521e304 100644
--- a/xmloff/source/core/xmltoken.cxx
+++ b/xmloff/source/core/xmltoken.cxx
@@ -3252,6 +3252,10 @@ namespace xmloff { namespace token {
TOKEN( "margin", XML_MARGIN),
TOKEN( "propertry-mapping", XML_PROPERTY_MAPPING),
TOKEN( "provider", XML_PROVIDER),
TOKEN( "data-mappings", XML_DATA_MAPPINGS),
TOKEN( "data-mapping", XML_DATA_MAPPING),
TOKEN( "frequency", XML_DATA_FREQUENCY),
// regina, ODF1.2 additional symbols in charts
TOKEN( "star", XML_STAR ),
diff --git a/xmloff/source/token/tokens.txt b/xmloff/source/token/tokens.txt
index e874670..fc54aaf 100644
--- a/xmloff/source/token/tokens.txt
+++ b/xmloff/source/token/tokens.txt
@@ -3033,6 +3033,10 @@ min-value
max-value
margin
propertry-mapping
provider
data-mappings
data-mapping
frequency
star
asterisk
horizontal-bar
