tdf#106899 - Import concordance file using appropriate charset
At the beginning of the import process for the various index entries,
try to determine the correct character set for the tox concordance file.
Change-Id: I3f48325a80ed08c2c06c295a24b2fc29ce1adf99
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/128194
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
Reviewed-by: Andreas Heinisch <andreas.heinisch@yahoo.de>
Tested-by: Jenkins
Signed-off-by: Xisco Fauli <xiscofauli@libreoffice.org>
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/129206
diff --git a/sw/inc/iodetect.hxx b/sw/inc/iodetect.hxx
index 9069b01..ce24e08 100644
--- a/sw/inc/iodetect.hxx
+++ b/sw/inc/iodetect.hxx
@@ -28,6 +28,8 @@
#include <tools/solar.h>
#include "swdllapi.h"
#define DETECT_ENCODING_BUFFER_SIZE 4096
#define FILTER_RTF "RTF" ///< RTF filter
#define sRtfWH "WH_RTF"
#define FILTER_TEXT u"TEXT" ///< text filter with default codeset
@@ -105,6 +107,8 @@ public:
static bool IsValidStgFilter( SotStorage& , const SfxFilter& );
static bool IsValidStgFilter( const css::uno::Reference < css::embed::XStorage >& rStg, const SfxFilter& rFilter);
// tdf#106899 - wrapper around IsDetectableText to retrieve the text encoding for a given stream
static rtl_TextEncoding GetTextEncoding(SvStream&);
static bool IsDetectableText( const char* pBuf, sal_uLong &rLen,
rtl_TextEncoding *pCharSet, bool *pSwap, LineEnd *pLineEnd, bool *pBom);
diff --git a/sw/qa/uitest/data/tdf106899.odt b/sw/qa/uitest/data/tdf106899.odt
new file mode 100644
index 0000000..3f77c82
--- /dev/null
+++ b/sw/qa/uitest/data/tdf106899.odt
Binary files differ
diff --git a/sw/qa/uitest/data/tdf106899.sdi b/sw/qa/uitest/data/tdf106899.sdi
new file mode 100644
index 0000000..6b110b2
--- /dev/null
+++ b/sw/qa/uitest/data/tdf106899.sdi
@@ -0,0 +1 @@
Nguyễn Khánh
\ No newline at end of file
diff --git a/sw/qa/uitest/writer_tests5/tdf106899.py b/sw/qa/uitest/writer_tests5/tdf106899.py
new file mode 100644
index 0000000..389552e
--- /dev/null
+++ b/sw/qa/uitest/writer_tests5/tdf106899.py
@@ -0,0 +1,44 @@
# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
#
# This file is part of the LibreOffice project.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
import org.libreoffice.unotest
from uitest.framework import UITestCase
from uitest.uihelper.common import get_url_for_data_file
import time
class tdf106899(UITestCase):
def test_tdf106899_alphabetical_index_utf8(self):
# Copy concordance file containing an utf8 index entry
org.libreoffice.unotest.makeCopyFromTDOC("tdf106899.sdi")
with self.ui_test.load_file(get_url_for_data_file("tdf106899.odt")) as document:
xWriterDoc = self.xUITest.getTopFocusWindow()
# Update the alphabetical index and check if it contains the utf8 index entry
xDocumentIndexes = document.DocumentIndexes
self.assertEqual(xDocumentIndexes.getCount(), 1)
self.assertEqual(xDocumentIndexes.hasByName("Alphabetical Index1"), True)
xDocumentIndex = xDocumentIndexes.getByName("Alphabetical Index1")
xIndexAnchor = xDocumentIndex.getAnchor()
self.assertEqual("Nguyễn Khánh" in xIndexAnchor.getString(), False)
# TODO Bug Report - Refresh of the index does only work using .uno:UpdateAllIndexes
# It does not work with xDocumentIndex.refresh() nor with xDocumentIndex.update()
self.xUITest.executeCommand(".uno:UpdateAllIndexes")
# TODO Bug Report - Retrieving the text of the updated index only works using the cursor
# It does not work with xIndexAnchor.getString()
xCursor = document.getText().createTextCursor()
xCursor.gotoRange(xDocumentIndex.getAnchor().getEnd(), False)
xCursor.gotoStartOfParagraph(True)
# Without the fix in place the index does not contain the ut8 index entry
self.assertEqual("Nguyễn Khánh" in xCursor.getString(), True)
# vim: set shiftwidth=4 softtabstop=4 expandtab:
diff --git a/sw/source/core/edit/edtox.cxx b/sw/source/core/edit/edtox.cxx
index c10f366..df58e49 100644
--- a/sw/source/core/edit/edtox.cxx
+++ b/sw/source/core/edit/edtox.cxx
@@ -42,6 +42,7 @@
#include <docary.hxx>
#include <mdiexp.hxx>
#include <strings.hrc>
#include <iodetect.hxx>
using namespace ::com::sun::star;
using namespace ::com::sun::star::i18n;
@@ -295,7 +296,10 @@ void SwEditShell::ApplyAutoMark()
SfxMedium aMedium( sAutoMarkURL, StreamMode::STD_READ );
SvStream& rStrm = *aMedium.GetInStream();
Push();
rtl_TextEncoding eChrSet = ::osl_getThreadTextEncoding();
// tdf#106899 - import tox concordance file using the appropriate character set
rtl_TextEncoding eChrSet = SwIoSystem::GetTextEncoding(rStrm);
if (eChrSet == RTL_TEXTENCODING_DONTKNOW)
eChrSet = ::osl_getThreadTextEncoding();
// SearchOptions to be used in loop below
sal_Int32 const nLEV_Other = 2; // -> changedChars;
diff --git a/sw/source/filter/basflt/iodetect.cxx b/sw/source/filter/basflt/iodetect.cxx
index e4d2143..bc91a84 100644
--- a/sw/source/filter/basflt/iodetect.cxx
+++ b/sw/source/filter/basflt/iodetect.cxx
@@ -20,6 +20,7 @@
#include <iodetect.hxx>
#include <memory>
#include <osl/endian.h>
#include <osl/thread.h>
#include <sot/storage.hxx>
#include <tools/urlobj.hxx>
#include <unotools/moduleoptions.hxx>
@@ -238,6 +239,22 @@ std::shared_ptr<const SfxFilter> SwIoSystem::GetFileFilter(const OUString& rFile
return SwIoSystem::GetFilterOfFormat(FILTER_TEXT);
}
rtl_TextEncoding SwIoSystem::GetTextEncoding(SvStream& rStrm)
{
sal_Size nLen, nOrig;
char aBuf[DETECT_ENCODING_BUFFER_SIZE];
nOrig = nLen = rStrm.ReadBytes(aBuf, DETECT_ENCODING_BUFFER_SIZE);
rtl_TextEncoding eCharSet;
const bool bRet = SwIoSystem::IsDetectableText(aBuf, nLen, &eCharSet, nullptr, nullptr, nullptr);
if (bRet && eCharSet != RTL_TEXTENCODING_DONTKNOW)
rStrm.SeekRel(-(tools::Long(nLen)));
else
rStrm.SeekRel(-(tools::Long(nOrig)));
return eCharSet;
}
bool SwIoSystem::IsDetectableText(const char* pBuf, sal_uLong &rLen,
rtl_TextEncoding *pCharSet, bool *pSwap, LineEnd *pLineEnd, bool *pBom)
{
