tdf#125662: disable parallel-zip if the memory...
required to carry it out will cause a breach of addressable memory
limit. This can happen in 32bit OSes, where the physical memory limit is
more than the process addressable memory limit. For example in Win32,
a process can address only upto 2GB.
More specifically, avoid using ZipOutputEntryParallel
which rougly needs twice the memory compared to its counterparts.
Change-Id: I4d1abbf75e928188bcf806fcf1e5872b1e51b502
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/84394
Tested-by: Jenkins
Reviewed-by: Dennis Francis <dennis.francis@collabora.com>
diff --git a/comphelper/Library_comphelper.mk b/comphelper/Library_comphelper.mk
index e0bf1a8..b5de3f5 100644
--- a/comphelper/Library_comphelper.mk
+++ b/comphelper/Library_comphelper.mk
@@ -117,6 +117,7 @@ $(eval $(call gb_Library_add_exception_objects,comphelper,\
comphelper/source/misc/listenernotification \
comphelper/source/misc/logging \
comphelper/source/misc/lok \
comphelper/source/misc/meminfo \
comphelper/source/misc/mimeconfighelper \
comphelper/source/misc/namedvaluecollection \
comphelper/source/misc/numberedcollection \
diff --git a/comphelper/source/misc/meminfo.cxx b/comphelper/source/misc/meminfo.cxx
new file mode 100644
index 0000000..d6508e9
--- /dev/null
+++ b/comphelper/source/misc/meminfo.cxx
@@ -0,0 +1,139 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*
* This file incorporates work covered by the following license notice:
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed
* with this work for additional information regarding copyright
* ownership. The ASF licenses this file to you under the Apache
* License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of
* the License at http://www.apache.org/licenses/LICENSE-2.0 .
*/
#include <sal/types.h>
#include <comphelper/meminfo.hxx>
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include "psapi.h"
#elif defined(UNX)
#if defined(MACOSX)
#include <sys/types.h>
#include <mach/mach.h>
#include <mach/host_info.h>
#include <mach/mach_host.h>
#else // Linux
#include <fstream>
#include <unistd.h>
#endif
#endif
namespace comphelper
{
// Returns the virtual memory used currently by the process.
static sal_Int64 getMemUsedBySelf()
{
#if defined(UNX)
#if defined(MACOSX)
vm_size_t pageSize;
struct task_basic_info tInfo;
mach_msg_type_number_t tInfoCount = TASK_BASIC_INFO_COUNT;
if (host_page_size(mach_host_self(), &pageSize) != KERN_SUCCESS)
return -1;
if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&tInfo, &tInfoCount)
!= KERN_SUCCESS)
return -1;
return static_cast<sal_Int64>(tInfo.virtual_size) * pageSize;
#elif defined(LINUX)
sal_Int64 pageSize = sysconf(_SC_PAGESIZE);
if (pageSize <= 0)
return -1;
std::ifstream ifs("/proc/self/statm", std::ifstream::in);
if (!ifs.is_open())
return -1;
sal_Int64 vmSize;
if (ifs >> vmSize)
return vmSize * pageSize;
return -1;
#else
return -1;
#endif
#elif defined(_WIN32)
PROCESS_MEMORY_COUNTERS_EX pmc;
GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&pmc),
sizeof(pmc));
return static_cast<sal_Int64>(pmc.PrivateUsage);
#else
return -1;
#endif
}
static sal_Int64 getMaxAddressableMemLimit()
{
#if defined(_WIN64)
return 8796093022208ll; // 8 TB
#elif defined(_WIN32)
return 2147483648ll; // 2 GB
#elif defined(__x86_64__)
// TODO: check for artificial limits imposed by 'ulimit -Sv' too for UNX ?
return 140737488355328ll; // 128 TB
#elif defined(__i386__)
return 3221225472ll; // 3 GB
#else
return 2147483648ll; // 2 GB
#endif
}
// canAlloc() checks whether allocSize bytes can be allocated without exceeding
// addressable memory limit. This is useful in case of 32-bit systems where
// process addressable memory limit is less than physical memory limit.
bool canAlloc(sal_Int64 allocSize)
{
if (allocSize <= 0)
return true;
sal_Int64 vmSize = getMemUsedBySelf();
if (vmSize < 0)
return true;
sal_Int64 maxSize = getMaxAddressableMemLimit();
if (vmSize + allocSize <= maxSize)
return true;
return false;
}
} // namespace comphelper
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/include/comphelper/meminfo.hxx b/include/comphelper/meminfo.hxx
new file mode 100644
index 0000000..795d7a4
--- /dev/null
+++ b/include/comphelper/meminfo.hxx
@@ -0,0 +1,37 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*
* This file incorporates work covered by the following license notice:
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed
* with this work for additional information regarding copyright
* ownership. The ASF licenses this file to you under the Apache
* License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of
* the License at http://www.apache.org/licenses/LICENSE-2.0 .
*/
#ifndef INCLUDED_COMPHELPER_MEMINFO_HXX
#define INCLUDED_COMPHELPER_MEMINFO_HXX
#include <comphelper/comphelperdllapi.h>
namespace comphelper
{
/**
* canAlloc() checks whether allocSize bytes can be allocated without exceeding
* addressable memory limit. This is useful in case of 32-bit systems where
* process addressable memory limit is less than physical memory limit.
*/
COMPHELPER_DLLPUBLIC bool canAlloc(sal_Int64 allocSize);
}
#endif // INCLUDED_COMPHELPER_MEMINFO_HXX
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */
diff --git a/package/source/zippackage/ZipPackageStream.cxx b/package/source/zippackage/ZipPackageStream.cxx
index 745157c..d8f9c59 100644
--- a/package/source/zippackage/ZipPackageStream.cxx
+++ b/package/source/zippackage/ZipPackageStream.cxx
@@ -48,6 +48,7 @@
#include <comphelper/seekableinput.hxx>
#include <comphelper/servicehelper.hxx>
#include <comphelper/storagehelper.hxx>
#include <comphelper/meminfo.hxx>
#include <cppuhelper/exc_hlp.hxx>
#include <cppuhelper/supportsservice.hxx>
#include <cppuhelper/typeprovider.hxx>
@@ -768,7 +769,9 @@ bool ZipPackageStream::saveChild(
// the full size, but it appears that at this point it usually is.
sal_Int64 estimatedSize = xSeek.is() ? xSeek->getLength() : xStream->available();
if (estimatedSize > 1000000)
// ZipOutputEntryParallel/ThreadedDeflater needs to allocate ~twice the estimatedSize.
// Check if we can allocate that much without breaching addressable memory limits.
if (estimatedSize > 1000000 && comphelper::canAlloc(2*estimatedSize))
{
// Use ThreadDeflater which will split the stream into blocks and compress
// them in threads, but not in background (i.e. writeStream() will block).
