Fix `make create-update-info` (for Windows, at least)
* The shell scripts that had once been copied from Mozilla to bin/update/*.sh
are now included in
onlineupdate-c003be8b9727672e7d30972983b375f4c200233f-2.tar.xz (which is still
generated from the same
<https://github.com/mozilla/gecko-dev/commit/c003be8b9727672e7d30972983b375f4c200233f>
that was used for the original tarball in
3a445cb49795fabe0d8caaf12bfc38eb9e12d5fc "Turn onlineupdate into
external/onlineupdate").
* The additional modifications in external/onlineupdate/lo.patch are:
** Allowing to pass the list of files into the mar tool via -f instead of on the
command line, to avoid "command line too long" errors on Windows, inspired by
the modifications once made directly in our old downstream sources with
4165dd4e465a86ba6abe9afb3abfda5ef72493ea "add a way to create mar file from
file", 8e4d49340bd235a7db8fde1d24dd1d63ddc4d571 "use the new file based
approach for the mar creation", and fb13ed6955cd66017e5348b915af371a184ea633
"add the manifest file to the mar file". (To keep things simple for now, it
still uses a hard-coded maximum of 10000 lines in the file, marked with a
TODO.)
** Not failing when "precomplete file is missing!" (There is a comment
// Applications aren't required to have a precomplete manifest. The mar
// generation scripts enforce the presence of a precomplete manifest.
in
workdir/UnpackedTarball/onlineupdate/onlineupdate/source/update/updater/updater.cpp
and it appears to be OK that we don't have such a precomplete manifest file
and just skip that test.)
* In the Makefile.gbuild create-update-info, the create_full_mar.py script needs
to be called with a Unix pathname on Windows, or else the
#!/usr/bin/env python3
shebang in that script would get confused.
* The related Makefile.gbuild targets upload-update-info and create-partial-info
have not been addressed yet.
Change-Id: Iab4e083ddbe99e07d846e202f20c6031e2983e1b
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161656
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <stephan.bergmann@allotropia.de>
(cherry picked from commit c9973227dae088a5fb04cee12b2c99e0f62d6e28)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161649
diff --git a/Makefile.gbuild b/Makefile.gbuild
index 2dfbce1..d86a0cd 100644
--- a/Makefile.gbuild
+++ b/Makefile.gbuild
@@ -41,7 +41,7 @@ create-update-info:
rm -rf $(MAR_DIR) || true
rm -rf $(UPDATE_DIR) || true
mkdir -p $(MAR_DIR)/language
MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)"
MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_full_mar.py),$(SRCDIR)/bin/update/create_full_mar.py) "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)'
$(if $(filter WNT,$(OS)),, \
MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_full_mar_for_languages.py "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" \
)
diff --git a/bin/update/common.sh b/bin/update/common.sh
deleted file mode 100644
index dcdbea8..0000000
--- a/bin/update/common.sh
+++ /dev/null
@@ -1,222 +0,0 @@
#!/usr/bin/env bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Code shared by update packaging scripts.
# Author: Darin Fisher
#
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
MAR=${MAR:-mar}
BZIP2=${BZIP2:-bzip2}
MBSDIFF=${MBSDIFF:-mbsdiff}
# -----------------------------------------------------------------------------
# Helper routines
notice() {
echo "$*" 1>&2
}
get_file_size() {
info=($(ls -ln "$1"))
echo ${info[4]}
}
check_externals() {
# check whether we can call the mar executable
"$MAR" --version > /dev/null 2>&1
if [ $? != 0 ]; then
notice "Could not find a valid mar executable in the path or in the MAR environment variable"
exit 1
fi
# check whether we can access the bzip2 executable
"$BZIP2" --help > /dev/null 2>&1
if [ $? != 0 ]; then
notice "Could not find a valid bzip2 executable in the PATH or in the BZIP2 environment variable"
exit 1
fi
}
copy_perm() {
reference="$1"
target="$2"
if [ -x "$reference" ]; then
chmod 0755 "$target"
else
chmod 0644 "$target"
fi
}
make_add_instruction() {
f="$1"
filev2="$2"
# The third param will be an empty string when a file add instruction is only
# needed in the version 2 manifest. This only happens when the file has an
# add-if-not instruction in the version 3 manifest. This is due to the
# precomplete file prior to the version 3 manifest having a remove instruction
# for this file so the file is removed before applying a complete update.
filev3="$3"
# Used to log to the console
if [ $4 ]; then
forced=" (forced)"
else
forced=
fi
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if \"$testdir\" \"$f\""
echo "add-if \"$testdir\" \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add-if \"$testdir\" \"$f\"" >> $filev3
fi
else
notice " add \"$f\"$forced"
echo "add \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add \"$f\"" >> $filev3
fi
fi
}
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
`basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
make_add_if_not_instruction() {
f="$1"
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
echo "add-if-not \"$f\" \"$f\"" >> $filev3
}
make_patch_instruction() {
f="$1"
filev2="$2"
filev3="$3"
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
else
notice " patch \"$f.patch\" \"$f\""
echo "patch \"$f.patch\" \"$f\"" >> $filev2
echo "patch \"$f.patch\" \"$f\"" >> $filev3
fi
}
append_remove_instructions() {
dir="$1"
filev2="$2"
filev3="$3"
if [ -f "$dir/removed-files" ]; then
listfile="$dir/removed-files"
elif [ -f "$dir/Contents/Resources/removed-files" ]; then
listfile="$dir/Contents/Resources/removed-files"
fi
if [ -n "$listfile" ]; then
# Map spaces to pipes so that we correctly handle filenames with spaces.
files=($(cat "$listfile" | tr " " "|" | sort -r))
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
# Map pipes back to whitespace and remove carriage returns
f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
# Trim whitespace
f=$(echo $f)
# Exclude blank lines.
if [ -n "$f" ]; then
# Exclude comments
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
notice " rmdir \"$f\""
echo "rmdir \"$f\"" >> $filev2
echo "rmdir \"$f\"" >> $filev3
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
# Remove the *
f=$(echo "$f" | sed -e 's:\*$::')
notice " rmrfdir \"$f\""
echo "rmrfdir \"$f\"" >> $filev2
echo "rmrfdir \"$f\"" >> $filev3
else
notice " remove \"$f\""
echo "remove \"$f\"" >> $filev2
echo "remove \"$f\"" >> $filev3
fi
fi
fi
done
fi
}
# List all files in the current directory, stripping leading "./"
# Pass a variable name and it will be filled as an array.
list_files() {
count=0
find . -type f \
! -name "update.manifest" \
! -name "updatev2.manifest" \
! -name "updatev3.manifest" \
! -name "temp-dirlist" \
! -name "temp-filelist" \
| sed 's/\.\/\(.*\)/\1/' \
| sort -r > "temp-filelist"
while read file; do
eval "${1}[$count]=\"$file\""
(( count++ ))
done < "temp-filelist"
rm "temp-filelist"
}
# List all directories in the current directory, stripping leading "./"
list_dirs() {
count=0
find . -type d \
! -name "." \
! -name ".." \
| sed 's/\.\/\(.*\)/\1/' \
| sort -r > "temp-dirlist"
while read dir; do
eval "${1}[$count]=\"$dir\""
(( count++ ))
done < "temp-dirlist"
rm "temp-dirlist"
}
diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py
index c0d31592..65e25e2 100755
--- a/bin/update/create_full_mar.py
+++ b/bin/update/create_full_mar.py
@@ -10,8 +10,6 @@ from tools import uncompress_file_to_dir, get_file_info, make_complete_mar_name
from signing import sign_mar_file
from path import UpdaterPath, convert_to_unix, convert_to_native
current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__)))
def main():
parser = argparse.ArgumentParser()
@@ -21,6 +19,7 @@ def main():
parser.add_argument('certificate_path')
parser.add_argument('certificate_name')
parser.add_argument('base_url')
parser.add_argument('version')
args = parser.parse_args()
certificate_path = args.certificate_path
@@ -29,6 +28,7 @@ def main():
filename_prefix = args.filename_prefix
workdir = args.workdir
product_name = args.product_name
version = args.version
update_path = UpdaterPath(workdir)
update_path.ensure_dir_exist()
@@ -42,7 +42,10 @@ def main():
uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir)
mar_file = make_complete_mar_name(target_dir, filename_prefix)
path = os.path.join(current_dir_path, 'make_full_update.sh')
path = os.path.join(
workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh')
os.putenv('MOZ_PRODUCT_VERSION', version)
os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
subprocess.call([path, convert_to_native(mar_file), convert_to_native(uncompress_dir)])
sign_mar_file(target_dir, certificate_path, certificate_name, mar_file, filename_prefix)
diff --git a/bin/update/make_full_update.sh b/bin/update/make_full_update.sh
deleted file mode 100755
index 4140eca..0000000
--- a/bin/update/make_full_update.sh
+++ /dev/null
@@ -1,122 +0,0 @@
#!/usr/bin/env bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This tool generates full update packages for the update system.
# Author: Darin Fisher
#
. $(dirname "$0")/common.sh
# -----------------------------------------------------------------------------
print_usage() {
notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY"
}
if [ $# = 0 ]; then
print_usage
exit 1
fi
if [ $1 = -h ]; then
print_usage
notice ""
notice "The contents of DIRECTORY will be stored in ARCHIVE."
notice ""
notice "Options:"
notice " -h show this help text"
notice ""
exit 1
fi
check_externals
# -----------------------------------------------------------------------------
archive="$1"
targetdir="$2"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
# Remove the /
targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
fi
workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
targetfiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
echo "updatev2.manifest" >> $workdir/files.txt
echo "updatev3.manifest" >> $workdir/files.txt
# Generate a list of all files in the target directory.
pushd "$targetdir"
if test $? -ne 0 ; then
exit 1
fi
# if [ ! -f "precomplete" ]; then
# if [ ! -f "Contents/Resources/precomplete" ]; then
# notice "precomplete file is missing!"
# exit 1
# fi
# fi
list_files files
popd
# Add the type of update to the beginning of the update manifests.
> $updatemanifestv2
> $updatemanifestv3
notice ""
notice "Adding type instruction to update manifests"
notice " type complete"
echo "type \"complete\"" >> $updatemanifestv2
echo "type \"complete\"" >> $updatemanifestv3
notice ""
notice "Adding file add instructions to update manifests"
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
f="${files[$i]}"
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\""
echo $f >> $workdir/files.txt
done
# Append remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
eval "$MAR -C \"$workdir\" -c output.mar -f $workdir/files.txt"
mv -f "$workdir/output.mar" "$archive"
# cleanup
rm -fr "$workdir"
notice ""
notice "Finished"
notice ""
diff --git a/bin/update/make_incremental_update.sh b/bin/update/make_incremental_update.sh
deleted file mode 100755
index 31bddab..0000000
--- a/bin/update/make_incremental_update.sh
+++ /dev/null
@@ -1,318 +0,0 @@
#!/usr/bin/env bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This tool generates incremental update packages for the update system.
# Author: Darin Fisher
#
. $(dirname "$0")/common.sh
# -----------------------------------------------------------------------------
print_usage() {
notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
notice ""
notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
notice ""
notice "Options:"
notice " -h show this help text"
notice " -f clobber this file in the installation"
notice " Must be a path to a file to clobber in the partial update."
notice ""
}
check_for_forced_update() {
force_list="$1"
forced_file_chk="$2"
local f
if [ "$forced_file_chk" = "precomplete" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "removed-files" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then
## "true" *giggle*
return 0;
fi
if [ "${forced_file_chk##*.}" = "chk" ]; then
## "true" *giggle*
return 0;
fi
for f in $force_list; do
#echo comparing $forced_file_chk to $f
if [ "$forced_file_chk" = "$f" ]; then
## "true" *giggle*
return 0;
fi
done
## 'false'... because this is bash. Oh yay!
return 1;
}
if [ $# = 0 ]; then
print_usage
exit 1
fi
requested_forced_updates='Contents/MacOS/firefox'
while getopts "hf:" flag
do
case "$flag" in
h) print_usage; exit 0
;;
f) requested_forced_updates="$requested_forced_updates $OPTARG"
;;
?) print_usage; exit 1
;;
esac
done
# -----------------------------------------------------------------------------
let arg_start=$OPTIND-1
shift $arg_start
archive="$1"
olddir="$2"
newdir="$3"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
# Remove the /
newdir=$(echo "$newdir" | sed -e 's:\/$::')
fi
workdir="$newdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
mkdir -p "$workdir"
echo "updatev2.manifest" >> $workdir/files.txt
echo "updatev3.manifest" >> $workdir/files.txt
# Generate a list of all files in the target directory.
pushd "$olddir"
if test $? -ne 0 ; then
exit 1
fi
list_files oldfiles
list_dirs olddirs
popd
pushd "$newdir"
if test $? -ne 0 ; then
exit 1
fi
# if [ ! -f "precomplete" ]; then
# if [ ! -f "Contents/Resources/precomplete" ]; then
# notice "precomplete file is missing!"
# exit 1
# fi
# fi
list_dirs newdirs
list_files newfiles
popd
# Add the type of update to the beginning of the update manifests.
notice ""
notice "Adding type instruction to update manifests"
> $updatemanifestv2
> $updatemanifestv3
notice " type partial"
echo "type \"partial\"" >> $updatemanifestv2
echo "type \"partial\"" >> $updatemanifestv3
notice ""
notice "Adding file patch and add instructions to update manifests"
num_oldfiles=${#oldfiles[*]}
remove_array=
num_removes=0
for ((i=0; $i<$num_oldfiles; i=$i+1)); do
f="${oldfiles[$i]}"
# If this file exists in the new directory as well, then check if it differs.
if [ -f "$newdir/$f" ]; then
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3"
echo $f >> $workdir/files.txt
continue 1
fi
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
echo $f >> $workdir/files.txt
continue 1
fi
if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
# Compute both the compressed binary diff and the compressed file, and
# compare the sizes. Then choose the smaller of the two to package.
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
notice "diffing \"$f\""
# MBSDIFF_HOOK represents the communication interface with funsize and,
# if enabled, caches the intermediate patches for future use and
# compute avoidance
#
# An example of MBSDIFF_HOOK env variable could look like this:
# export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
# where myscript.sh has the following usage:
# myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
# PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
#
# Note: patches are bzipped stashed in funsize to gain more speed
# if service is not enabled then default to old behavior
if [ -z "$MBSDIFF_HOOK" ]; then
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
else
# if service enabled then check patch existence for retrieval
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
fi
fi
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
patchfile="$workdir/$f.patch.bz2"
patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f")
if [ $patchsize -lt $fullsize ]; then
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
mv -f "$patchfile" "$workdir/$f.patch"
rm -f "$workdir/$f"
echo $f.patch >> $workdir/files.txt
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
rm -f "$patchfile"
echo $f >> $workdir/files.txt
fi
fi
else
# remove instructions are added after add / patch instructions for
# consistency with make_incremental_updates.py
remove_array[$num_removes]=$f
(( num_removes++ ))
fi
done
# Newly added files
notice ""
notice "Adding file add instructions to update manifests"
num_newfiles=${#newfiles[*]}
for ((i=0; $i<$num_newfiles; i=$i+1)); do
f="${newfiles[$i]}"
# If we've already tested this file, then skip it
for ((j=0; $j<$num_oldfiles; j=$j+1)); do
if [ "$f" = "${oldfiles[j]}" ]; then
continue 2
fi
done
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
echo $f >> $workdir/files.txt
done
notice ""
notice "Adding file remove instructions to update manifests"
for ((i=0; $i<$num_removes; i=$i+1)); do
f="${remove_array[$i]}"
notice " remove \"$f\""
echo "remove \"$f\"" >> $updatemanifestv2
echo "remove \"$f\"" >> $updatemanifestv3
done
# Add remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
notice ""
notice "Adding directory remove instructions for directories that no longer exist"
num_olddirs=${#olddirs[*]}
for ((i=0; $i<$num_olddirs; i=$i+1)); do
f="${olddirs[$i]}"
# If this dir doesn't exist in the new directory remove it.
if [ ! -d "$newdir/$f" ]; then
notice " rmdir $f/"
echo "rmdir \"$f/\"" >> $updatemanifestv2
echo "rmdir \"$f/\"" >> $updatemanifestv3
fi
done
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
mar_command="$MAR"
if [[ -n $PRODUCT_VERSION ]]
then
mar_command="$mar_command -V $PRODUCT_VERSION"
fi
if [[ -n $CHANNEL_ID ]]
then
mar_command="$mar_command -H $CHANNEL_ID"
fi
mar_command="$mar_command -C \"$workdir\" -c output.mar -f $workdir/files.txt"
eval "$mar_command"
mv -f "$workdir/output.mar" "$archive"
# cleanup
rm -fr "$workdir"
notice ""
notice "Finished"
notice ""
diff --git a/bin/update/tools.py b/bin/update/tools.py
index 35e635c..ab38d10 100644
--- a/bin/update/tools.py
+++ b/bin/update/tools.py
@@ -41,7 +41,7 @@ def get_hash(file_path):
def get_file_info(mar_file, url):
filesize = os.path.getsize(mar_file)
data = {'hash': get_hash(mar_file),
'hashFunction': 'sha512',
'hash_function': 'sha512',
'size': filesize,
'url': url + os.path.basename(mar_file)}
diff --git a/download.lst b/download.lst
index 6191a35..302e27c 100644
--- a/download.lst
+++ b/download.lst
@@ -498,8 +498,8 @@ OFFICEOTRON_JAR := 8249374c274932a21846fa7629c2aa9b-officeotron-0.7.4-master.jar
# three static lines
# so that git cherry-pick
# will not run into conflicts
ONLINEUPDATE_SHA256SUM := cfd00948425424d7934f5f3e26c830eda9ebbf2d18e13656bd20e4f6cd99e4ac
ONLINEUPDATE_TARBALL := onlineupdate-c003be8b9727672e7d30972983b375f4c200233f.tar.xz
ONLINEUPDATE_SHA256SUM := 37206cf981e8409d048b59ac5839621ea107ff49af72beb9d7769a2f41da8d90
ONLINEUPDATE_TARBALL := onlineupdate-c003be8b9727672e7d30972983b375f4c200233f-2.tar.xz
# three static lines
# so that git cherry-pick
# will not run into conflicts
diff --git a/external/onlineupdate/generate-sources.sh b/external/onlineupdate/generate-sources.sh
index bc39603..8f11faf 100755
--- a/external/onlineupdate/generate-sources.sh
+++ b/external/onlineupdate/generate-sources.sh
@@ -185,6 +185,9 @@ copyto toolkit/mozapps/update/common/updateutils_win.cpp toolkit/mozapps/update/
copyto toolkit/mozapps/update/common/updateutils_win.h toolkit/mozapps/update/common/updateutils_win.h
copyto toolkit/mozapps/update/updater/crctable.h toolkit/mozapps/update/updater/crctable.h
copyto toolkit/xre/nsWindowsRestart.cpp toolkit/xre/nsWindowsRestart.cpp
copyto tools/update-packaging/common.sh tools/update-packaging/common.sh
copyto tools/update-packaging/make_full_update.sh tools/update-packaging/make_full_update.sh
copyto tools/update-packaging/make_incremental_update.sh tools/update-packaging/make_incremental_update.sh
copyto xpcom/base/nsAlgorithm.h xpcom/base/nsAlgorithm.h
copyto xpcom/base/nsAutoRef.h xpcom/base/nsAutoRef.h
copyto xpcom/base/nsWindowsHelpers.h xpcom/base/nsWindowsHelpers.h
diff --git a/external/onlineupdate/lo.patch b/external/onlineupdate/lo.patch
index a8b9283..0021c3a 100644
--- a/external/onlineupdate/lo.patch
+++ b/external/onlineupdate/lo.patch
@@ -1,3 +1,96 @@
--- onlineupdate/source/libmar/tool/mar.c
+++ onlineupdate/source/libmar/tool/mar.c
@@ -14,7 +14,9 @@
# include <windows.h>
# include <direct.h>
# define chdir _chdir
+# define PATH_MAX MAX_PATH
#else
+# include <limits.h>
# include <unistd.h>
#endif
@@ -39,7 +41,7 @@
printf("Create a MAR file:\n");
printf(
" mar -H MARChannelID -V ProductVersion [-C workingDir] "
- "-c archive.mar [files...]\n");
+ "-c archive.mar [files...|-f files]\n");
printf("Extract a MAR file:\n");
printf(" mar [-C workingDir] -x archive.mar\n");
@@ -244,6 +246,8 @@
switch (argv[1][1]) {
case 'c': {
+ int numfiles;
+ char** files;
struct ProductInformationBlock infoBlock;
if (!productVersion) {
fprintf(stderr,
@@ -256,9 +260,61 @@
"<mar-channel-id>`).\n");
return -1;
}
+ if (argc == 5 && !strcmp(argv[3], "-f")) {
+ FILE* in;
+ in = fopen(argv[4], "r");
+ if (!in) {
+ fprintf(stderr,
+ "ERROR: Cannot open file `%s` for reading.\n", argv[4]);
+ return -1;
+ }
+ numfiles = 0;
+ files = malloc(sizeof(char*) * 10000); /*TODO*/
+ if (!files) {
+ fprintf(stderr,
+ "ERROR: Cannot allocate memory");
+ return -1;
+ }
+ for (;;) {
+ char buf[PATH_MAX + 1];
+ size_t len;
+ if (!fgets(buf, PATH_MAX + 1, in)) {
+ if (feof(in)) {
+ break;
+ }
+ fprintf(stderr,
+ "ERROR: Cannot read from file `%s`.\n", argv[4]);
+ return -1;
+ }
+ len = strlen(buf);
+ if (len != 0 && buf[len - 1] == '\n') {
+ buf[len - 1] = '\0';
+ } else if (!feof(in)) {
+ fprintf(stderr,
+ "ERROR: Too long line in file `%s`.\n", argv[4]);
+ return -1;
+ }
+ if (numfiles == 10000) { /*TODO*/
+ fprintf(stderr,
+ "ERROR: Too many lines in file `%s`.\n", argv[4]);
+ return -1;
+ }
+ files[numfiles] = strdup(buf);
+ if (!files[numfiles]) {
+ fprintf(stderr,
+ "ERROR: Cannot allocate memory");
+ return -1;
+ }
+ ++numfiles;
+ }
+ fclose(in);
+ } else {
+ numfiles = argc - 3;
+ files = argv + 3;
+ }
infoBlock.MARChannelID = MARChannelID;
infoBlock.productVersion = productVersion;
- return mar_create(argv[2], argc - 3, argv + 3, &infoBlock);
+ return mar_create(argv[2], numfiles, files, &infoBlock);
}
case 'i': {
if (!productVersion) {
--- onlineupdate/source/service/serviceinstall.cpp
+++ onlineupdate/source/service/serviceinstall.cpp
@@ -25,7 +25,7 @@
@@ -130,3 +223,43 @@
LOG_WARN(("Install directory updater could not be determined."));
result = FALSE;
}
--- tools/update-packaging/make_full_update.sh
+++ tools/update-packaging/make_full_update.sh
@@ -53,9 +53,10 @@
fi
workdir="$targetdir.work"
updatemanifestv3="$workdir/updatev3.manifest"
-targetfiles="updatev3.manifest"
mkdir -p "$workdir"
+
+printf 'updatev3.manifest\n' >"$workdir/files.txt"
# Generate a list of all files in the target directory.
pushd "$targetdir"
@@ -66,7 +67,6 @@
if [ ! -f "precomplete" ]; then
if [ ! -f "Contents/Resources/precomplete" ]; then
notice "precomplete file is missing!"
- exit 1
fi
fi
@@ -99,7 +99,7 @@
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f"
- targetfiles="$targetfiles \"$f\""
+ printf '%s\n' "$f" >>"$workdir/files.txt"
done
# Append remove instructions for any dead files.
@@ -110,7 +110,7 @@
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
mar_command="$mar_command -C \"$workdir\" -c output.mar"
-eval "$mar_command $targetfiles"
+eval "$mar_command -f $workdir/files.txt"
mv -f "$workdir/output.mar" "$archive"
# cleanup
