merge 'after squeeze release'-stuff

[ David Kalnischkies ]
* apt-pkg/depcache.cc:
  - add SetCandidateRelease() to set a candidate version and
    the candidates of dependencies if needed to a specified
    release (Closes: #572709)
* cmdline/apt-get.cc:
  - if --print-uris is used don't setup downloader as we don't need
    progress, lock nor the directories it would create otherwise
  - show dependencies of essential packages which are going to remove
    only if they cause the remove of this essential (Closes: #601961)
  - keep not installed garbage packages uninstalled instead of showing
    in the autoremove section and installing those (Closes: #604222)
  - change pkg/release behavior to use the new SetCandidateRelease
    so installing packages from experimental or backports is easier
  - really do not show packages in the extra section if they were
    requested on the commandline, e.g. with a modifier (Closes: #184730)
* debian/control:
  - add Vcs-Browser now that loggerhead works again (Closes: #511168)
  - depend on debhelper 7 to raise compat level
  - depend on dpkg-dev (>= 1.15.8) to have c++ symbol mangling
* apt-pkg/contrib/fileutl.cc:
  - add a RealFileExists method and check that your configuration files
    are real files to avoid endless loops if not (Closes: #604401)
  - ignore non-regular files in GetListOfFilesInDir (Closes: #594694)
* apt-pkg/contrib/weakptr.h:
  - include stddefs.h to fix compile error (undefined NULL) with gcc-4.6
* methods/https.cc:
  - fix CURLOPT_SSL_VERIFYHOST by really passing 2 to it if enabled
* deb/dpkgpm.cc:
  - fix popen/fclose mismatch reported by cppcheck. Thanks to Petter
    Reinholdtsen for report and patch! (Closes: #607803)
* doc/apt.conf.5.xml:
  - fix multipl{y,e} spelling error reported by Jakub Wilk (Closes: #607636)
* apt-inst/contrib/extracttar.cc:
  - let apt-utils work with encoded tar headers if uid/gid are large.
    Thanks to Nobuhiro Hayashi for the patch! (Closes: #330162)
* apt-pkg/cacheiterator.h:
  - do not segfault if cache is not build (Closes: #254770)
* doc/apt-get.8.xml:
  - remove duplicated mentioning of --install-recommends
* doc/sources.list.5.xml:
  - remove obsolete references to non-us (Closes: #594495)
* debian/rules:
  - use -- instead of deprecated -u for dh_gencontrol
  - remove shlibs.local creation and usage
  - show differences in the symbol files, but never fail
* pre-build.sh:
  - remove as it is not needed for a working 'bzr bd'
* debian/{apt,apt-utils}.symbols:
  - ship experimental unmangled c++ symbol files
* methods/rred.cc:
  - operate optional on gzip compressed pdiffs
* apt-pkg/acquire-item.cc:
  - don't uncompress downloaded pdiff files before feeding it to rred
  - try downloading clearsigned InRelease before trying Release.gpg
  - change the internal handling of Extensions in pkgAcqIndex
  - add a special uncompressed compression type to prefer those files
  - download and use i18n/Index to choose which Translations to download
* cmdline/apt-key:
  - don't set trustdb-name as non-root so 'list' and 'finger'
    can be used without being root (Closes: #393005, #592107)
* apt-pkg/deb/deblistparser.cc:
  - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles
* ftparchive/writer.cc:
  - add config option to search for more patterns in release command
  - include Index files by default in the Release file
* methods/{gzip,bzip}.cc:
  - print a good error message if FileSize() is zero
* apt-pkg/aptconfiguration.cc:
  - remove the inbuilt Translation files whitelist

1 files changed, 152 insertions, 34 deletions

diff --git a/test/integration/framework b/test/integration/framework
index f78ae2ee1..fe1db14bc 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -39,6 +39,10 @@ fi
 if [ $MSGLEVEL -le 2 ]; then
 	msgmsg() { true; }
 	msgnmsg() { true; }
+	msgtest() { true; }
+	msgpass() { echo -n " ${CPASS}P${CNORMAL}" >&2; }
+	msgskip() { echo -n " ${CWARNING}S${CNORMAL}" >&2; }
+	msgfail() { echo -n " ${CFAIL}FAIL${CNORMAL}" >&2; }
 fi
 if [ $MSGLEVEL -le 3 ]; then
 	msginfo() { true; }
@@ -78,6 +82,20 @@ aptkey() { runapt apt-key $*; }
 dpkg() {
 	$(which dpkg) --root=${TMPWORKINGDIRECTORY}/rootdir --force-not-root --force-bad-path --log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log $*
 }
+aptitude() {
+	if [ -f ./aptconfig.conf ]; then
+		APT_CONFIG=aptconfig.conf LD_LIBRARY_PATH=${BUILDDIRECTORY}  $(which aptitude) $*
+	elif [ -f ../aptconfig.conf ]; then
+		APT_CONFIG=../aptconfig.conf LD_LIBRARY_PATH=${BUILDDIRECTORY} $(which aptitude) $*
+	else
+		LD_LIBRARY_PATH=${BUILDDIRECTORY}  $(which aptitude) $*
+	fi
+}
+
+addtrap() {
+	CURRENTTRAP="$CURRENTTRAP $1"
+	trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
+}
 
 setupenvironment() {
 	TMPWORKINGDIRECTORY=$(mktemp -d)
@@ -86,8 +104,7 @@ setupenvironment() {
 	BUILDDIRECTORY="${TESTDIR}/../../build/bin"
 	test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first"
 	local OLDWORKINGDIRECTORY=$(pwd)
-	CURRENTTRAP="cd /; rm -rf $TMPWORKINGDIRECTORY; cd $OLDWORKINGDIRECTORY"
-	trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
+	addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY; cd $OLDWORKINGDIRECTORY;"
 	cd $TMPWORKINGDIRECTORY
 	mkdir rootdir aptarchive keys
 	cd rootdir
@@ -107,8 +124,10 @@ setupenvironment() {
 	local PACKAGESFILE=$(echo "$(basename $0)" | sed -e 's/^test-/Packages-/' -e 's/^skip-/Packages-/')
 	if [ -f "${TESTDIR}/${PACKAGESFILE}" ]; then
 		cp "${TESTDIR}/${PACKAGESFILE}" aptarchive/Packages
-	else
-		touch aptarchive/Packages
+	fi
+	local SOURCESSFILE=$(echo "$(basename $0)" | sed -e 's/^test-/Sources-/' -e 's/^skip-/Sources-/')
+	if [ -f "${TESTDIR}/${SOURCESSFILE}" ]; then
+		cp "${TESTDIR}/${SOURCESSFILE}" aptarchive/Sources
 	fi
 	cp $(find $TESTDIR -name '*.pub' -o -name '*.sec') keys/
 	ln -s ${TMPWORKINGDIRECTORY}/keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
@@ -116,6 +135,7 @@ setupenvironment() {
 	echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
 	echo "Debug::NoLocking \"true\";" >> aptconfig.conf
 	echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf
+	echo "Dir::Bin::Methods \"${BUILDDIRECTORY}/methods\";" >> aptconfig.conf
 	echo "Dir::Bin::dpkg \"fakeroot\";" >> aptconfig.conf
 	echo "Dir::Bin::methods \"${BUILDDIRECTORY}/methods\";" >> aptconfig.conf
 	echo "DPKG::options:: \"dpkg\";" >> aptconfig.conf
@@ -123,6 +143,7 @@ setupenvironment() {
 	echo "DPKG::options:: \"--force-not-root\";" >> aptconfig.conf
 	echo "DPKG::options:: \"--force-bad-path\";" >> aptconfig.conf
 	echo "DPKG::options:: \"--log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log\";" >> aptconfig.conf
+	echo 'quiet::NoUpdate "true";' >> aptconfig.conf
 	export LC_ALL=C
 	msgdone "info"
 }
@@ -168,8 +189,12 @@ Maintainer: Joe Sixpack <joe@example.org>
 Build-Depends: debhelper (>= 7)
 Standards-Version: 3.9.1
 
-Package: $NAME
-Architecture: $ARCH" > debian/control
+Package: $NAME" > debian/control
+	if [ "$ARCH" = 'all' ]; then
+		echo "Architecture: all" >> debian/control
+	else
+		echo "Architecture: any" >> debian/control
+	fi
 	test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> debian/control
 	if [ -z "$DESCRIPTION" ]; then
 		echo "Description: an autogenerated dummy ${NAME}=${VERSION}/${RELEASE}
@@ -199,9 +224,57 @@ buildsimplenativepackage() {
 	else
 		DISTSECTION="$(echo "$SECTION" | cut -d'/' -f 1)"
 	fi
-	setupsimplenativepackage "$NAME" "$ARCH" "$VERSION" "$RELEASE" "$DEPENDENCIES" "$DESCRIPTION" "$SECTION"
-	buildpackage "incoming/${NAME}-${VERSION}" "$RELEASE" "$DISTSECTION"
-	rm -rf "incoming/${NAME}-${VERSION}"
+	local BUILDDIR=${TMPWORKINGDIRECTORY}/incoming/${NAME}-${VERSION}
+
+	msgninfo "Build package ${NAME} in ${VERSION} for ${RELEASE} in ${DISTSECTION}… "
+	mkdir -p $BUILDDIR/debian/source
+	echo "* most suckless software product ever" > ${BUILDDIR}/FEATURES
+	echo "#!/bin/sh
+echo '$NAME says \"Hello!\"'" > ${BUILDDIR}/${NAME}
+
+	echo "Copyleft by Joe Sixpack $(date +%Y)" > ${BUILDDIR}/debian/copyright
+	echo "$NAME ($VERSION) $RELEASE; urgency=low
+
+  * Initial release
+
+ -- Joe Sixpack <joe@example.org>  $(date -R)" > ${BUILDDIR}/debian/changelog
+	echo "Source: $NAME
+Section: $SECTION
+Priority: optional
+Maintainer: Joe Sixpack <joe@example.org>
+Standards-Version: 3.9.1
+
+Package: $NAME" > ${BUILDDIR}/debian/control
+	if [ "$ARCH" = 'all' ]; then
+		echo "Architecture: all" >> ${BUILDDIR}/debian/control
+	else
+		echo "Architecture: any" >> ${BUILDDIR}/debian/control
+	fi
+	test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> ${BUILDDIR}/debian/control
+	if [ -z "$DESCRIPTION" ]; then
+		echo "Description: an autogenerated dummy ${NAME}=${VERSION}/${RELEASE}
+ If you find such a package installed on your system,
+ YOU did something horribly wrong! They are autogenerated
+ und used only by testcases for APT and surf no other propose…" >> ${BUILDDIR}/debian/control
+	else
+		echo "Description: $DESCRIPTION" >> ${BUILDIR}/debian/control
+	fi
+	echo '3.0 (native)' > ${BUILDDIR}/debian/source/format
+	local SRCS="$( (cd ${BUILDDIR}/..; dpkg-source -b ${NAME}-${VERSION} 2>&1) | grep '^dpkg-source: info: building' | grep -o '[a-z0-9._+~-]*$')"
+
+	mkdir -p ${BUILDDIR}/debian/tmp/DEBIAN ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME} ${BUILDDIR}/debian/tmp/usr/bin
+	cp ${BUILDDIR}/debian/copyright ${BUILDDIR}/debian/changelog ${BUILDDIR}/FEATURES ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME}
+	cp ${BUILDDIR}/${NAME} ${BUILDDIR}/debian/tmp/usr/bin
+	(cd ${BUILDDIR}; dpkg-gencontrol -DArchitecture=$ARCH)
+	(cd ${BUILDDIR}/debian/tmp; md5sum $(find usr/ -type f) > DEBIAN/md5sums)
+
+	dpkg-deb --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/.. > /dev/null
+	echo "pool/${NAME}_${VERSION}_${ARCH}.deb" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.pkglist
+	for SRC in $SRCS; do
+		echo "pool/${SRC}" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.srclist
+	done
+	rm -rf "${BUILDDIR}"
+	msgdone "info"
 }
 
 buildpackage() {
@@ -255,6 +328,8 @@ Default {
 	Packages::Compress ". gzip bzip2 lzma";
 	Sources::Compress ". gzip bzip2 lzma";
 	Contents::Compress ". gzip bzip2 lzma";
+	Translation::Compress ". gzip bzip2 lzma";
+	LongDescription "false";
 };
 TreeDefault {
 	Directory "pool/";
@@ -308,6 +383,38 @@ buildaptftparchivedirectorystructure() {
 	done
 }
 
+insertpackage() {
+	local RELEASE="$1"
+	local NAME="$2"
+	local ARCH="$3"
+	local VERSION="$4"
+	local DEPENDENCIES="$5"
+	local ARCHS="$ARCH"
+	if [ "$ARCHS" = "all" ]; then
+		ARCHS="$(aptconfig dump | grep APT::Architecture | cut -d'"' -f 2 | sed '/^$/ d' | sort | uniq | tr '\n' ' ')"
+	fi
+	for BUILDARCH in $ARCHS; do
+		local PPATH="aptarchive/dists/${RELEASE}/main/binary-${BUILDARCH}"
+		mkdir -p $PPATH aptarchive/dists/${RELEASE}/main/source
+		touch aptarchive/dists/${RELEASE}/main/source/Sources
+		local FILE="${PPATH}/Packages"
+		echo "Package: $NAME
+Priority: optional
+Section: other
+Installed-Size: 42
+Maintainer: Joe Sixpack <joe@example.org>
+Architecture: $ARCH
+Version: $VERSION
+Filename: pool/main/${NAME}/${NAME}_${VERSION}_${ARCH}.deb" >> $FILE
+		test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
+		echo "Description: an autogenerated dummy ${NAME}=${VERSION}/${RELEASE}
+ If you find such a package installed on your system,
+ YOU did something horribly wrong! They are autogenerated
+ und used only by testcases for APT and surf no other propose…
+" >> $FILE
+	done
+}
+
 buildaptarchivefromincoming() {
 	msginfo "Build APT archive for ${CCMD}$(basename $0)${CINFO} based on incoming packages…"
 	cd aptarchive
@@ -316,36 +423,47 @@ buildaptarchivefromincoming() {
 	[ -e dists ] || buildaptftparchivedirectorystructure
 	msgninfo "\tGenerate Packages, Sources and Contents files… "
 	aptftparchive -qq generate ftparchive.conf
-	msgdone "info"
-	msgninfo "\tGenerate Release files… "
-	for dir in $(find ./dists -mindepth 1 -maxdepth 1 -type d); do
-		aptftparchive -qq release $dir -o APT::FTPArchive::Release::Codename="$(echo "$dir" | cut -d'/' -f 3)" | sed -e '/0 Release$/ d' > $dir/Release # remove the self reference
-	done
 	cd - > /dev/null
 	msgdone "info"
+	generatereleasefiles
 }
 
 buildaptarchivefromfiles() {
 	msginfo "Build APT archive for ${CCMD}$(basename $0)${CINFO} based on prebuild files…"
-	cd aptarchive
-	if [ -f Packages ]; then
-		msgninfo "\tPackages file… "
-		cat Packages | gzip > Packages.gz
-		cat Packages | bzip2 > Packages.bz2
-		cat Packages | lzma > Packages.lzma
+	find aptarchive -name 'Packages' -o -name 'Sources' | while read line; do
+		msgninfo "\t${line} file… "
+		cat ${line} | gzip > ${line}.gz
+		cat ${line} | bzip2 > ${line}.bz2
+		cat ${line} | lzma > ${line}.lzma
 		msgdone "info"
+	done
+	generatereleasefiles
+}
+
+generatereleasefiles() {
+	msgninfo "\tGenerate Release files… "
+	local DATE="${1:-now}"
+	if [ -e aptarchive/dists ]; then
+		for dir in $(find ./aptarchive/dists -mindepth 3 -maxdepth 3 -type d -name 'i18n'); do
+			aptftparchive -qq release $dir -o APT::FTPArchive::Release::Patterns::='Translation-*' > $dir/Index
+		done
+		for dir in $(find ./aptarchive/dists -mindepth 1 -maxdepth 1 -type d); do
+			local CODENAME="$(echo "$dir" | cut -d'/' -f 4)"
+			aptftparchive -qq release $dir -o APT::FTPArchive::Release::Suite="${CODENAME}" -o APT::FTPArchive::Release::Codename="${CODENAME}" | sed -e '/0 Release$/ d' > $dir/Release # remove the self reference
+			if [ "$CODENAME" = "experimental" -o "$CODENAME" = "experimental2" ]; then
+				sed -i '/^Date: / a\
+NotAutomatic: yes' $dir/Release
+			fi
+		done
+	else
+		aptftparchive -qq release ./aptarchive | sed -e '/0 Release$/ d' > aptarchive/Release # remove the self reference
 	fi
-	if [ -f Sources ]; then
-		msgninfo "\tSources file… "
-		cat Sources | gzip > Sources.gz
-		cat Sources | bzip2 > Sources.bz2
-		cat Sources | lzma > Sources.lzma
-		msgdone "info"
+	if [ "$DATE" != "now" ]; then
+		for release in $(find ./aptarchive -name 'Release'); do
+			touch -d "$1" $release
+		done
 	fi
-	msgninfo "\tRelease file… "
-	aptftparchive -qq release . | sed -e '/0 Release$/ d' > Release # remove the self reference
 	msgdone "info"
-	cd ..
 }
 
 setupdistsaptarchive() {
@@ -405,6 +523,7 @@ signreleasefiles() {
 	done
 	for RELEASE in $(find aptarchive/ -name Release); do
 		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" -abs -o ${RELEASE}.gpg ${RELEASE}
+		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" --clearsign -o "$(echo "${RELEASE}" | sed 's#/Release$#/InRelease#')" $RELEASE
 	done
 	msgdone "info"
 }
@@ -412,8 +531,7 @@ signreleasefiles() {
 changetowebserver() {
 	if which weborf > /dev/null; then
 		weborf -xb aptarchive/ 2>&1 > /dev/null &
-		CURRENTTRAP="kill $(ps | grep weborf | sed -e 's#^[ ]*##' | cut -d' ' -f 1); $CURRENTTRAP"
-		trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
+		addtrap "kill $!;"
 		local APTARCHIVE="file://$(readlink -f ./aptarchive)"
 		for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
 			sed -i $LIST -e "s#$APTARCHIVE#http://localhost:8080/#"
@@ -447,17 +565,18 @@ testfileequal() {
 
 testequal() {
 	local COMPAREFILE=$(mktemp)
+	addtrap "rm $COMPAREFILE;"
 	echo "$1" > $COMPAREFILE
 	shift
 	msgtest "Test for equality of" "$*"
 	$* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
-	rm $COMPAREFILE
 }
 
 testequalor2() {
 	local COMPAREFILE1=$(mktemp)
 	local COMPAREFILE2=$(mktemp)
 	local COMPAREAGAINST=$(mktemp)
+	addtrap "rm $COMPAREFILE1 $COMPAREFILE2 $COMPAREAGAINST;"
 	echo "$1" > $COMPAREFILE1
 	echo "$2" > $COMPAREFILE2
 	shift 2
@@ -468,7 +587,6 @@ testequalor2() {
 		( echo "\n${CINFO}Diff against OR 1${CNORMAL}" "$(checkdiff $COMPAREFILE1 $COMPAREAGAINST)" \
 		       "\n${CINFO}Diff against OR 2${CNORMAL}" "$(checkdiff $COMPAREFILE2 $COMPAREAGAINST)" &&
 		  msgfail )
-	rm $COMPAREFILE1 $COMPAREFILE2 $COMPAREAGAINST
 }
 
 testshowvirtual() {
@@ -485,11 +603,11 @@ N: Can't select versions from package '$1' as it purely virtual"
 	VIRTUAL="${VIRTUAL}
 N: No packages found"
 	local COMPAREFILE=$(mktemp)
+	addtrap "rm $COMPAREFILE;"
 	local ARCH=$(dpkg-architecture -qDEB_HOST_ARCH_CPU)
 	eval `apt-config shell ARCH APT::Architecture`
 	echo "$VIRTUAL" | sed -e "s/:$ARCH//" -e 's/:all//' > $COMPAREFILE
 	aptcache show -q=0 $PACKAGE 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
-	rm $COMPAREFILE
 }
 
 testnopackage() {