From 63d0f85391839a666957add1833e67f7638c8a83 Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@debian.org>
Date: Fri, 1 Aug 2014 19:25:00 +0200
Subject: make i-m-s work again

---
 test/integration/test-apt-update-ims | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100755 test/integration/test-apt-update-ims

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
new file mode 100755
index 000000000..cf2b28bb5
--- /dev/null
+++ b/test/integration/test-apt-update-ims
@@ -0,0 +1,22 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
+
+setupaptarchive
+changetowebserver
+
+testsuccess aptget update
+
+# check that I-M-S header is kept in redirections
+testequal "Hit http://localhost:8080 unstable InRelease
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists..." aptget update 
+
-- 
cgit v1.2.3


From 63b7249e6930c1bcb69bac32f10108119eeacc2a Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@debian.org>
Date: Sat, 2 Aug 2014 05:37:43 +0200
Subject: add ims check verify

---
 test/integration/test-apt-update-ims | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index cf2b28bb5..3bd6e843c 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -20,3 +20,6 @@ Hit http://localhost:8080 unstable/main amd64 Packages
 Hit http://localhost:8080 unstable/main Translation-en
 Reading package lists..." aptget update 
 
+# ensure that we still do a hash check on ims hit
+msgtest 'Test I-M-S reverify'
+aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
-- 
cgit v1.2.3


From 47450dea0904298c8d5ea06b15ea26368da5a4ee Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Wed, 1 Oct 2014 18:01:14 +0200
Subject: fix leftover files from Acquire::GzipIndex

---
 test/integration/test-apt-update-ims | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 3bd6e843c..946dfc7af 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -11,15 +11,30 @@ buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
 setupaptarchive
 changetowebserver
 
-testsuccess aptget update
+runtest() {
+    rm -f rootdir/var/lib/apt/lists/localhost*
 
-# check that I-M-S header is kept in redirections
-testequal "Hit http://localhost:8080 unstable InRelease
+    testsuccess aptget update
+
+    # ensure no leftovers in partial
+    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+
+    # check that I-M-S header is kept in redirections
+    testequal "Hit http://localhost:8080 unstable InRelease
 Hit http://localhost:8080 unstable/main Sources
 Hit http://localhost:8080 unstable/main amd64 Packages
 Hit http://localhost:8080 unstable/main Translation-en
-Reading package lists..." aptget update 
+Reading package lists..." aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
+    
+    # ensure that we still do a hash check on ims hit
+    msgtest 'Test I-M-S reverify'
+    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+
+    # ensure no leftovers in partial
+    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+}
+
+runtest
 
-# ensure that we still do a hash check on ims hit
-msgtest 'Test I-M-S reverify'
-aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
-- 
cgit v1.2.3


From 1ce243188c2ba218f5dce8ec8b40556d58ed8ec2 Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Thu, 2 Oct 2014 18:28:55 +0200
Subject: cleanup around pkgAcqMetaSig and improved tests

---
 test/integration/test-apt-update-ims | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 946dfc7af..38dcd73fd 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -20,11 +20,7 @@ runtest() {
     testfailure ls "rootdir/var/lib/apt/lists/partial/*"
 
     # check that I-M-S header is kept in redirections
-    testequal "Hit http://localhost:8080 unstable InRelease
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
-Reading package lists..." aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
+    testequal "$EXPECT" aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
     
     # ensure that we still do a hash check on ims hit
     msgtest 'Test I-M-S reverify'
@@ -34,6 +30,34 @@ Reading package lists..." aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug
     testfailure ls "rootdir/var/lib/apt/lists/partial/*"
 }
 
+EXPECT="Hit http://localhost:8080 unstable InRelease
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists..."
+# with InRelease
+runtest
+
+# with gzip
+echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
+
+# FIXME: how can we get rid of this extra line
+#  "Get:1 http://localhost:8080 unstable Release.gpg" 
+#
+# with Release/Release.gpg 
+EXPECT="Ign http://localhost:8080 unstable InRelease
+Hit http://localhost:8080 unstable Release
+Hit http://localhost:8080 unstable Release.gpg
+Get:1 http://localhost:8080 unstable Release.gpg
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists..."
+
+find aptarchive -name "InRelease" | xargs rm -f
+
+echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
 echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
-- 
cgit v1.2.3


From f30976478e684fc19e48d71881805454ceb6ecae Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Mon, 6 Oct 2014 11:45:42 +0200
Subject: Rework pkgAcqMeta{Index,Sig,ClearSig}::Done() for readability

Move common code out but do not use subclassing for ::Done
to make it easier to understand what each class is doing when
its done
---
 test/integration/test-apt-update-ims | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 38dcd73fd..61b808b0f 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -30,6 +30,7 @@ runtest() {
     testfailure ls "rootdir/var/lib/apt/lists/partial/*"
 }
 
+msgmsg "InRelease"
 EXPECT="Hit http://localhost:8080 unstable InRelease
 Hit http://localhost:8080 unstable/main Sources
 Hit http://localhost:8080 unstable/main amd64 Packages
@@ -42,14 +43,11 @@ runtest
 echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
-# FIXME: how can we get rid of this extra line
-#  "Get:1 http://localhost:8080 unstable Release.gpg" 
-#
+msgmsg "Release/Release.gpg"
 # with Release/Release.gpg 
 EXPECT="Ign http://localhost:8080 unstable InRelease
 Hit http://localhost:8080 unstable Release
 Hit http://localhost:8080 unstable Release.gpg
-Get:1 http://localhost:8080 unstable Release.gpg
 Hit http://localhost:8080 unstable/main Sources
 Hit http://localhost:8080 unstable/main amd64 Packages
 Hit http://localhost:8080 unstable/main Translation-en
@@ -62,3 +60,23 @@ runtest
 
 echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
+
+
+# no Release.gpg or InRelease
+msgmsg "Release only"
+EXPECT="Ign http://localhost:8080 unstable InRelease
+Hit http://localhost:8080 unstable Release
+Ign http://localhost:8080 unstable Release.gpg
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists..."
+
+find aptarchive -name "Release.gpg" | xargs rm -f
+
+echo 'Acquire::AllowInsecureRepositories "1";' > rootdir/etc/apt/apt.conf.d/insecure.conf
+echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
+
+echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
-- 
cgit v1.2.3


From 4dbfe436c60880f2625e4d3a9d0127a83dd6276e Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Tue, 7 Oct 2014 01:46:30 +0200
Subject: display errortext for all Err as well as Ign logs

consistently using Item::Failed in all specializec classes helps setting
up some information bits otherwise unset, so some errors had an empty
reason as an error. Ign is upgraded to display the error message we
ignored to further help in understanding what happens.
---
 test/integration/test-apt-update-ims | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 61b808b0f..8aa5a7262 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -44,8 +44,9 @@ echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
 msgmsg "Release/Release.gpg"
-# with Release/Release.gpg 
+# with Release/Release.gpg
 EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
 Hit http://localhost:8080 unstable Release
 Hit http://localhost:8080 unstable Release.gpg
 Hit http://localhost:8080 unstable/main Sources
@@ -65,8 +66,10 @@ runtest
 # no Release.gpg or InRelease
 msgmsg "Release only"
 EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
 Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
+  404  Not Found
 Hit http://localhost:8080 unstable/main Sources
 Hit http://localhost:8080 unstable/main amd64 Packages
 Hit http://localhost:8080 unstable/main Translation-en
-- 
cgit v1.2.3


From 9d653a6de2ca952247cc4e628256259d225570a6 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Mon, 13 Oct 2014 09:54:21 +0200
Subject: fix compile and tests error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I am pretty sure I did that before committing broken stuff…

Git-Dch: Ignore
---
 test/integration/test-apt-update-ims | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 8aa5a7262..06b9c2b62 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -73,7 +73,8 @@ Ign http://localhost:8080 unstable Release.gpg
 Hit http://localhost:8080 unstable/main Sources
 Hit http://localhost:8080 unstable/main amd64 Packages
 Hit http://localhost:8080 unstable/main Translation-en
-Reading package lists..."
+Reading package lists...
+W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated."
 
 find aptarchive -name "Release.gpg" | xargs rm -f
 
-- 
cgit v1.2.3


From d4f4bcf76bb2035b7df370a82b081384140b3083 Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Tue, 14 Oct 2014 17:00:56 +0200
Subject: Add new configallowinsecurerepositories to the test framework

Add a new configallowinsecurerepositories that controls the value
of Acquire::AllowInsecureRepositories for the tests. Set it to
"false" for most of the testsuite and only enable it where its
really needed. We want to switch the default for this post-jessie.
---
 test/integration/test-apt-update-ims | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 06b9c2b62..c74058c5d 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -64,6 +64,8 @@ runtest
 
 
 # no Release.gpg or InRelease
+configallowinsecurerepositories "true"
+
 msgmsg "Release only"
 EXPECT="Ign http://localhost:8080 unstable InRelease
   404  Not Found
-- 
cgit v1.2.3


From ab25bf1f08ca605d3eb4c05619e8df495ccca30a Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Sat, 18 Oct 2014 20:00:25 +0200
Subject: autorun permission tests for all apt-get update calls

Adds some infrastructure to run tests automatically for certain
commands. The first command being 'apt-get update' (and 'apt update')
which check for correct permission and owner of the files in lists/.

Git-Dch: Ignore
---
 test/integration/test-apt-update-ims | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index c74058c5d..6746837a4 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -54,7 +54,7 @@ Hit http://localhost:8080 unstable/main amd64 Packages
 Hit http://localhost:8080 unstable/main Translation-en
 Reading package lists..."
 
-find aptarchive -name "InRelease" | xargs rm -f
+find aptarchive -name 'InRelease' -delete
 
 echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
@@ -78,9 +78,8 @@ Hit http://localhost:8080 unstable/main Translation-en
 Reading package lists...
 W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated."
 
-find aptarchive -name "Release.gpg" | xargs rm -f
+find aptarchive -name 'Release.gpg' -delete
 
-echo 'Acquire::AllowInsecureRepositories "1";' > rootdir/etc/apt/apt.conf.d/insecure.conf
 echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
-- 
cgit v1.2.3


From 846bc058cb0c1bf7ce7c2fb30b9c277e96e9eaf7 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Sat, 18 Oct 2014 22:46:48 +0200
Subject: check lists/ content in tests doing rollback

Git-Dch: Ignore
---
 test/integration/test-apt-update-ims | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 6746837a4..eece0c84c 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -8,7 +8,7 @@ configarchitecture 'amd64'
 
 buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
 
-setupaptarchive
+setupaptarchive --no-update
 changetowebserver
 
 runtest() {
@@ -23,7 +23,7 @@ runtest() {
     testequal "$EXPECT" aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
     
     # ensure that we still do a hash check on ims hit
-    msgtest 'Test I-M-S reverify'
+    msgtest 'Test I-M-S' 'reverify'
     aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
 
     # ensure no leftovers in partial
-- 
cgit v1.2.3


From 4fa34122cbe347d21b3a162ff2fa75dd2e73c3a8 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Mon, 20 Oct 2014 10:23:41 +0200
Subject: testcases: do not allow warnings in testsuccess

Adds a new testwarning which tests for zero exit and the presents of a
warning in the output, failing if either is not the case or if an error
is found, too. This allows us to change testsuccess to accept only
totally successful executions (= without warnings) which should help
finding regressions.

Git-Dch: Ignore
---
 test/integration/test-apt-update-ims | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index eece0c84c..afae99563 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -12,9 +12,15 @@ setupaptarchive --no-update
 changetowebserver
 
 runtest() {
+    configallowinsecurerepositories "${1:-false}"
+
     rm -f rootdir/var/lib/apt/lists/localhost*
 
-    testsuccess aptget update
+    if [ "$1" = 'true' ]; then
+	testwarning aptget update
+    else
+	testsuccess aptget update
+    fi
 
     # ensure no leftovers in partial
     testfailure ls "rootdir/var/lib/apt/lists/partial/*"
@@ -62,10 +68,7 @@ runtest
 echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
-
 # no Release.gpg or InRelease
-configallowinsecurerepositories "true"
-
 msgmsg "Release only"
 EXPECT="Ign http://localhost:8080 unstable InRelease
   404  Not Found
@@ -81,7 +84,7 @@ W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Pac
 find aptarchive -name 'Release.gpg' -delete
 
 echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
-runtest
+runtest "true"
 
 echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
-runtest
+runtest "true"
-- 
cgit v1.2.3


From 23397c9d7d4d455461176600bb45c81185493504 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Thu, 23 Oct 2014 16:54:00 +0200
Subject: promote filesize to a hashstring

It is a very simple hashstring, which is why it isn't contributing to
the usability of a list of them, but it is also trivial to check and
calculate, so it doesn't hurt checking it either as it can combined even
with the simplest other hashes greatly complicate attacks on them as you
suddenly need a same-size hash collision, which is usually a lot harder
to achieve.
---
 test/integration/test-apt-update-ims | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index afae99563..5394a9f30 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -30,7 +30,7 @@ runtest() {
     
     # ensure that we still do a hash check on ims hit
     msgtest 'Test I-M-S' 'reverify'
-    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
 
     # ensure no leftovers in partial
     testfailure ls "rootdir/var/lib/apt/lists/partial/*"
-- 
cgit v1.2.3


From 20801f613690b330c79b4f7a30dc3ff52b722468 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Sat, 8 Nov 2014 14:23:18 +0100
Subject: fix variable naming typo used in debug output

Git-Dch: Ignore
---
 test/integration/test-apt-update-ims | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 5394a9f30..0fa882d78 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -30,7 +30,7 @@ runtest() {
     
     # ensure that we still do a hash check on ims hit
     msgtest 'Test I-M-S' 'reverify'
-    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'ReceivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
 
     # ensure no leftovers in partial
     testfailure ls "rootdir/var/lib/apt/lists/partial/*"
-- 
cgit v1.2.3


From ba6b79bd0090077724fa1272ea4d3a31706fcd5a Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Sun, 12 Apr 2015 17:08:46 +0200
Subject: a hit on Release files means the indexes will be hits too

If we get a IMSHit for the Transaction-Manager (= the InRelease file or
as its still supported fallback Release + Release.gpg combo) we can
assume that every file we would queue based on this manager, but already
have locally is current and hence would get an IMSHit, too. We therefore
save us and the server the trouble and skip the queuing in this case.
Beside speeding up repetative executions of 'apt-get update' this way we
also avoid hitting hashsum errors if the indexes are in fact already
updated, but the Release file isn't yet as it is the case on well
behaving mirrors as Release files is updated last.

The implementation is a bit harder than the theory makes it sound as we
still have to keep reverifying the Release files (e.g. to detect now expired
once to avoid an attacker being able to silently stale us) and have to
handle cases in which the Release file hits, but some indexes aren't
present (e.g. user added a new foreign architecture).
---
 test/integration/test-apt-update-ims | 150 ++++++++++++++++++++++++-----------
 1 file changed, 105 insertions(+), 45 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 0fa882d78..f091bffaa 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -6,85 +6,145 @@ TESTDIR=$(readlink -f $(dirname $0))
 setupenvironment
 configarchitecture 'amd64'
 
-buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
+insertpackage 'unstable' 'unrelated' 'all' '0.5~squeeze1'
+insertsource 'unstable' 'unrelated' 'all' '0.5~squeeze1'
 
 setupaptarchive --no-update
 changetowebserver
 
 runtest() {
-    configallowinsecurerepositories "${1:-false}"
+    if [ -n "$1" ]; then
+	configallowinsecurerepositories 'true'
+    else
+	configallowinsecurerepositories 'false'
+    fi
 
-    rm -f rootdir/var/lib/apt/lists/localhost*
+    rm -rf rootdir/var/lib/apt/lists/
 
-    if [ "$1" = 'true' ]; then
-	testwarning aptget update
-    else
-	testsuccess aptget update
+    local TEST="test${1:-success}"
+    $TEST aptget update
+    if [ "$1" = 'failure' ]; then
+	# accept the outdated Release file so we can check Hit behaviour
+	"test${2:-success}" aptget update -o Acquire::Min-ValidTime=99999999999
     fi
+    listcurrentlistsdirectory > listsdir.lst
+    testsuccess grep '_Packages\(\.gz\)\?$' listsdir.lst
+    testsuccess grep '_Sources\(\.gz\)\?$' listsdir.lst
+    testsuccess grep '_Translation-en\(\.gz\)\?$' listsdir.lst
 
     # ensure no leftovers in partial
-    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+    testfailure ls 'rootdir/var/lib/apt/lists/partial/*'
 
     # check that I-M-S header is kept in redirections
-    testequal "$EXPECT" aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
-    
-    # ensure that we still do a hash check on ims hit
-    msgtest 'Test I-M-S' 'reverify'
-    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'ReceivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+    echo "$EXPECT" | sed -e 's#(invalid since [^)]\+)#(invalid since)#' > expected.output
+    $TEST aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
+    sed -i -e 's#(invalid since [^)]\+)#(invalid since)#' rootdir/tmp/${TEST}.output
+    testequal "$(cat expected.output)" cat rootdir/tmp/${TEST}.output
+    testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+    # ensure that we still do a hash check for other files on ims hit of Release
+    if grep -q '^Hit .* \(InRelease\|Release.gpg\)$' expected.output ; then
+	    $TEST aptget update -o Debug::Acquire::gpgv=1
+	    cp rootdir/tmp/${TEST}.output goodsign.output
+	    testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+	    testsuccess grep '^Got GOODSIG, key ID:GOODSIG' goodsign.output
+    fi
 
     # ensure no leftovers in partial
-    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+    testfailure ls 'rootdir/var/lib/apt/lists/partial/*'
 }
 
-msgmsg "InRelease"
-EXPECT="Hit http://localhost:8080 unstable InRelease
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
-Reading package lists..."
-# with InRelease
+msgmsg 'InRelease'
+EXPECT='Hit http://localhost:8080 unstable InRelease
+Reading package lists...'
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
-
-# with gzip
-echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
-msgmsg "Release/Release.gpg"
-# with Release/Release.gpg
-EXPECT="Ign http://localhost:8080 unstable InRelease
+msgmsg 'Release/Release.gpg'
+EXPECT='Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
 Hit http://localhost:8080 unstable Release.gpg
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
-Reading package lists..."
-
+Reading package lists...'
 find aptarchive -name 'InRelease' -delete
-
-echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
-
-echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
-# no Release.gpg or InRelease
-msgmsg "Release only"
+msgmsg 'Release only'
 EXPECT="Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
   404  Not Found
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
 Reading package lists...
 W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated."
+find aptarchive -name 'Release.gpg' -delete
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
+
+
+# make the release file old
+find aptarchive -name '*Release' -exec sed -i \
+	-e "s#^Date: .*\$#Date: $(date -d '-2 weeks' '+%a, %d %b %Y %H:%M:%S %Z')#" \
+	-e '/^Valid-Until: / d' -e "/^Date: / a\
+Valid-Until: $(date -d '-1 weeks' '+%a, %d %b %Y %H:%M:%S %Z')" '{}' \;
+signreleasefiles
+
+msgmsg 'expired InRelease'
+EXPECT='Hit http://localhost:8080 unstable InRelease
+E: Release file for http://localhost:8080/dists/unstable/InRelease is expired (invalid since). Updates for this repository will not be applied.'
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
+
+msgmsg 'expired Release/Release.gpg'
+EXPECT='Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Hit http://localhost:8080 unstable Release
+Hit http://localhost:8080 unstable Release.gpg
+E: Release file for http://localhost:8080/dists/unstable/Release.gpg is expired (invalid since). Updates for this repository will not be applied.'
+find aptarchive -name 'InRelease' -delete
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
 
+msgmsg 'expired Release only'
+EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Hit http://localhost:8080 unstable Release
+Ign http://localhost:8080 unstable Release.gpg
+  404  Not Found
+W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated.
+E: Release file for http://localhost:8080/dists/unstable/InRelease is expired (invalid since). Updates for this repository will not be applied."
 find aptarchive -name 'Release.gpg' -delete
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure' 'warning'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure' 'warning'
 
-echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
-runtest "true"
 
-echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
-runtest "true"
+msgmsg 'no Release at all'
+EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Ign http://localhost:8080 unstable Release
+  404  Not Found
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists...
+W: The repository 'http://localhost:8080 unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository."
+find aptarchive -name '*Release*' -delete
+echo 'Acquire::GzipIndexes "0";
+Acquire::PDiffs "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
+echo 'Acquire::GzipIndexes "1";
+Acquire::PDiffs "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
-- 
cgit v1.2.3


From 6bf93605fdb8e858d3f0a79a124c1d39f760094d Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Mon, 18 May 2015 22:15:06 +0200
Subject: treat older Release files than we already have as an IMSHit

Valid-Until protects us from long-living downgrade attacks, but not all
repositories have it and an attacker could still use older but still
valid files to downgrade us. While this makes it sounds like a security
improvement now, its a bit theoretical at best as an attacker with
capabilities to pull this off could just as well always keep us days
(but in the valid period) behind and always knows which state we have,
as we tell him with the If-Modified-Since header. This is also why this
is 'silently' ignored and treated as an IMSHit rather than screamed at
the user as this can at best be an annoyance for attackers.

An error here would 'regularily' be encountered by users by out-of-sync
mirrors serving a single run (e.g. load balancer) or in two consecutive
runs on the other hand, so it would just help teaching people ignore it.

That said, most of the code churn is caused by enforcing this additional
requirement. Crisscross from InRelease to Release.gpg is e.g. very
unlikely in practice, but if we would ignore it an attacker could
sidestep it this way.
---
 test/integration/test-apt-update-ims | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

(limited to 'test/integration/test-apt-update-ims')

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index f091bffaa..7385e701a 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -43,7 +43,7 @@ runtest() {
     testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
     # ensure that we still do a hash check for other files on ims hit of Release
-    if grep -q '^Hit .* \(InRelease\|Release.gpg\)$' expected.output ; then
+    if grep -q '^Hit .* InRelease$' expected.output || ! grep -q '^Ign .* Release\(\.gpg\)\?$' expected.output; then
 	    $TEST aptget update -o Debug::Acquire::gpgv=1
 	    cp rootdir/tmp/${TEST}.output goodsign.output
 	    testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
@@ -66,7 +66,6 @@ msgmsg 'Release/Release.gpg'
 EXPECT='Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
-Hit http://localhost:8080 unstable Release.gpg
 Reading package lists...'
 find aptarchive -name 'InRelease' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
@@ -81,7 +80,7 @@ Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
   404  Not Found
 Reading package lists...
-W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated."
+W: The data from 'http://localhost:8080 unstable Release' is not signed. Packages from that repository can not be authenticated."
 find aptarchive -name 'Release.gpg' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'warning'
@@ -108,8 +107,7 @@ msgmsg 'expired Release/Release.gpg'
 EXPECT='Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
-Hit http://localhost:8080 unstable Release.gpg
-E: Release file for http://localhost:8080/dists/unstable/Release.gpg is expired (invalid since). Updates for this repository will not be applied.'
+E: Release file for http://localhost:8080/dists/unstable/Release is expired (invalid since). Updates for this repository will not be applied.'
 find aptarchive -name 'InRelease' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'failure'
@@ -122,8 +120,8 @@ EXPECT="Ign http://localhost:8080 unstable InRelease
 Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
   404  Not Found
-W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated.
-E: Release file for http://localhost:8080/dists/unstable/InRelease is expired (invalid since). Updates for this repository will not be applied."
+W: The data from 'http://localhost:8080 unstable Release' is not signed. Packages from that repository can not be authenticated.
+E: Release file for http://localhost:8080/dists/unstable/Release is expired (invalid since). Updates for this repository will not be applied."
 find aptarchive -name 'Release.gpg' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'failure' 'warning'
-- 
cgit v1.2.3