From 448c38bdcd72b52f11ec5f326f822cf57653f81c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 6 Jun 2015 12:28:00 +0200 Subject: rework hashsum verification in the acquire system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having every item having its own code to verify the file(s) it handles is an errorprune process and easy to break, especially if items move through various stages (download, uncompress, patching, …). With a giant rework we centralize (most of) the verification to have a better enforcement rate and (hopefully) less chance for bugs, but it breaks the ABI bigtime in exchange – and as we break it anyway, it is broken even harder. It shouldn't effect most frontends as they don't deal with the acquire system at all or implement their own items, but some do and will need to be patched (might be an opportunity to use apt on-board material). The theory is simple: Items implement methods to decide if hashes need to be checked (in this stage) and to return the expected hashes for this item (in this stage). The verification itself is done in worker message passing which has the benefit that a hashsum error is now a proper error for the acquire system rather than a Done() which is later revised to a Failed(). --- test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum') diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum index ec74a750b..555d8fcaa 100755 --- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -176,7 +176,11 @@ testmismatch() { Building dependency tree... Need to get 6 B of source archives. Get:1 http://localhost:8080/ $1 1.0 (dsc) [3 B] +Err http://localhost:8080/ $1 1.0 (dsc) + Hash Sum mismatch Get:2 http://localhost:8080/ $1 1.0 (tar) [3 B] +Err http://localhost:8080/ $1 1.0 (tar) + Hash Sum mismatch E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch E: Failed to fetch http://localhost:8080/${1}_1.0.tar.gz Hash Sum mismatch @@ -238,6 +242,8 @@ Building dependency tree... Need to get 6 B of source archives. Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar) [3 B] Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) [3 B] +Err http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) + Hash Sum mismatch E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad @@ -247,6 +253,8 @@ testfailureequal 'Reading package lists... Building dependency tree... Need to get 6 B of source archives. Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) [3 B] +Err http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) + Hash Sum mismatch Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc) [3 B] E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch -- cgit v1.2.3 From 1da3b7b8e15b642135b54684e70a0c271471f07a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 11 Jun 2015 10:56:31 +0200 Subject: show URI.Path in all acquire item descriptions It is a rather strange sight that index items use SiteOnly which strips the Path, while e.g. deb files are downloaded with NoUserPassword which does not. Important to note here is that for the file transport Path is pretty important as there is no Host which would be displayed by Site, which always resulted in "interesting" unspecific errors for "file:". Adding a 'middle' ground between the two which does show the Path but potentially modifies it (it strips a pending / at the end if existing) solves this "file:" issue, syncs the output and in the end helps to identify which file is meant exactly in progress output and co as a single site can have multiple repositories in different paths. --- .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 28 +++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum') diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum index 555d8fcaa..6abb5d12a 100755 --- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -148,8 +148,8 @@ testok() { testsuccessequal "Reading package lists... Building dependency tree... Need to get 6 B of source archives. -Get:1 http://localhost:8080/ $1 1.0 (dsc) [3 B] -Get:2 http://localhost:8080/ $1 1.0 (tar) [3 B] +Get:1 http://localhost:8080 $1 1.0 (dsc) [3 B] +Get:2 http://localhost:8080 $1 1.0 (tar) [3 B] Download complete and in download only mode" aptget source -d "$@" msgtest 'Files were successfully downloaded for' "$1" testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz @@ -175,11 +175,11 @@ testmismatch() { testfailureequal "Reading package lists... Building dependency tree... Need to get 6 B of source archives. -Get:1 http://localhost:8080/ $1 1.0 (dsc) [3 B] -Err http://localhost:8080/ $1 1.0 (dsc) +Get:1 http://localhost:8080 $1 1.0 (dsc) [3 B] +Err http://localhost:8080 $1 1.0 (dsc) Hash Sum mismatch -Get:2 http://localhost:8080/ $1 1.0 (tar) [3 B] -Err http://localhost:8080/ $1 1.0 (tar) +Get:2 http://localhost:8080 $1 1.0 (tar) [3 B] +Err http://localhost:8080 $1 1.0 (tar) Hash Sum mismatch E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch @@ -203,8 +203,8 @@ Download complete and in download only mode" aptget source -d "$@" -o Acquire::F testsuccessequal "Reading package lists... Building dependency tree... Need to get 6 B of source archives. -Get:1 http://localhost:8080/ $1 1.0 (dsc) [3 B] -Get:2 http://localhost:8080/ $1 1.0 (tar) [3 B] +Get:1 http://localhost:8080 $1 1.0 (dsc) [3 B] +Get:2 http://localhost:8080 $1 1.0 (tar) [3 B] Download complete and in download only mode" aptget source --allow-unauthenticated -d "$@" -o Acquire::ForceHash=ROT26 msgtest 'Files were downloaded unauthenticated as user allowed it' "$1" testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz @@ -240,9 +240,9 @@ testok pkg-mixed-ok testfailureequal 'Reading package lists... Building dependency tree... Need to get 6 B of source archives. -Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar) [3 B] -Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) [3 B] -Err http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) +Get:1 http://localhost:8080 pkg-mixed-sha1-bad 1.0 (tar) [3 B] +Get:2 http://localhost:8080 pkg-mixed-sha1-bad 1.0 (dsc) [3 B] +Err http://localhost:8080 pkg-mixed-sha1-bad 1.0 (dsc) Hash Sum mismatch E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch @@ -252,10 +252,10 @@ testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-ba testfailureequal 'Reading package lists... Building dependency tree... Need to get 6 B of source archives. -Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) [3 B] -Err http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) +Get:1 http://localhost:8080 pkg-mixed-sha2-bad 1.0 (tar) [3 B] +Err http://localhost:8080 pkg-mixed-sha2-bad 1.0 (tar) Hash Sum mismatch -Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc) [3 B] +Get:2 http://localhost:8080 pkg-mixed-sha2-bad 1.0 (dsc) [3 B] E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha2-bad -- cgit v1.2.3 From 1eb1836f4b5397497bd34f0cf516e6e4e73117bf Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 15 Jun 2015 16:41:43 +0200 Subject: show item ID in Hit, Ign and Err lines as well Again, consistency is the main sellingpoint here, but this way it is now also easier to explain that some files move through different stages and lines are printed for them hence multiple times: That is a bit hard to believe if the number is changing all the time, but now that it keeps consistent. --- test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum') diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum index 6abb5d12a..48a7f0562 100755 --- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -176,10 +176,10 @@ testmismatch() { Building dependency tree... Need to get 6 B of source archives. Get:1 http://localhost:8080 $1 1.0 (dsc) [3 B] -Err http://localhost:8080 $1 1.0 (dsc) +Err:1 http://localhost:8080 $1 1.0 (dsc) Hash Sum mismatch Get:2 http://localhost:8080 $1 1.0 (tar) [3 B] -Err http://localhost:8080 $1 1.0 (tar) +Err:2 http://localhost:8080 $1 1.0 (tar) Hash Sum mismatch E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch @@ -242,7 +242,7 @@ Building dependency tree... Need to get 6 B of source archives. Get:1 http://localhost:8080 pkg-mixed-sha1-bad 1.0 (tar) [3 B] Get:2 http://localhost:8080 pkg-mixed-sha1-bad 1.0 (dsc) [3 B] -Err http://localhost:8080 pkg-mixed-sha1-bad 1.0 (dsc) +Err:2 http://localhost:8080 pkg-mixed-sha1-bad 1.0 (dsc) Hash Sum mismatch E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch @@ -253,7 +253,7 @@ testfailureequal 'Reading package lists... Building dependency tree... Need to get 6 B of source archives. Get:1 http://localhost:8080 pkg-mixed-sha2-bad 1.0 (tar) [3 B] -Err http://localhost:8080 pkg-mixed-sha2-bad 1.0 (tar) +Err:1 http://localhost:8080 pkg-mixed-sha2-bad 1.0 (tar) Hash Sum mismatch Get:2 http://localhost:8080 pkg-mixed-sha2-bad 1.0 (dsc) [3 B] E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch -- cgit v1.2.3