From 1e0f0f28e1025f42a8172eb72f3e87984eb2b939 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 9 Jun 2015 11:59:22 +0200 Subject: configureable acquire targets to download additional files First pass at making the acquire system capable of downloading files based on configuration rather than hardcoded entries. It is now possible to instruct 'deb' and 'deb-src' sources.list lines to download more than just Packages/Translation-* and Sources files. Details on how to do that can be found in the included documentation file. --- test/integration/test-apt-acquire-additional-files | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 test/integration/test-apt-acquire-additional-files (limited to 'test/integration/test-apt-acquire-additional-files') diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files new file mode 100755 index 000000000..150a50980 --- /dev/null +++ b/test/integration/test-apt-acquire-additional-files @@ -0,0 +1,54 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' +configcompression '.' 'gz' + +buildsimplenativepackage 'foo' 'amd64' '1' 'unstable' + +setupaptarchive --no-update +changetowebserver + +testsuccessequal "Get:1 http://localhost:8080 unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B] +Get:2 http://localhost:8080 unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B] +Get:3 http://localhost:8080 unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B] +Get:4 http://localhost:8080 unstable/main Translation-en [$(stat -c%s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B] +Reading package lists..." aptget update + +testempty find rootdir/var/lib/apt/lists -name '*Contents*' + +cat > rootdir/etc/apt/apt.conf.d/content-target.conf < rootdir/etc/apt/apt.conf.d/content-target.conf < Date: Wed, 10 Jun 2015 19:22:41 +0200 Subject: abstract the code to iterate over all targets a bit We have two places in the code which need to iterate over targets and do certain things with it. The first one is actually creating these targets for download and the second instance pepares certain targets for reading. Git-Dch: Ignore --- test/integration/test-apt-acquire-additional-files | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'test/integration/test-apt-acquire-additional-files') diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index 150a50980..0e59233e9 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -6,6 +6,9 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'amd64' + +# note that in --print-uri we talk about .bz2 because that is the default. +# This doesn't mean it is actually attempt to download it. configcompression '.' 'gz' buildsimplenativepackage 'foo' 'amd64' '1' 'unstable' @@ -13,6 +16,11 @@ buildsimplenativepackage 'foo' 'amd64' '1' 'unstable' setupaptarchive --no-update changetowebserver +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris + testsuccessequal "Get:1 http://localhost:8080 unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B] Get:2 http://localhost:8080 unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B] Get:3 http://localhost:8080 unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B] @@ -29,6 +37,12 @@ APT::Acquire::Targets::deb::Contents { }; EOF +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 +'http://localhost:8080/dists/unstable/main/Contents-amd64.bz2' localhost:8080_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris + testsuccessequal "Hit http://localhost:8080 unstable InRelease Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] Reading package lists..." aptget update @@ -46,9 +60,28 @@ APT::Acquire::Targets::deb::Contents { }; EOF +# the last line is utter bogus of course, but how should apt know… +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 +'http://localhost:8080/dists/unstable/main/Contents-amd64.gz.bz2' localhost:8080_dists_unstable_main_Contents-amd64.gz 0 " aptget update --print-uris + testsuccessequal "Hit http://localhost:8080 unstable InRelease Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] Reading package lists..." aptget update testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*' testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz' + +rm -f rootdir/etc/apt/apt.conf.d/content-target.conf + +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris + +testsuccessequal "Hit http://localhost:8080 unstable InRelease +Reading package lists..." aptget update + +testempty find rootdir/var/lib/apt/lists -name '*Contents*' -- cgit v1.2.3 From d3a869e35503638e3483228fbfc95b7143568ad0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 10 Jun 2015 21:24:47 +0200 Subject: store all targets data in IndexTarget struct We still need an API for the targets, so slowly prepare the IndexTargets to let them take this job. Git-Dch: Ignore --- test/integration/test-apt-acquire-additional-files | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/integration/test-apt-acquire-additional-files') diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index 0e59233e9..4a6845f40 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -31,7 +31,7 @@ testempty find rootdir/var/lib/apt/lists -name '*Contents*' cat > rootdir/etc/apt/apt.conf.d/content-target.conf < rootdir/etc/apt/apt.conf.d/content-target.conf < Date: Thu, 11 Jun 2015 16:40:45 +0200 Subject: implement 'apt-get files' to access index targets Downloading additional files is only half the job. We still need a way to allow external tools to know where the files are they requested for download given that we don't want them to choose their own location. 'apt-get files' is our answer to this showing by default in a deb822 format information about each IndexTarget with the potential to filter the records based on lines and an option to change the output format. The command serves also as an example on how to get to this information via libapt. --- test/integration/test-apt-acquire-additional-files | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test/integration/test-apt-acquire-additional-files') diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index 4a6845f40..971cfac2e 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -48,6 +48,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive Reading package lists..." aptget update testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*' +testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --format '$(FILENAME)' 'Created-By: Contents' testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64' # no automatic uncompress based on the name please, @@ -72,6 +73,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarch Reading package lists..." aptget update testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*' +testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents' testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz' rm -f rootdir/etc/apt/apt.conf.d/content-target.conf -- cgit v1.2.3 From 3fd89e62e985c89b1f9a545ab72c20987b756aff Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 12 Jun 2015 12:06:29 +0200 Subject: implement default apt-get file --release-info mode Selecting targets based on the Release they belong to isn't to unrealistic. In fact, it is assumed to be the most used case so it is made the default especially as this allows to bundle another thing we have to be careful with: Filenames and only showing targets we have acquired. Closes: 752702 --- test/integration/test-apt-acquire-additional-files | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'test/integration/test-apt-acquire-additional-files') diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index 971cfac2e..ffb9f3135 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -37,6 +37,12 @@ APT::Acquire::Targets::deb::Contents { }; EOF +testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --no-release-info --format '$(FILENAME)' 'Created-By: Contents' +testempty aptget files --format '$(FILENAME)' 'Created-By: Contents' +# lets fake the existence of a compressed Contents file +touch ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz +testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents' + testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 -- cgit v1.2.3 From 1eb1836f4b5397497bd34f0cf516e6e4e73117bf Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 15 Jun 2015 16:41:43 +0200 Subject: show item ID in Hit, Ign and Err lines as well Again, consistency is the main sellingpoint here, but this way it is now also easier to explain that some files move through different stages and lines are printed for them hence multiple times: That is a bit hard to believe if the number is changing all the time, but now that it keeps consistent. --- test/integration/test-apt-acquire-additional-files | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'test/integration/test-apt-acquire-additional-files') diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index ffb9f3135..3465c0a16 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -49,8 +49,8 @@ testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists 'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 'http://localhost:8080/dists/unstable/main/Contents-amd64.bz2' localhost:8080_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris -testsuccessequal "Hit http://localhost:8080 unstable InRelease -Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] +testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease +Get:2 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] Reading package lists..." aptget update testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*' @@ -74,8 +74,8 @@ testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists 'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 'http://localhost:8080/dists/unstable/main/Contents-amd64.gz.bz2' localhost:8080_dists_unstable_main_Contents-amd64.gz 0 " aptget update --print-uris -testsuccessequal "Hit http://localhost:8080 unstable InRelease -Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] +testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease +Get:2 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] Reading package lists..." aptget update testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*' @@ -89,7 +89,7 @@ testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris -testsuccessequal "Hit http://localhost:8080 unstable InRelease +testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease Reading package lists..." aptget update testempty find rootdir/var/lib/apt/lists -name '*Contents*' -- cgit v1.2.3