Merge branch 'feature/altretries'

Generalizing the behaviour of retrying a download on the same server (if
enabled via options) as well as retrying a download via a different
alternative server from the acquire item responsible for deb files to
the handling of items in general so that all are effected.

4 files changed, 118 insertions, 6 deletions

diff --git a/test/integration/framework b/test/integration/framework
index bf6a8155e..f9d98835c 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1273,8 +1273,8 @@ webserverconfig() {
 		NOCHECK=true
 		shift
 	fi
-	local DOWNLOG='rootdir/tmp/download-testfile.log'
-	local STATUS='downloaded/webserverconfig.status'
+	local DOWNLOG="${TMPWORKINGDIRECTORY}/rootdir/tmp/download-testfile.log"
+	local STATUS="${TMPWORKINGDIRECTORY}/downloaded/webserverconfig.status"
 	rm -f "$STATUS" "$DOWNLOG"
 	# very very basic URI encoding
 	local URI
@@ -1298,7 +1298,7 @@ webserverconfig() {
 rewritesourceslist() {
 	local APTARCHIVE="file://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')"
 	local APTARCHIVE2="copy://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')"
-	for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
+	for LIST in $(find "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list.d/" -name 'apt-test-*.list'); do
 		sed -i $LIST -e "s#$APTARCHIVE#${1}#" -e "s#$APTARCHIVE2#${1}#" \
 			-e "s#http://[^@]*@\?localhost:${APTHTTPPORT}/\?#${1}#" \
 			-e "s#https://[^@]*@\?localhost:${APTHTTPSPORT}/\?#${1}#"
@@ -1937,8 +1937,8 @@ testaccessrights() {
 
 testwebserverlaststatuscode() {
 	msggroup 'testwebserverlaststatuscode'
-	local DOWNLOG='rootdir/tmp/webserverstatus-testfile.log'
-	local STATUS='downloaded/webserverstatus-statusfile.log'
+	local DOWNLOG="${TMPWORKINGDIRECTORY}/rootdir/tmp/webserverstatus-testfile.log"
+	local STATUS="${TMPWORKINGDIRECTORY}/downloaded/webserverstatus-statusfile.log"
 	rm -f "$DOWNLOG" "$STATUS"
 	msgtest 'Test last status code from the webserver was' "$1"
 	if downloadfile "http://localhost:${APTHTTPPORT}/_config/find/aptwebserver::last-status-code" "$STATUS" > "$DOWNLOG" && [ "$(cat "$STATUS")" = "$1" ]; then
diff --git a/test/integration/test-bug-869859-retry-downloads b/test/integration/test-bug-869859-retry-downloads
new file mode 100755
index 000000000..86203f794
--- /dev/null
+++ b/test/integration/test-bug-869859-retry-downloads
@@ -0,0 +1,55 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture 'amd64'
+
+buildsimplenativepackage 'testpkg' 'all' '1' 'stable'
+
+setupaptarchive --no-update
+changetowebserver
+testsuccess apt update
+
+cd downloaded
+testsuccess apt download testpkg
+testsuccess test -f testpkg_1_all.deb
+rm -f testpkg_1_all.deb
+
+msgmsg 'Fail after too many retries'
+webserverconfig 'aptwebserver::failrequest' '429'
+webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '99'
+testfailure apt download testpkg -o acquire::retries=3
+testfailure test -f testpkg_1_all.deb
+
+msgmsg 'Success in the third try'
+webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '2'
+testsuccess apt download testpkg -o acquire::retries=3
+testsuccess test -f testpkg_1_all.deb
+rm -f testpkg_1_all.deb
+
+msgmsg 'Do not try everything again, hard failures keep hard failures'
+webserverconfig 'aptwebserver::failrequest' '404'
+webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '2'
+testfailure apt download testpkg -o acquire::retries=3
+testfailure test -f testpkg_1_all.deb
+
+cat ../rootdir/etc/apt/sources.list.d/apt-test-*.list > ../rootdir/etc/apt/sources.list.d/00http-source.list
+changetohttpswebserver
+
+msgmsg 'Check download from alternative sources if first failed'
+webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '0'
+testsuccess apt update
+testsuccess apt download testpkg -o acquire::retries=0
+testsuccess test -f testpkg_1_all.deb
+rm -f testpkg_1_all.deb
+
+# we make the first source fail by disabling http support
+webserverconfig 'aptwebserver::support::http' 'false'
+testsuccess apt download testpkg -o acquire::retries=0
+cp ../rootdir/tmp/testsuccess.output alt.output
+testsuccess grep '^  400  Bad Request' alt.output
+testsuccess test -f testpkg_1_all.deb
+rm -f testpkg_1_all.deb
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 11f9b4b4f..4bc344178 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -82,7 +82,10 @@ static std::string httpcodeToStr(int const httpcode)			/*{{{*/
       case 504: return _config->Find("aptwebserver::httpcode::504", "504 Gateway Time-out");
       case 505: return _config->Find("aptwebserver::httpcode::505", "505 HTTP Version not supported");
    }
-   return "";
+   std::string codeconf, code;
+   strprintf(codeconf, "aptwebserver::httpcode::%i", httpcode);
+   strprintf(code, "%i Unknown HTTP code", httpcode);
+   return _config->Find(codeconf, code);
 }
 									/*}}}*/
 static bool chunkedTransferEncoding(std::list<std::string> const &headers) {
@@ -696,6 +699,18 @@ static void * handleClient(int const client, size_t const id)		/*{{{*/
 	    }
 	 }
 
+	 // automatic retry can be tested with this
+	 {
+	    int failrequests = _config->FindI("aptwebserver::failrequest::" + filename, 0);
+	    if (failrequests != 0)
+	    {
+	       --failrequests;
+	       _config->Set(("aptwebserver::failrequest::" + filename).c_str(), failrequests);
+	       sendError(log, client, _config->FindI("aptwebserver::failrequest", 400), *m, sendContent, "Server is configured to fail this file.", headers);
+	       continue;
+	    }
+	 }
+
 	 // deal with the request
 	 unsigned int const httpsport = _config->FindI("aptwebserver::port::https", 4433);
 	 std::string hosthttpsport;
diff --git a/test/libapt/strutil_test.cc b/test/libapt/strutil_test.cc
index 9c192a58b..f531c2bf9 100644
--- a/test/libapt/strutil_test.cc
+++ b/test/libapt/strutil_test.cc
@@ -319,3 +319,45 @@ TEST(StrUtilTest,RFC1123StrToTime)
    EXPECT_FALSE(RFC1123StrToTime("Sunday, 06-Nov-94 08:49:37 -0100", t));
    EXPECT_FALSE(RFC1123StrToTime("Sunday, 06-Nov-94 08:49:37 -0.1", t));
 }
+TEST(StrUtilTest, LookupTag)
+{
+   EXPECT_EQ("", LookupTag("", "Field", ""));
+   EXPECT_EQ("", LookupTag("", "Field", nullptr));
+   EXPECT_EQ("default", LookupTag("", "Field", "default"));
+   EXPECT_EQ("default", LookupTag("Field1: yes", "Field", "default"));
+   EXPECT_EQ("default", LookupTag("Fiel: yes", "Field", "default"));
+   EXPECT_EQ("default", LookupTag("Fiel d: yes", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("Field: foo", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("Field: foo\n", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("\nField: foo\n", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("Field:foo", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("Field:foo\n", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("\nField:foo\n", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("Field:\tfoo\n", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("Field:  foo  \t", "Field", "default"));
+   EXPECT_EQ("foo", LookupTag("Field:  foo  \t\n", "Field", "default"));
+   EXPECT_EQ("Field : yes", LookupTag("Field:  Field : yes  \t\n", "Field", "default"));
+   EXPECT_EQ("Field : yes", LookupTag("Field:\n Field : yes  \t\n", "Field", "default"));
+   EXPECT_EQ("Field : yes", LookupTag("Foo: bar\nField:  Field : yes  \t\n", "Field", "default"));
+   EXPECT_EQ("line1\nline2", LookupTag("Multi: line1\n line2", "Multi", "default"));
+   EXPECT_EQ("line1\nline2", LookupTag("Multi: line1\n line2\n", "Multi", "default"));
+   EXPECT_EQ("line1\nline2", LookupTag("Multi:\n line1\n line2\n", "Multi", "default"));
+   EXPECT_EQ("line1\n\nline2", LookupTag("Multi:\n line1\n .\n line2\n", "Multi", "default"));
+   EXPECT_EQ("line1\na\nline2", LookupTag("Multi:\n line1\n a\n line2\n", "Multi", "default"));
+   EXPECT_EQ("line1\nfoo\nline2", LookupTag("Multi:\n line1\n foo\n line2\n", "Multi", "default"));
+   EXPECT_EQ("line1\n  line2", LookupTag("Multi: line1\n   line2", "Multi", "default"));
+   EXPECT_EQ("  line1\n \t line2", LookupTag("Multi:\t \n   line1\n  \t line2\n", "Multi", "default"));
+   EXPECT_EQ("  line1\n\n\n \t line2", LookupTag("Multi:\t \n   line1\n .\n .   \n  \t line2\n", "Multi", "default"));
+
+   std::string const msg =
+      "Field1: Value1\nField2:Value2\nField3:\t   Value3\n"
+      "Multi-Field1: Line1\n Line2\nMulti-Field2:\n Line1\n Line2\n"
+      "Field4: Value4\nField5:Value5";
+   EXPECT_EQ("Value1", LookupTag(msg, "Field1", ""));
+   EXPECT_EQ("Value2", LookupTag(msg, "Field2", ""));
+   EXPECT_EQ("Value3", LookupTag(msg, "Field3", ""));
+   EXPECT_EQ("Line1\nLine2", LookupTag(msg, "Multi-Field1", ""));
+   EXPECT_EQ("Line1\nLine2", LookupTag(msg, "Multi-Field2", ""));
+   EXPECT_EQ("Value4", LookupTag(msg, "Field4", ""));
+   EXPECT_EQ("Value5", LookupTag(msg, "Field5", ""));
+}