From 07cd99066c30e70a9f41851c80e7c51f4e507163 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 17 Jul 2017 10:52:09 +0200 Subject: support multiline values in LookupTag LookupTag is a little helper to deal with rfc822-style strings we use in apt e.g. to pass acquire messages around for cases in which our usual rfc822 parser is too heavy. All the fields it had to deal with so far were single line, but if they aren't it should really produce the right output and not just return the first line. Error messages are a prime candidate for becoming multiline as at the moment they are stripped of potential newlines due to the previous insufficiency of LookupTag. --- test/libapt/strutil_test.cc | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'test') diff --git a/test/libapt/strutil_test.cc b/test/libapt/strutil_test.cc index 9c192a58b..f531c2bf9 100644 --- a/test/libapt/strutil_test.cc +++ b/test/libapt/strutil_test.cc @@ -319,3 +319,45 @@ TEST(StrUtilTest,RFC1123StrToTime) EXPECT_FALSE(RFC1123StrToTime("Sunday, 06-Nov-94 08:49:37 -0100", t)); EXPECT_FALSE(RFC1123StrToTime("Sunday, 06-Nov-94 08:49:37 -0.1", t)); } +TEST(StrUtilTest, LookupTag) +{ + EXPECT_EQ("", LookupTag("", "Field", "")); + EXPECT_EQ("", LookupTag("", "Field", nullptr)); + EXPECT_EQ("default", LookupTag("", "Field", "default")); + EXPECT_EQ("default", LookupTag("Field1: yes", "Field", "default")); + EXPECT_EQ("default", LookupTag("Fiel: yes", "Field", "default")); + EXPECT_EQ("default", LookupTag("Fiel d: yes", "Field", "default")); + EXPECT_EQ("foo", LookupTag("Field: foo", "Field", "default")); + EXPECT_EQ("foo", LookupTag("Field: foo\n", "Field", "default")); + EXPECT_EQ("foo", LookupTag("\nField: foo\n", "Field", "default")); + EXPECT_EQ("foo", LookupTag("Field:foo", "Field", "default")); + EXPECT_EQ("foo", LookupTag("Field:foo\n", "Field", "default")); + EXPECT_EQ("foo", LookupTag("\nField:foo\n", "Field", "default")); + EXPECT_EQ("foo", LookupTag("Field:\tfoo\n", "Field", "default")); + EXPECT_EQ("foo", LookupTag("Field: foo \t", "Field", "default")); + EXPECT_EQ("foo", LookupTag("Field: foo \t\n", "Field", "default")); + EXPECT_EQ("Field : yes", LookupTag("Field: Field : yes \t\n", "Field", "default")); + EXPECT_EQ("Field : yes", LookupTag("Field:\n Field : yes \t\n", "Field", "default")); + EXPECT_EQ("Field : yes", LookupTag("Foo: bar\nField: Field : yes \t\n", "Field", "default")); + EXPECT_EQ("line1\nline2", LookupTag("Multi: line1\n line2", "Multi", "default")); + EXPECT_EQ("line1\nline2", LookupTag("Multi: line1\n line2\n", "Multi", "default")); + EXPECT_EQ("line1\nline2", LookupTag("Multi:\n line1\n line2\n", "Multi", "default")); + EXPECT_EQ("line1\n\nline2", LookupTag("Multi:\n line1\n .\n line2\n", "Multi", "default")); + EXPECT_EQ("line1\na\nline2", LookupTag("Multi:\n line1\n a\n line2\n", "Multi", "default")); + EXPECT_EQ("line1\nfoo\nline2", LookupTag("Multi:\n line1\n foo\n line2\n", "Multi", "default")); + EXPECT_EQ("line1\n line2", LookupTag("Multi: line1\n line2", "Multi", "default")); + EXPECT_EQ(" line1\n \t line2", LookupTag("Multi:\t \n line1\n \t line2\n", "Multi", "default")); + EXPECT_EQ(" line1\n\n\n \t line2", LookupTag("Multi:\t \n line1\n .\n . \n \t line2\n", "Multi", "default")); + + std::string const msg = + "Field1: Value1\nField2:Value2\nField3:\t Value3\n" + "Multi-Field1: Line1\n Line2\nMulti-Field2:\n Line1\n Line2\n" + "Field4: Value4\nField5:Value5"; + EXPECT_EQ("Value1", LookupTag(msg, "Field1", "")); + EXPECT_EQ("Value2", LookupTag(msg, "Field2", "")); + EXPECT_EQ("Value3", LookupTag(msg, "Field3", "")); + EXPECT_EQ("Line1\nLine2", LookupTag(msg, "Multi-Field1", "")); + EXPECT_EQ("Line1\nLine2", LookupTag(msg, "Multi-Field2", "")); + EXPECT_EQ("Value4", LookupTag(msg, "Field4", "")); + EXPECT_EQ("Value5", LookupTag(msg, "Field5", "")); +} -- cgit v1.2.3 From dff555d40bb9776b5b809e06527e46b15e78736c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 26 Oct 2017 01:09:48 +0200 Subject: implement Acquire::Retries support for all items Moving the Retry-implementation from individual items to the worker implementation not only gives every file retry capability instead of just a selected few but also avoids needing to implement it in each item (incorrectly). --- test/integration/framework | 8 ++--- test/integration/test-bug-869859-retry-downloads | 37 ++++++++++++++++++++++++ test/interactive-helper/aptwebserver.cc | 17 ++++++++++- 3 files changed, 57 insertions(+), 5 deletions(-) create mode 100755 test/integration/test-bug-869859-retry-downloads (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index bf6a8155e..bff6c0e65 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1273,8 +1273,8 @@ webserverconfig() { NOCHECK=true shift fi - local DOWNLOG='rootdir/tmp/download-testfile.log' - local STATUS='downloaded/webserverconfig.status' + local DOWNLOG="${TMPWORKINGDIRECTORY}/rootdir/tmp/download-testfile.log" + local STATUS="${TMPWORKINGDIRECTORY}/downloaded/webserverconfig.status" rm -f "$STATUS" "$DOWNLOG" # very very basic URI encoding local URI @@ -1937,8 +1937,8 @@ testaccessrights() { testwebserverlaststatuscode() { msggroup 'testwebserverlaststatuscode' - local DOWNLOG='rootdir/tmp/webserverstatus-testfile.log' - local STATUS='downloaded/webserverstatus-statusfile.log' + local DOWNLOG="${TMPWORKINGDIRECTORY}/rootdir/tmp/webserverstatus-testfile.log" + local STATUS="${TMPWORKINGDIRECTORY}/downloaded/webserverstatus-statusfile.log" rm -f "$DOWNLOG" "$STATUS" msgtest 'Test last status code from the webserver was' "$1" if downloadfile "http://localhost:${APTHTTPPORT}/_config/find/aptwebserver::last-status-code" "$STATUS" > "$DOWNLOG" && [ "$(cat "$STATUS")" = "$1" ]; then diff --git a/test/integration/test-bug-869859-retry-downloads b/test/integration/test-bug-869859-retry-downloads new file mode 100755 index 000000000..a62429a53 --- /dev/null +++ b/test/integration/test-bug-869859-retry-downloads @@ -0,0 +1,37 @@ +#!/bin/sh +set -e + +TESTDIR="$(readlink -f "$(dirname "$0")")" +. "$TESTDIR/framework" + +setupenvironment +configarchitecture 'amd64' + +buildsimplenativepackage 'testpkg' 'all' '1' 'stable' + +setupaptarchive --no-update +changetowebserver +testsuccess apt update + +cd downloaded +testsuccess apt download testpkg +testsuccess test -f testpkg_1_all.deb +rm -f testpkg_1_all.deb + +msgmsg 'Fail after too many retries' +webserverconfig 'aptwebserver::failrequest' '429' +webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '99' +testfailure apt download testpkg -o acquire::retries=3 +testfailure test -f testpkg_1_all.deb + +msgmsg 'Success in the third try' +webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '2' +testsuccess apt download testpkg -o acquire::retries=3 +testsuccess test -f testpkg_1_all.deb +rm -f testpkg_1_all.deb + +msgmsg 'Do not try everything again, hard failures keep hard failures' +webserverconfig 'aptwebserver::failrequest' '404' +webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '2' +testfailure apt download testpkg -o acquire::retries=3 +testfailure test -f testpkg_1_all.deb diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 11f9b4b4f..4bc344178 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -82,7 +82,10 @@ static std::string httpcodeToStr(int const httpcode) /*{{{*/ case 504: return _config->Find("aptwebserver::httpcode::504", "504 Gateway Time-out"); case 505: return _config->Find("aptwebserver::httpcode::505", "505 HTTP Version not supported"); } - return ""; + std::string codeconf, code; + strprintf(codeconf, "aptwebserver::httpcode::%i", httpcode); + strprintf(code, "%i Unknown HTTP code", httpcode); + return _config->Find(codeconf, code); } /*}}}*/ static bool chunkedTransferEncoding(std::list const &headers) { @@ -696,6 +699,18 @@ static void * handleClient(int const client, size_t const id) /*{{{*/ } } + // automatic retry can be tested with this + { + int failrequests = _config->FindI("aptwebserver::failrequest::" + filename, 0); + if (failrequests != 0) + { + --failrequests; + _config->Set(("aptwebserver::failrequest::" + filename).c_str(), failrequests); + sendError(log, client, _config->FindI("aptwebserver::failrequest", 400), *m, sendContent, "Server is configured to fail this file.", headers); + continue; + } + } + // deal with the request unsigned int const httpsport = _config->FindI("aptwebserver::port::https", 4433); std::string hosthttpsport; -- cgit v1.2.3 From 355e1aceac1dd05c4c7daf3420b09bd860fd169d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Oct 2017 19:09:45 +0200 Subject: implement fallback to alternative URIs for all items For deb files we always supported falling back from one server to the other if one failed to download the deb, but that was hardwired in the handling of this specific item. Moving this alongside the retry infrastructure we can implement it for all items and allow methods to use this as well by providing additional URIs in a redirect. --- test/integration/framework | 2 +- test/integration/test-bug-869859-retry-downloads | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index bff6c0e65..f9d98835c 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1298,7 +1298,7 @@ webserverconfig() { rewritesourceslist() { local APTARCHIVE="file://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')" local APTARCHIVE2="copy://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')" - for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do + for LIST in $(find "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list.d/" -name 'apt-test-*.list'); do sed -i $LIST -e "s#$APTARCHIVE#${1}#" -e "s#$APTARCHIVE2#${1}#" \ -e "s#http://[^@]*@\?localhost:${APTHTTPPORT}/\?#${1}#" \ -e "s#https://[^@]*@\?localhost:${APTHTTPSPORT}/\?#${1}#" diff --git a/test/integration/test-bug-869859-retry-downloads b/test/integration/test-bug-869859-retry-downloads index a62429a53..86203f794 100755 --- a/test/integration/test-bug-869859-retry-downloads +++ b/test/integration/test-bug-869859-retry-downloads @@ -35,3 +35,21 @@ webserverconfig 'aptwebserver::failrequest' '404' webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '2' testfailure apt download testpkg -o acquire::retries=3 testfailure test -f testpkg_1_all.deb + +cat ../rootdir/etc/apt/sources.list.d/apt-test-*.list > ../rootdir/etc/apt/sources.list.d/00http-source.list +changetohttpswebserver + +msgmsg 'Check download from alternative sources if first failed' +webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '0' +testsuccess apt update +testsuccess apt download testpkg -o acquire::retries=0 +testsuccess test -f testpkg_1_all.deb +rm -f testpkg_1_all.deb + +# we make the first source fail by disabling http support +webserverconfig 'aptwebserver::support::http' 'false' +testsuccess apt download testpkg -o acquire::retries=0 +cp ../rootdir/tmp/testsuccess.output alt.output +testsuccess grep '^ 400 Bad Request' alt.output +testsuccess test -f testpkg_1_all.deb +rm -f testpkg_1_all.deb -- cgit v1.2.3