From 721b05b8501b3ffffcaef4d90b4f24e2a72b0fef Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jul 2012 15:28:28 +0200 Subject: rename the paywall testcase and reformat it a bit --- .../test-ubuntu-bug-346386-apt-get-update-paywall | 47 ++++++++++++++++++++++ test/integration/test-ubuntu-bug346386 | 47 ---------------------- 2 files changed, 47 insertions(+), 47 deletions(-) create mode 100755 test/integration/test-ubuntu-bug-346386-apt-get-update-paywall delete mode 100755 test/integration/test-ubuntu-bug346386 (limited to 'test') diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall new file mode 100755 index 000000000..25cccf067 --- /dev/null +++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall @@ -0,0 +1,47 @@ +#!/bin/sh +set -e + +ensure_n_canary_strings_in_dir() { + DIR=$1 + CANARY_STRING=$2 + EXPECTED_N=$3 + + msgtest "Testing for $EXPECTED_N canary strings '$CANARY_STRING' in in" "$DIR" + + N=$(grep "$CANARY_STRING" $DIR/* 2>/dev/null |wc -l ) + if [ "$N" = "$EXPECTED_N" ]; then + msgpass + return 0 + else + msgfail "Expected $EXPECTED_N canaries, got $N" + return 1 + fi +} + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'native' + +insertpackage 'unstable' 'unrelated' 'all' '1.0' 'stable' + +setupaptarchive +changetowebserver --simulate-paywall + +rm -rf rootdir/var/lib/apt/lists +msgtest 'excpected failure of' 'apt-get update' +aptget update -qq 2>/dev/null && msgfail || msgpass + +ensure_n_canary_strings_in_dir rootdir/var/lib/apt/lists/ 'ni ni ni' 0 +testequal 'partial' ls rootdir/var/lib/apt/lists/ + +# again, this time with pre-existing files valid data +for f in Release Release.gpg main_binary-amd64_Packages stable_main_source_Sources; do + echo "canary" > rootdir/var/lib/apt/lists/localhost:8080_dists_stable_${f} +done + +# this will fail, the important part is that the canaries remain +msgtest 'excpected failure of' 'apt-get update' +aptget update -qq 2>/dev/null && msgfail || msgpass +ensure_n_canary_strings_in_dir rootdir/var/lib/apt/lists/ 'canary' 4 diff --git a/test/integration/test-ubuntu-bug346386 b/test/integration/test-ubuntu-bug346386 deleted file mode 100755 index 1fbfb5ca4..000000000 --- a/test/integration/test-ubuntu-bug346386 +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -set -e - -ensure_n_canary_strings_in_dir() { - DIR=$1 - CANARY_STRING=$2 - EXPECTED_N=$3 - - msgtest "Testing for $EXPECTED_N canary strings '$CANARY_STRING' in in $DIR" - - - N=$(grep "$CANARY_STRING" $DIR/* 2>/dev/null |wc -l ) - if [ "$N" = "$EXPECTED_N" ]; then - msgpass - return 0 - else - msgfail "Expected $EXPECTED_N canaries, got $N" - return 1 - fi -} - -TESTDIR=$(readlink -f $(dirname $0)) -. $TESTDIR/framework - -setupenvironment -configarchitecture 'amd64' - -buildsimplenativepackage 'apt' 'all' '1.0' 'stable' - -setupaptarchive -changetowebserver --simulate-paywall - -rm -rf rootdir/var/lib/apt/lists -if aptget update -qq 2>/dev/null; then - msgfail "excpected apt-get update failure" -fi -ensure_n_canary_strings_in_dir rootdir/var/lib/apt/lists/ "ni ni ni" 0 -testequal 'partial' ls rootdir/var/lib/apt/lists/ - -# again, this time with pre-existing files valid data -for f in Release Release.gpg main_binary-amd64_Packages stable_main_source_Sources; do - echo "canary" > rootdir/var/lib/apt/lists/localhost:8080_dists_stable_${f} -done -# this will fail, the important part is that the canaries remain -aptget update -qq 2>/dev/null || true -ensure_n_canary_strings_in_dir rootdir/var/lib/apt/lists/ "canary" 4 - -- cgit v1.2.3 From dc57a59b1e7a904ad7263245fa2b1d6a5a55655e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jul 2012 18:00:11 +0200 Subject: reformat the aptwebserver code to look more like the rest of APT --- test/interactive-helper/aptwebserver.cc | 89 ++++++++++++++++----------------- 1 file changed, 44 insertions(+), 45 deletions(-) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 8fbb9eab9..97a298c70 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -18,7 +18,7 @@ #include #include -char const * const httpcodeToStr(int httpcode) { /*{{{*/ +char const * const httpcodeToStr(int const httpcode) { /*{{{*/ switch (httpcode) { // Informational 1xx case 100: return "100 Continue"; @@ -67,9 +67,9 @@ char const * const httpcodeToStr(int httpcode) { /*{{{*/ case 505: return "HTTP Version not supported"; } return NULL; -} /*}}}*/ - -void addFileHeaders(std::list &headers, FileFd &data) { /*{{{*/ +} + /*}}}*/ +void addFileHeaders(std::list &headers, FileFd &data) { /*{{{*/ std::ostringstream contentlength; contentlength << "Content-Length: " << data.FileSize(); headers.push_back(contentlength.str()); @@ -77,15 +77,15 @@ void addFileHeaders(std::list &headers, FileFd &data) { /*{{{*/ std::string lastmodified("Last-Modified: "); lastmodified.append(TimeRFC1123(data.ModificationTime())); headers.push_back(lastmodified); -} /*}}}*/ - +} + /*}}}*/ void addDataHeaders(std::list &headers, std::string &data) {/*{{{*/ std::ostringstream contentlength; contentlength << "Content-Length: " << data.size(); headers.push_back(contentlength.str()); -} /*}}}*/ - -bool sendHead(int client, int httpcode, std::list &headers) { /*{{{*/ +} + /*}}}*/ +bool sendHead(int const client, int const httpcode, std::list &headers) { /*{{{*/ string response("HTTP/1.1 "); response.append(httpcodeToStr(httpcode)); headers.push_front(response); @@ -107,9 +107,9 @@ bool sendHead(int client, int httpcode, std::list &headers) { /*{{{ Success &= FileFd::Write(client, "\r\n", 2); std::clog << "<<<<<<<<<<<<<<<<" << std::endl; return Success; -} /*}}}*/ - -bool sendFile(int client, FileFd &data) { /*{{{*/ +} + /*}}}*/ +bool sendFile(int const client, FileFd &data) { /*{{{*/ bool Success = true; char buffer[500]; unsigned long long actual = 0; @@ -120,34 +120,32 @@ bool sendFile(int client, FileFd &data) { /*{{{*/ } Success &= FileFd::Write(client, "\r\n", 2); return Success; -} /*}}}*/ - -bool sendData(int client, std::string &data) { /*{{{*/ +} + /*}}}*/ +bool sendData(int const client, std::string const &data) { /*{{{*/ bool Success = true; Success &= FileFd::Write(client, data.c_str(), data.size()); Success &= FileFd::Write(client, "\r\n", 2); return Success; -} /*}}}*/ - -void sendError(int client, int httpcode, string request, bool content) { /*{{{*/ +} + /*}}}*/ +void sendError(int const client, int const httpcode, string const &request, bool content) { /*{{{*/ std::list headers; - string response; - if (content == true) { - response.append(""); - response.append(httpcodeToStr(httpcode)).append(""); - response.append("

").append(httpcodeToStr(httpcode)).append(""); - response.append(request).append(""); - addDataHeaders(headers, response); - } + string response(""); + response.append(httpcodeToStr(httpcode)).append(""); + response.append("

").append(httpcodeToStr(httpcode)).append(""); + response.append(request).append(""); + addDataHeaders(headers, response); sendHead(client, httpcode, headers); - sendData(client, response); -} /*}}}*/ - -int main(int argc, const char *argv[]) + if (content == true) + sendData(client, response); +} + /*}}}*/ +int main(int const argc, const char * argv[]) { CommandLine::Args Args[] = { - {0, "simulate-paywall", "aptwebserver::Simulate-Paywall", + {0, "simulate-paywall", "aptwebserver::Simulate-Paywall", CommandLine::Boolean}, {0, "port", "aptwebserver::port", CommandLine::HasArg}, {0,0,0,0} @@ -198,18 +196,19 @@ int main(int argc, const char *argv[]) } listen(sock, 1); + /*}}}*/ std::vector messages; int client; while ((client = accept(sock, NULL, NULL)) != -1) { std::clog << "ACCEPT client " << client - << " on socket " << sock << std::endl; + << " on socket " << sock << std::endl; while (ReadMessages(client, messages)) { for (std::vector::const_iterator m = messages.begin(); m != messages.end(); ++m) { - std::clog << ">>> REQUEST >>>>" << std::endl << *m - << std::endl << "<<<<<<<<<<<<<<<<" << std::endl; + std::clog << ">>> REQUEST >>>>" << std::endl << *m + << std::endl << "<<<<<<<<<<<<<<<<" << std::endl; std::list headers; bool sendContent = true; if (strncmp(m->c_str(), "HEAD ", 5) == 0) @@ -227,12 +226,12 @@ int main(int argc, const char *argv[]) size_t const filestart = m->find(' ', 5); string filename = m->substr(5, filestart - 5); - if (simulate_broken_server == true) { - string data("ni ni ni\n"); - addDataHeaders(headers, data); - sendHead(client, 200, headers); - sendData(client, data); - } + if (simulate_broken_server == true) { + string data("ni ni ni\n"); + addDataHeaders(headers, data); + sendHead(client, 200, headers); + sendData(client, data); + } else if (RealFileExists(filename) == false) sendError(client, 404, *m, sendContent); else { @@ -240,8 +239,8 @@ int main(int argc, const char *argv[]) std::string condition = LookupTag(*m, "If-Modified-Since", ""); if (condition.empty() == false) { time_t cache; - if (RFC1123StrToTime(condition.c_str(), cache) == true && - cache >= data.ModificationTime()) { + if (RFC1123StrToTime(condition.c_str(), cache) == true && + cache >= data.ModificationTime()) { sendError(client, 304, *m, false); continue; } @@ -256,8 +255,8 @@ int main(int argc, const char *argv[]) messages.clear(); } - std::clog << "CLOSE client " << client - << " on socket " << sock << std::endl; + std::clog << "CLOSE client " << client + << " on socket " << sock << std::endl; close(client); } return 0; -- cgit v1.2.3 From d64053295c5544c62d7743c13bd415e42cf8f84d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jul 2012 20:03:00 +0200 Subject: implement directory listing in your webserver --- test/interactive-helper/aptwebserver.cc | 95 +++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 97a298c70..0b2720dad 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -12,11 +12,13 @@ #include #include +#include #include #include #include #include #include +#include char const * const httpcodeToStr(int const httpcode) { /*{{{*/ switch (httpcode) { @@ -142,6 +144,86 @@ void sendError(int const client, int const httpcode, string const &request, bool sendData(client, response); } /*}}}*/ +// sendDirectoryLisiting /*{{{*/ +int filter_hidden_files(const struct dirent *a) { + if (a->d_name[0] == '.') + return 0; +#ifdef _DIRENT_HAVE_D_TYPE + // if we have the d_type check that only files and dirs will be included + if (a->d_type != DT_UNKNOWN && + a->d_type != DT_REG && + a->d_type != DT_LNK && // this includes links to regular files + a->d_type != DT_DIR) + return 0; +#endif + return 1; +} +int grouped_alpha_case_sort(const struct dirent **a, const struct dirent **b) { +#ifdef _DIRENT_HAVE_D_TYPE + if ((*a)->d_type == DT_DIR && (*b)->d_type == DT_DIR); + else if ((*a)->d_type == DT_DIR && (*b)->d_type == DT_REG) + return -1; + else if ((*b)->d_type == DT_DIR && (*a)->d_type == DT_REG) + return 1; + else +#endif + { + struct stat f_prop; //File's property + stat((*a)->d_name, &f_prop); + int const amode = f_prop.st_mode; + stat((*b)->d_name, &f_prop); + int const bmode = f_prop.st_mode; + if (S_ISDIR(amode) && S_ISDIR(bmode)); + else if (S_ISDIR(amode)) + return -1; + else if (S_ISDIR(bmode)) + return 1; + } + return strcasecmp((*a)->d_name, (*b)->d_name); +} +void sendDirectoryListing(int const client, string const &dir, string const &request, bool content) { + std::list headers; + std::ostringstream listing; + + struct dirent **namelist; + int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort); + if (counter == -1) { + sendError(client, 500, request, content); + return; + } + + listing << "Index of " << dir << "" + << "" + << "" << std::endl + << "

Index of " << dir << "

" << std::endl + << "" << std::endl; + if (dir != ".") + listing << ""; + for (int i = 0; i < counter; ++i) { + struct stat fs; + std::string filename(dir); + filename.append("/").append(namelist[i]->d_name); + stat(filename.c_str(), &fs); + listing << "" + << ""; + if (S_ISDIR(fs.st_mode)) + listing << ""; + else + listing << ""; + listing << "" << std::endl; + } + listing << "
#NameSizeLast-Modified
dParent Directory--
" << ((S_ISDIR(fs.st_mode)) ? 'd' : 'f') << "d_name << "\">" << namelist[i]->d_name << "-" << SizeToStr(fs.st_size) << "B" << TimeRFC1123(fs.st_mtime) << "
" << std::endl; + + std::string response(listing.str()); + addDataHeaders(headers, response); + sendHead(client, 200, headers); + if (content == true) + sendData(client, response); +} + /*}}}*/ int main(int const argc, const char * argv[]) { CommandLine::Args Args[] = { @@ -225,6 +307,8 @@ int main(int const argc, const char * argv[]) size_t const filestart = m->find(' ', 5); string filename = m->substr(5, filestart - 5); + if (filename.empty() == true) + filename = "."; if (simulate_broken_server == true) { string data("ni ni ni\n"); @@ -232,16 +316,14 @@ int main(int const argc, const char * argv[]) sendHead(client, 200, headers); sendData(client, data); } - else if (RealFileExists(filename) == false) - sendError(client, 404, *m, sendContent); - else { + else if (RealFileExists(filename) == true) { FileFd data(filename, FileFd::ReadOnly); std::string condition = LookupTag(*m, "If-Modified-Since", ""); if (condition.empty() == false) { time_t cache; if (RFC1123StrToTime(condition.c_str(), cache) == true && cache >= data.ModificationTime()) { - sendError(client, 304, *m, false); + sendHead(client, 304, headers); continue; } } @@ -250,6 +332,11 @@ int main(int const argc, const char * argv[]) if (sendContent == true) sendFile(client, data); } + else if (DirectoryExists(filename) == true) { + sendDirectoryListing(client, filename, *m, sendContent); + } + else + sendError(client, 404, *m, false); } _error->DumpErrors(std::cerr); messages.clear(); -- cgit v1.2.3 From 4958ba98e66af264e1f03f5dbacc713e51cd437d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jul 2012 20:05:05 +0200 Subject: include our usual config.h in the webserver --- test/interactive-helper/aptwebserver.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 0b2720dad..0780288a4 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -1,3 +1,5 @@ +#include + #include #include #include @@ -88,7 +90,7 @@ void addDataHeaders(std::list &headers, std::string &data) {/*{{{*/ } /*}}}*/ bool sendHead(int const client, int const httpcode, std::list &headers) { /*{{{*/ - string response("HTTP/1.1 "); + std::string response("HTTP/1.1 "); response.append(httpcodeToStr(httpcode)); headers.push_front(response); @@ -131,9 +133,9 @@ bool sendData(int const client, std::string const &data) { /*{{{*/ return Success; } /*}}}*/ -void sendError(int const client, int const httpcode, string const &request, bool content) { /*{{{*/ +void sendError(int const client, int const httpcode, std::string const &request, bool content) { /*{{{*/ std::list headers; - string response(""); + std::string response("<html><head><title>"); response.append(httpcodeToStr(httpcode)).append(""); response.append("

").append(httpcodeToStr(httpcode)).append(""); @@ -181,7 +183,7 @@ int grouped_alpha_case_sort(const struct dirent **a, const struct dirent **b) { } return strcasecmp((*a)->d_name, (*b)->d_name); } -void sendDirectoryListing(int const client, string const &dir, string const &request, bool content) { +void sendDirectoryListing(int const client, std::string const &dir, std::string const &request, bool content) { std::list headers; std::ostringstream listing; @@ -306,12 +308,12 @@ int main(int const argc, const char * argv[]) } size_t const filestart = m->find(' ', 5); - string filename = m->substr(5, filestart - 5); + std::string filename = m->substr(5, filestart - 5); if (filename.empty() == true) filename = "."; if (simulate_broken_server == true) { - string data("ni ni ni\n"); + std::string data("ni ni ni\n"); addDataHeaders(headers, data); sendHead(client, 200, headers); sendData(client, data); -- cgit v1.2.3 From 59fe94ea0135b1f8bc3d66e97460bd481054b061 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jul 2012 20:08:58 +0200 Subject: http get requests need to be dequoted --- test/interactive-helper/aptwebserver.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 0780288a4..920ab3bcc 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -311,6 +311,8 @@ int main(int const argc, const char * argv[]) std::string filename = m->substr(5, filestart - 5); if (filename.empty() == true) filename = "."; + else + filename = DeQuoteString(filename); if (simulate_broken_server == true) { std::string data("ni ni ni\n"); -- cgit v1.2.3 From 64a28515923aa67a1d109a82aba1892cd227bb15 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 11 Jul 2012 00:46:27 +0200 Subject: ensure that directories are accessed with a slash at the end of the url --- test/interactive-helper/aptwebserver.cc | 40 +++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 920ab3bcc..2052fe6d8 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -146,6 +146,26 @@ void sendError(int const client, int const httpcode, std::string const &request, sendData(client, response); } /*}}}*/ +void sendRedirect(int const client, int const httpcode, std::string const &uri, std::string const &request, bool content) { /*{{{*/ + std::list headers; + std::string response(""); + response.append(httpcodeToStr(httpcode)).append(""); + response.append("

").append(httpcodeToStr(httpcode)).append("You should be redirected to ").append(uri).append("

"); + response.append("This page is a result of the request: 
");
+   response.append(request).append("
"); + addDataHeaders(headers, response); + std::string location("Location: "); + if (strncmp(uri.c_str(), "http://", 7) != 0) + location.append("http://").append(LookupTag(request, "Host")).append("/").append(uri); + else + location.append(uri); + headers.push_back(location); + sendHead(client, httpcode, headers); + if (content == true) + sendData(client, response); +} + /*}}}*/ // sendDirectoryLisiting /*{{{*/ int filter_hidden_files(const struct dirent *a) { if (a->d_name[0] == '.') @@ -209,12 +229,15 @@ void sendDirectoryListing(int const client, std::string const &dir, std::string std::string filename(dir); filename.append("/").append(namelist[i]->d_name); stat(filename.c_str(), &fs); - listing << "" << ((S_ISDIR(fs.st_mode)) ? 'd' : 'f') << "" - << "d_name << "\">" << namelist[i]->d_name << ""; - if (S_ISDIR(fs.st_mode)) - listing << "-"; - else - listing << "" << SizeToStr(fs.st_size) << "B"; + if (S_ISDIR(fs.st_mode)) { + listing << "d" + << "d_name << "/\">" << namelist[i]->d_name << "" + << "-"; + } else { + listing << "f" + << "d_name << "\">" << namelist[i]->d_name << "" + << "" << SizeToStr(fs.st_size) << "B"; + } listing << "" << TimeRFC1123(fs.st_mtime) << "" << std::endl; } listing << "" << std::endl; @@ -337,7 +360,10 @@ int main(int const argc, const char * argv[]) sendFile(client, data); } else if (DirectoryExists(filename) == true) { - sendDirectoryListing(client, filename, *m, sendContent); + if (filename == "." || filename[filename.length()-1] == '/') + sendDirectoryListing(client, filename, *m, sendContent); + else + sendRedirect(client, 301, filename.append("/"), *m, sendContent); } else sendError(client, 404, *m, false); -- cgit v1.2.3 From 3522b1a8f16f45cf9c0c45b8b86cc886f1368df4 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 11 Jul 2012 10:39:51 +0200 Subject: do not hardcode /dev/null in changetowebserver so it can be changed for debugging easily --- test/integration/framework | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index d15fd0e01..965f984ca 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -688,30 +688,30 @@ signreleasefiles() { } changetowebserver() { + if [ -n "$1" ] && ! test -x ${BUILDDIRECTORY}/aptwebserver; then + msgdie 'Need the aptwebserver when passing arguments' + fi - if [ -n "$1" ] && ! test -x ${BUILDDIRECTORY}/aptwebserver; then - msgdie 'Need the aptwebserver when passing arguments' - fi - + local LOG='/dev/null' if test -x ${BUILDDIRECTORY}/aptwebserver; then cd aptarchive - LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/aptwebserver $@ 2> /dev/null > /dev/null & + LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/aptwebserver $@ 2> $LOG > $LOG & addtrap "kill $!;" cd - > /dev/null elif which weborf > /dev/null; then - weborf -xb aptarchive/ 2>&1 > /dev/null & + weborf -xb aptarchive/ 2> $LOG > $LOG & addtrap "kill $!;" elif which gatling > /dev/null; then cd aptarchive - gatling -p 8080 -F -S 2>&1 > /dev/null & + gatling -p 8080 -F -S 2> $LOG > $LOG & addtrap "kill $!;" cd - > /dev/null elif which lighttpd > /dev/null; then echo "server.document-root = \"$(readlink -f ./aptarchive)\" server.port = 8080 server.stat-cache-engine = \"disable\"" > lighttpd.conf - lighttpd -t -f lighttpd.conf >/dev/null || msgdie 'Can not change to webserver: our lighttpd config is invalid' - lighttpd -D -f lighttpd.conf 2>/dev/null >/dev/null & + lighttpd -t -f lighttpd.conf 2> $LOG > $LOG || msgdie 'Can not change to webserver: our lighttpd config is invalid' + lighttpd -D -f lighttpd.conf 2> $LOG > $LOG & addtrap "kill $!;" else msgdie 'You have to install weborf or lighttpd first' -- cgit v1.2.3 From d37911acee3eb34368e6d9e6a0046c9150d2bce6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 11 Jul 2012 11:07:54 +0200 Subject: add simple url rewriting to the webserver --- test/integration/skip-bug-602412-dequote-redirect | 38 ----------------------- test/integration/test-bug-602412-dequote-redirect | 29 +++++++++++++++++ test/interactive-helper/aptwebserver.cc | 19 ++++++++++-- 3 files changed, 46 insertions(+), 40 deletions(-) delete mode 100755 test/integration/skip-bug-602412-dequote-redirect create mode 100755 test/integration/test-bug-602412-dequote-redirect (limited to 'test') diff --git a/test/integration/skip-bug-602412-dequote-redirect b/test/integration/skip-bug-602412-dequote-redirect deleted file mode 100755 index 689b671ce..000000000 --- a/test/integration/skip-bug-602412-dequote-redirect +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -set -e - -TESTDIR=$(readlink -f $(dirname $0)) -. $TESTDIR/framework -setupenvironment -configarchitecture 'i386' - -if ! which lighttpd > /dev/null; then - msgdie 'You need lighttpd for this testcase, sorry…' - exit 1 -fi - -buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable' - -setupaptarchive - -echo "server.modules = ( \"mod_redirect\" ) -server.document-root = \"$(readlink -f ./aptarchive)\" -server.port = 8080 -server.stat-cache-engine = \"disable\" -url.redirect = ( \"^/pool/(.*)$\" => \"/newpool/\$1\", - \"^/dists/(.*)$\" => \"/newdists/\$1\" )" > lighttpd.conf - -mv aptarchive/pool aptarchive/newpool -mv aptarchive/dists aptarchive/newdists - -lighttpd -t -f lighttpd.conf >/dev/null || msgdie 'Can not change to webserver: our lighttpd config is invalid' -lighttpd -D -f lighttpd.conf 2>/dev/null >/dev/null & -addtrap "kill $!;" - -APTARCHIVE="file://$(readlink -f ./aptarchive)" -for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do - sed -i $LIST -e "s#$APTARCHIVE#http://localhost:8080/#" -done - -aptget update || msgdie 'apt-get update failed' -aptget install unrelated --download-only || msgdie 'downloading package failed' diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect new file mode 100755 index 000000000..f1e67c6d8 --- /dev/null +++ b/test/integration/test-bug-602412-dequote-redirect @@ -0,0 +1,29 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' + +buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable' + +setupaptarchive +changetowebserver -o aptwebserver::redirect::replace::/pool/=/newpool/ \ + -o aptwebserver::redirect::replace::/dists/=/newdists/ + +mv aptarchive/pool aptarchive/newpool +mv aptarchive/dists aptarchive/newdists + +msgtest 'Test redirection works in' 'apt-get update' +aptget update -qq && msgpass || msgfail + +# check that I-M-S header is kept in redirections +testequal 'Hit http://localhost unstable InRelease +Hit http://localhost unstable/main Sources +Hit http://localhost unstable/main amd64 Packages +Hit http://localhost unstable/main Translation-en +Reading package lists...' aptget update + +msgtest 'Test redirection works in' 'package download' +aptget install unrelated --download-only -qq && msgpass || msgfail diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 2052fe6d8..7fa322ea9 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -137,7 +137,7 @@ void sendError(int const client, int const httpcode, std::string const &request, std::list headers; std::string response(""); response.append(httpcodeToStr(httpcode)).append(""); - response.append("

").append(httpcodeToStr(httpcode)).append("

").append(httpcodeToStr(httpcode)).append("

"); response.append("This error is a result of the request: 
");
    response.append(request).append("
"); addDataHeaders(headers, response); @@ -255,6 +255,8 @@ int main(int const argc, const char * argv[]) {0, "simulate-paywall", "aptwebserver::Simulate-Paywall", CommandLine::Boolean}, {0, "port", "aptwebserver::port", CommandLine::HasArg}, + {'c',"config-file",0,CommandLine::ConfigFile}, + {'o',"option",0,CommandLine::ArbItem}, {0,0,0,0} }; @@ -366,7 +368,20 @@ int main(int const argc, const char * argv[]) sendRedirect(client, 301, filename.append("/"), *m, sendContent); } else - sendError(client, 404, *m, false); + { + ::Configuration::Item const *Replaces = _config->Tree("aptwebserver::redirect::replace"); + if (Replaces != NULL) { + std::string redirect = "/" + filename; + for (::Configuration::Item *I = Replaces->Child; I != NULL; I = I->Next) + redirect = SubstVar(redirect, I->Tag, I->Value); + redirect.erase(0,1); + if (redirect != filename) { + sendRedirect(client, 301, redirect, *m, sendContent); + continue; + } + } + sendError(client, 404, *m, sendContent); + } } _error->DumpErrors(std::cerr); messages.clear(); -- cgit v1.2.3 From 7a1bed9d453379fcd22489bdb9f81f377abcaae2 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 11 Jul 2012 17:11:26 +0200 Subject: rework parsing of the first request-line to be more robust --- test/interactive-helper/aptwebserver.cc | 106 ++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 31 deletions(-) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 7fa322ea9..4746aed96 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -51,24 +51,24 @@ char const * const httpcodeToStr(int const httpcode) { /*{{{*/ case 404: return "404 Not Found"; case 405: return "405 Method Not Allowed"; case 406: return "406 Not Acceptable"; - case 407: return "Proxy Authentication Required"; - case 408: return "Request Time-out"; - case 409: return "Conflict"; - case 410: return "Gone"; - case 411: return "Length Required"; - case 412: return "Precondition Failed"; - case 413: return "Request Entity Too Large"; - case 414: return "Request-URI Too Large"; - case 415: return "Unsupported Media Type"; - case 416: return "Requested range not satisfiable"; - case 417: return "Expectation Failed"; + case 407: return "407 Proxy Authentication Required"; + case 408: return "408 Request Time-out"; + case 409: return "409 Conflict"; + case 410: return "410 Gone"; + case 411: return "411 Length Required"; + case 412: return "412 Precondition Failed"; + case 413: return "413 Request Entity Too Large"; + case 414: return "414 Request-URI Too Large"; + case 415: return "415 Unsupported Media Type"; + case 416: return "416 Requested range not satisfiable"; + case 417: return "417 Expectation Failed"; // Server error 5xx - case 500: return "Internal Server Error"; - case 501: return "Not Implemented"; - case 502: return "Bad Gateway"; - case 503: return "Service Unavailable"; - case 504: return "Gateway Time-out"; - case 505: return "HTTP Version not supported"; + case 500: return "500 Internal Server Error"; + case 501: return "501 Not Implemented"; + case 502: return "502 Bad Gateway"; + case 503: return "503 Service Unavailable"; + case 504: return "504 Gateway Time-out"; + case 505: return "505 HTTP Version not supported"; } return NULL; } @@ -133,11 +133,13 @@ bool sendData(int const client, std::string const &data) { /*{{{*/ return Success; } /*}}}*/ -void sendError(int const client, int const httpcode, std::string const &request, bool content) { /*{{{*/ +void sendError(int const client, int const httpcode, std::string const &request, bool content, std::string const &error = "") { /*{{{*/ std::list headers; std::string response(""); response.append(httpcodeToStr(httpcode)).append(""); response.append("

").append(httpcodeToStr(httpcode)).append("

"); + if (error.empty() == false) + response.append("

Error: ").append(error).append("

"); response.append("This error is a result of the request: 
");
    response.append(request).append("
"); addDataHeaders(headers, response); @@ -249,6 +251,55 @@ void sendDirectoryListing(int const client, std::string const &dir, std::string sendData(client, response); } /*}}}*/ +bool parseFirstLine(int const client, std::string const &request, std::string &filename, bool &sendContent) { /*{{{*/ + if (strncmp(request.c_str(), "HEAD ", 5) == 0) + sendContent = false; + if (strncmp(request.c_str(), "GET ", 4) != 0) + { + sendError(client, 501, request, true); + return false; + } + + size_t const lineend = request.find('\n'); + size_t filestart = request.find(' '); + for (; request[filestart] == ' '; ++filestart); + size_t fileend = request.rfind(' ', lineend); + if (lineend == std::string::npos || filestart == std::string::npos || + fileend == std::string::npos || filestart == fileend) { + sendError(client, 500, request, sendContent, "Filename can't be extracted"); + return false; + } + + size_t httpstart = fileend; + for (; request[httpstart] == ' '; ++httpstart); + if (strncmp(request.c_str() + httpstart, "HTTP/1.1\r", 9) != 0) { + sendError(client, 500, request, sendContent, "Not an HTTP/1.1 request"); + return false; + } + + filename = request.substr(filestart, fileend - filestart); + if (filename.find(' ') != std::string::npos) { + sendError(client, 500, request, sendContent, "Filename contains an unencoded space"); + return false; + } + filename = DeQuoteString(filename); + + // this is not a secure server, but at least prevent the obvious … + if (filename.empty() == true || filename[0] != '/' || + strncmp(filename.c_str(), "//", 2) == 0 || + filename.find_first_of("\r\n\t\f\v") != std::string::npos || + filename.find("/../") != std::string::npos) { + sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)"); + return false; + } + + // nuke the first character which is a / as we assured above + filename.erase(0, 1); + if (filename.empty() == true) + filename = "."; + return true; +} + /*}}}*/ int main(int const argc, const char * argv[]) { CommandLine::Args Args[] = { @@ -318,27 +369,20 @@ int main(int const argc, const char * argv[]) m != messages.end(); ++m) { std::clog << ">>> REQUEST >>>>" << std::endl << *m << std::endl << "<<<<<<<<<<<<<<<<" << std::endl; + std::list headers; + std::string filename; bool sendContent = true; - if (strncmp(m->c_str(), "HEAD ", 5) == 0) - sendContent = false; - if (strncmp(m->c_str(), "GET ", 4) != 0) - sendError(client, 501, *m, true); + if (parseFirstLine(client, *m, filename, sendContent) == false) + continue; std::string host = LookupTag(*m, "Host", ""); if (host.empty() == true) { - // RFC 2616 §14.23 Host - sendError(client, 400, *m, sendContent); + // RFC 2616 §14.23 requires Host + sendError(client, 400, *m, sendContent, "Host header is required"); continue; } - size_t const filestart = m->find(' ', 5); - std::string filename = m->substr(5, filestart - 5); - if (filename.empty() == true) - filename = "."; - else - filename = DeQuoteString(filename); - if (simulate_broken_server == true) { std::string data("ni ni ni\n"); addDataHeaders(headers, data); -- cgit v1.2.3 From 57d13de2fb64a97d1a43d493c253ad2132ffd566 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 11 Jul 2012 18:43:53 +0200 Subject: make the server a little more robust against write errors (e.g. broken pipe) --- test/interactive-helper/aptwebserver.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 4746aed96..dbc4a19e0 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -21,6 +21,7 @@ #include #include #include +#include char const * const httpcodeToStr(int const httpcode) { /*{{{*/ switch (httpcode) { @@ -105,10 +106,12 @@ bool sendHead(int const client, int const httpcode, std::list &head for (std::list::const_iterator h = headers.begin(); Success == true && h != headers.end(); ++h) { Success &= FileFd::Write(client, h->c_str(), h->size()); - Success &= FileFd::Write(client, "\r\n", 2); + if (Success == true) + Success &= FileFd::Write(client, "\r\n", 2); std::clog << *h << std::endl; } - Success &= FileFd::Write(client, "\r\n", 2); + if (Success == true) + Success &= FileFd::Write(client, "\r\n", 2); std::clog << "<<<<<<<<<<<<<<<<" << std::endl; return Success; } @@ -122,14 +125,16 @@ bool sendFile(int const client, FileFd &data) { /*{{{*/ break; Success &= FileFd::Write(client, buffer, actual); } - Success &= FileFd::Write(client, "\r\n", 2); + if (Success == true) + Success &= FileFd::Write(client, "\r\n", 2); return Success; } /*}}}*/ bool sendData(int const client, std::string const &data) { /*{{{*/ bool Success = true; Success &= FileFd::Write(client, data.c_str(), data.size()); - Success &= FileFd::Write(client, "\r\n", 2); + if (Success == true) + Success &= FileFd::Write(client, "\r\n", 2); return Success; } /*}}}*/ @@ -318,6 +323,8 @@ int main(int const argc, const char * argv[]) } // create socket, bind and listen to it {{{ + // ignore SIGPIPE, this can happen on write() if the socket closes connection + signal(SIGPIPE, SIG_IGN); int sock = socket(AF_INET6, SOCK_STREAM, 0); if(sock < 0 ) { _error->Errno("aptwerbserver", "Couldn't create socket"); -- cgit v1.2.3 From 06b3095f81d9a730e8cb95274c8208cb0604cdfe Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 11 Jul 2012 19:11:24 +0200 Subject: add option to send Content-Type based on file extension --- test/interactive-helper/aptwebserver.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index dbc4a19e0..4ef9631b8 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -82,6 +82,18 @@ void addFileHeaders(std::list &headers, FileFd &data) { /*{{{*/ std::string lastmodified("Last-Modified: "); lastmodified.append(TimeRFC1123(data.ModificationTime())); headers.push_back(lastmodified); + + std::string const fileext = flExtension(data.Name()); + if (fileext.empty() == false && fileext != data.Name()) { + std::string confcontenttype("aptwebserver::ContentType::"); + confcontenttype.append(fileext); + std::string const contenttype = _config->Find(confcontenttype); + if (contenttype.empty() == false) { + std::string header("Content-Type: "); + header.append(contenttype); + headers.push_back(header); + } + } } /*}}}*/ void addDataHeaders(std::list &headers, std::string &data) {/*{{{*/ -- cgit v1.2.3 From 056c36565706cad136df288db777c01555f4ecd9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 12 Jul 2012 12:04:36 +0200 Subject: add (partial) partial request support for the webserver --- test/interactive-helper/aptwebserver.cc | 71 +++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 8 deletions(-) (limited to 'test') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 4ef9631b8..ff60d64a3 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -35,7 +35,7 @@ char const * const httpcodeToStr(int const httpcode) { /*{{{*/ case 203: return "203 Non-Authoritative Information"; case 204: return "204 No Content"; case 205: return "205 Reset Content"; - case 206: return "206 Partial Conent"; + case 206: return "206 Partial Content"; // Redirections 3xx case 300: return "300 Multiple Choices"; case 301: return "301 Moved Permanently"; @@ -113,6 +113,8 @@ bool sendHead(int const client, int const httpcode, std::list &head date.append(TimeRFC1123(time(NULL))); headers.push_back(date); + headers.push_back("Accept-Ranges: bytes"); + std::clog << ">>> RESPONSE >>>" << std::endl; bool Success = true; for (std::list::const_iterator h = headers.begin(); @@ -135,7 +137,8 @@ bool sendFile(int const client, FileFd &data) { /*{{{*/ while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true) { if (actual == 0) break; - Success &= FileFd::Write(client, buffer, actual); + if (Success == true) + Success &= FileFd::Write(client, buffer, actual); } if (Success == true) Success &= FileFd::Write(client, "\r\n", 2); @@ -268,7 +271,7 @@ void sendDirectoryListing(int const client, std::string const &dir, std::string sendData(client, response); } /*}}}*/ -bool parseFirstLine(int const client, std::string const &request, std::string &filename, bool &sendContent) { /*{{{*/ +bool parseFirstLine(int const client, std::string const &request, std::string &filename, bool &sendContent, bool &closeConnection) { /*{{{*/ if (strncmp(request.c_str(), "HEAD ", 5) == 0) sendContent = false; if (strncmp(request.c_str(), "GET ", 4) != 0) @@ -289,8 +292,12 @@ bool parseFirstLine(int const client, std::string const &request, std::string &f size_t httpstart = fileend; for (; request[httpstart] == ' '; ++httpstart); - if (strncmp(request.c_str() + httpstart, "HTTP/1.1\r", 9) != 0) { - sendError(client, 500, request, sendContent, "Not an HTTP/1.1 request"); + if (strncmp(request.c_str() + httpstart, "HTTP/1.1\r", 9) == 0) + closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "Keep-Alive") != 0; + else if (strncmp(request.c_str() + httpstart, "HTTP/1.0\r", 9) == 0) + closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0; + else { + sendError(client, 500, request, sendContent, "Not an HTTP/1.{0,1} request"); return false; } @@ -384,15 +391,15 @@ int main(int const argc, const char * argv[]) << " on socket " << sock << std::endl; while (ReadMessages(client, messages)) { + bool closeConnection = false; for (std::vector::const_iterator m = messages.begin(); - m != messages.end(); ++m) { + m != messages.end() && closeConnection == false; ++m) { std::clog << ">>> REQUEST >>>>" << std::endl << *m << std::endl << "<<<<<<<<<<<<<<<<" << std::endl; - std::list headers; std::string filename; bool sendContent = true; - if (parseFirstLine(client, *m, filename, sendContent) == false) + if (parseFirstLine(client, *m, filename, sendContent, closeConnection) == false) continue; std::string host = LookupTag(*m, "Host", ""); @@ -419,6 +426,52 @@ int main(int const argc, const char * argv[]) continue; } } + condition = LookupTag(*m, "If-Range", ""); + bool ignoreRange = false; + if (condition.empty() == false) { + time_t cache; + if (RFC1123StrToTime(condition.c_str(), cache) == false || + cache < data.ModificationTime()) + ignoreRange = true; + } + condition = LookupTag(*m, "Range", ""); + if (ignoreRange == false && condition.empty() == false && + strncmp(condition.c_str(), "bytes=", 6) == 0) { + size_t end = condition.find(','); + // FIXME: support multiple byte-ranges + if (end == std::string::npos) { + size_t start = 6; + unsigned long long filestart = strtoull(condition.c_str() + start, NULL, 10); + // FIXME: no fileend support + size_t dash = condition.find('-') + 1; + unsigned long long fileend = strtoull(condition.c_str() + dash, NULL, 10); + unsigned long long filesize = data.FileSize(); + if (fileend == 0 || fileend == filesize) { + if (filesize > filestart) { + data.Skip(filestart); + std::ostringstream contentlength; + contentlength << "Content-Length: " << (filesize - filestart); + headers.push_back(contentlength.str()); + std::ostringstream contentrange; + contentrange << "Content-Range: bytes " << filestart << "-" + << filesize - 1 << "/" << filesize; + headers.push_back(contentrange.str()); + sendHead(client, 206, headers); + if (sendContent == true) + sendFile(client, data); + continue; + } else { + headers.push_back("Content-Length: 0"); + std::ostringstream contentrange; + contentrange << "Content-Range: bytes 0-0/" << filesize; + headers.push_back(contentrange.str()); + sendHead(client, 416, headers); + continue; + } + } + } + } + addFileHeaders(headers, data); sendHead(client, 200, headers); if (sendContent == true) @@ -448,6 +501,8 @@ int main(int const argc, const char * argv[]) } _error->DumpErrors(std::cerr); messages.clear(); + if (closeConnection == true) + break; } std::clog << "CLOSE client " << client -- cgit v1.2.3