From 93a99dac870584ed4ea78f1c2f262db8b5460962 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 11 Oct 2013 13:21:01 +0200 Subject: webserver: spurious newline after data confuses curl Webserver wrongly sends an additional newline after the data which causes curl to believe that the next request on this socket has no header data and so includes all headers in the data output. Git-Dch: Ignore --- test/interactive-helper/aptwebserver.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'test/interactive-helper/aptwebserver.cc') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 4dae342dd..6c5634de6 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -137,21 +137,21 @@ bool sendFile(int const client, FileFd &data) /*{{{*/ { if (actual == 0) break; - if (Success == true) - Success &= FileFd::Write(client, buffer, actual); + Success &= FileFd::Write(client, buffer, actual); } - if (Success == true) - Success &= FileFd::Write(client, "\r\n", 2); + if (Success == false) + std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl; return Success; } /*}}}*/ bool sendData(int const client, std::string const &data) /*{{{*/ { - bool Success = true; - Success &= FileFd::Write(client, data.c_str(), data.size()); - if (Success == true) - Success &= FileFd::Write(client, "\r\n", 2); - return Success; + if (FileFd::Write(client, data.c_str(), data.size()) == false) + { + std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl; + return false; + } + return true; } /*}}}*/ void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/ -- cgit v1.2.3 From eab3a9b2e233d7a142c0fa90bd1ed16a3c0e6cc6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 16 Oct 2013 16:46:55 +0200 Subject: webserver: implement ~user access to ~user/public_html/ pretty much useless for the testcases, but handy to test the webserver itself in 'real world' environments Git-Dch: Ignore --- test/interactive-helper/aptwebserver.cc | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) (limited to 'test/interactive-helper/aptwebserver.cc') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 6c5634de6..4ea1794bb 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -198,7 +198,17 @@ void sendRedirect(int const client, int const httpcode, std::string const &uri,/ addDataHeaders(headers, response); std::string location("Location: "); if (strncmp(uri.c_str(), "http://", 7) != 0) - location.append("http://").append(LookupTag(request, "Host")).append("/").append(uri); + { + location.append("http://").append(LookupTag(request, "Host")).append("/"); + if (strncmp("/home/", uri.c_str(), strlen("/home/")) == 0 && uri.find("/public_html/") != std::string::npos) + { + std::string homeuri = SubstVar(uri, "/home/", "~"); + homeuri = SubstVar(homeuri, "/public_html/", "/"); + location.append(homeuri); + } + else + location.append(uri); + } else location.append(uri); headers.push_back(location); @@ -381,6 +391,20 @@ bool parseFirstLine(int const client, std::string const &request, /*{{{*/ filename.erase(0, 1); if (filename.empty() == true) filename = "."; + // support ~user/ uris to refer to /home/user/public_html/ as a kind-of special directory + else if (filename[0] == '~') + { + // /home/user is actually not entirely correct, but good enough for now + size_t dashpos = filename.find('/'); + if (dashpos != std::string::npos) + { + std::string home = filename.substr(1, filename.find('/') - 1); + std::string pubhtml = filename.substr(filename.find('/') + 1); + filename = "/home/" + home + "/public_html/" + pubhtml; + } + else + filename = "/home/" + filename.substr(1) + "/public_html/"; + } return true; } /*}}}*/ -- cgit v1.2.3 From 3c16b5fe7950e32a0a13ad7544ed531ace535316 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 16 Oct 2013 18:43:21 +0200 Subject: webserver: add directoryIndex support defaulting to index.html Git-Dch: Ignore --- test/interactive-helper/aptwebserver.cc | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'test/interactive-helper/aptwebserver.cc') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 4ea1794bb..0e04826c5 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -277,7 +277,7 @@ void sendDirectoryListing(int const client, std::string const &dir, /*{{{*/ << "" << std::endl << "

Index of " << dir << "

" << std::endl << "" << std::endl; - if (dir != ".") + if (dir != "./") listing << ""; for (int i = 0; i < counter; ++i) { struct stat fs; @@ -390,7 +390,7 @@ bool parseFirstLine(int const client, std::string const &request, /*{{{*/ // nuke the first character which is a / as we assured above filename.erase(0, 1); if (filename.empty() == true) - filename = "."; + filename = "./"; // support ~user/ uris to refer to /home/user/public_html/ as a kind-of special directory else if (filename[0] == '~') { @@ -405,6 +405,17 @@ bool parseFirstLine(int const client, std::string const &request, /*{{{*/ else filename = "/home/" + filename.substr(1) + "/public_html/"; } + + // if no filename is given, but a valid directory see if we can use an index or + // have to resort to a autogenerated directory listing later on + if (DirectoryExists(filename) == true) + { + std::string const directoryIndex = _config->Find("aptwebserver::directoryindex"); + if (directoryIndex.empty() == false && directoryIndex == flNotDir(directoryIndex) && + RealFileExists(filename + directoryIndex) == true) + filename += directoryIndex; + } + return true; } /*}}}*/ @@ -543,6 +554,7 @@ int main(int const argc, const char * argv[]) _config->CndSet("aptwebserver::response-header::Server", "APT webserver"); _config->CndSet("aptwebserver::response-header::Accept-Ranges", "bytes"); + _config->CndSet("aptwebserver::directoryindex", "index.html"); std::vector messages; int client; @@ -693,7 +705,7 @@ int main(int const argc, const char * argv[]) } else if (DirectoryExists(filename) == true) { - if (filename == "." || filename[filename.length()-1] == '/') + if (filename[filename.length()-1] == '/') sendDirectoryListing(client, filename, *m, sendContent); else sendRedirect(client, 301, filename.append("/"), *m, sendContent); -- cgit v1.2.3 From d23bda42456bd092751deb24d8295c27a15721e8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 16 Oct 2013 22:43:01 +0200 Subject: webserver: strip parameters from filename Again, not (currently) used by the tests itself, but in interactive usage of the webserver itself. Git-Dch: Ignore --- test/interactive-helper/aptwebserver.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'test/interactive-helper/aptwebserver.cc') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 0e04826c5..94f63bb39 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -308,7 +308,7 @@ void sendDirectoryListing(int const client, std::string const &dir, /*{{{*/ } /*}}}*/ bool parseFirstLine(int const client, std::string const &request, /*{{{*/ - std::string &filename, bool &sendContent, + std::string &filename, std::string ¶ms, bool &sendContent, bool &closeConnection) { if (strncmp(request.c_str(), "HEAD ", 5) == 0) @@ -375,6 +375,14 @@ bool parseFirstLine(int const client, std::string const &request, /*{{{*/ sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that"); return false; } + + size_t paramspos = filename.find('?'); + if (paramspos != std::string::npos) + { + params = filename.substr(paramspos + 1); + filename.erase(paramspos); + } + filename = DeQuoteString(filename); // this is not a secure server, but at least prevent the obvious … -- cgit v1.2.3 From 575fe03ee310c740bfa2950aa55b3358e8a60eee Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 16 Oct 2013 22:45:37 +0200 Subject: webserver: use pthreads to handle multiple clients Clients like browsers prefer to open many connections and keep them open for a while, so that pages with lot of subelements would take a while to load (if at all), by using threads as all servers do some way or another we can resolve this. libapt is not intended to be pthread-safe and stuff like the storage of the last return code doesn't make too much sense if multiple clients interact with us, but it is good enough for now and an other interesting (mis)use of libapt in general. Git-Dch: Ignore --- test/interactive-helper/aptwebserver.cc | 362 ++++++++++++++++++-------------- 1 file changed, 200 insertions(+), 162 deletions(-) (limited to 'test/interactive-helper/aptwebserver.cc') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 94f63bb39..7d3589c9d 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -112,7 +112,7 @@ bool sendHead(int const client, int const httpcode, std::list &head date.append(TimeRFC1123(time(NULL))); headers.push_back(date); - std::clog << ">>> RESPONSE >>>" << std::endl; + std::clog << ">>> RESPONSE to " << client << " >>>" << std::endl; bool Success = true; for (std::list::const_iterator h = headers.begin(); Success == true && h != headers.end(); ++h) @@ -470,6 +470,173 @@ bool handleOnTheFlyReconfiguration(int const client, std::string const &request, return false; } /*}}}*/ +void * handleClient(void * voidclient) /*{{{*/ +{ + int client = *((int*)(voidclient)); + std::clog << "ACCEPT client " << client << std::endl; + std::vector messages; + while (ReadMessages(client, messages)) + { + bool closeConnection = false; + for (std::vector::const_iterator m = messages.begin(); + m != messages.end() && closeConnection == false; ++m) { + std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m + << std::endl << "<<<<<<<<<<<<<<<<" << std::endl; + std::list headers; + std::string filename; + std::string params; + bool sendContent = true; + if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false) + continue; + + // special webserver command request + if (filename.length() > 1 && filename[0] == '_') + { + std::vector parts = VectorizeString(filename, '/'); + if (parts[0] == "_config") + { + handleOnTheFlyReconfiguration(client, *m, parts); + continue; + } + } + + // string replacements in the requested filename + ::Configuration::Item const *Replaces = _config->Tree("aptwebserver::redirect::replace"); + if (Replaces != NULL) + { + std::string redirect = "/" + filename; + for (::Configuration::Item *I = Replaces->Child; I != NULL; I = I->Next) + redirect = SubstVar(redirect, I->Tag, I->Value); + redirect.erase(0,1); + if (redirect != filename) + { + sendRedirect(client, 301, redirect, *m, sendContent); + continue; + } + } + + ::Configuration::Item const *Overwrite = _config->Tree("aptwebserver::overwrite"); + if (Overwrite != NULL) + { + for (::Configuration::Item *I = Overwrite->Child; I != NULL; I = I->Next) + { + regex_t *pattern = new regex_t; + int const res = regcomp(pattern, I->Tag.c_str(), REG_EXTENDED | REG_ICASE | REG_NOSUB); + if (res != 0) + { + char error[300]; + regerror(res, pattern, error, sizeof(error)); + sendError(client, 500, *m, sendContent, error); + continue; + } + if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0) + { + filename = _config->Find("aptwebserver::overwrite::" + I->Tag + "::filename", filename); + if (filename[0] == '/') + filename.erase(0,1); + regfree(pattern); + break; + } + regfree(pattern); + } + } + + // deal with the request + if (RealFileExists(filename) == true) + { + FileFd data(filename, FileFd::ReadOnly); + std::string condition = LookupTag(*m, "If-Modified-Since", ""); + if (condition.empty() == false) + { + time_t cache; + if (RFC1123StrToTime(condition.c_str(), cache) == true && + cache >= data.ModificationTime()) + { + sendHead(client, 304, headers); + continue; + } + } + + if (_config->FindB("aptwebserver::support::range", true) == true) + condition = LookupTag(*m, "Range", ""); + else + condition.clear(); + if (condition.empty() == false && strncmp(condition.c_str(), "bytes=", 6) == 0) + { + time_t cache; + std::string ifrange; + if (_config->FindB("aptwebserver::support::if-range", true) == true) + ifrange = LookupTag(*m, "If-Range", ""); + bool validrange = (ifrange.empty() == true || + (RFC1123StrToTime(ifrange.c_str(), cache) == true && + cache <= data.ModificationTime())); + + // FIXME: support multiple byte-ranges (APT clients do not do this) + if (condition.find(',') == std::string::npos) + { + size_t start = 6; + unsigned long long filestart = strtoull(condition.c_str() + start, NULL, 10); + // FIXME: no support for last-byte-pos being not the end of the file (APT clients do not do this) + size_t dash = condition.find('-') + 1; + unsigned long long fileend = strtoull(condition.c_str() + dash, NULL, 10); + unsigned long long filesize = data.FileSize(); + if ((fileend == 0 || (fileend == filesize && fileend >= filestart)) && + validrange == true) + { + if (filesize > filestart) + { + data.Skip(filestart); + std::ostringstream contentlength; + contentlength << "Content-Length: " << (filesize - filestart); + headers.push_back(contentlength.str()); + std::ostringstream contentrange; + contentrange << "Content-Range: bytes " << filestart << "-" + << filesize - 1 << "/" << filesize; + headers.push_back(contentrange.str()); + sendHead(client, 206, headers); + if (sendContent == true) + sendFile(client, data); + continue; + } + else + { + headers.push_back("Content-Length: 0"); + std::ostringstream contentrange; + contentrange << "Content-Range: bytes */" << filesize; + headers.push_back(contentrange.str()); + sendHead(client, 416, headers); + continue; + } + } + } + } + + addFileHeaders(headers, data); + sendHead(client, 200, headers); + if (sendContent == true) + sendFile(client, data); + } + else if (DirectoryExists(filename) == true) + { + if (filename[filename.length()-1] == '/') + sendDirectoryListing(client, filename, *m, sendContent); + else + sendRedirect(client, 301, filename.append("/"), *m, sendContent); + } + else + sendError(client, 404, *m, sendContent); + } + _error->DumpErrors(std::cerr); + messages.clear(); + if (closeConnection == true) + break; + } + close(client); + std::clog << "CLOSE client " << client << std::endl; + return NULL; +} + /*}}}*/ + int main(int const argc, const char * argv[]) { CommandLine::Args Args[] = { @@ -490,6 +657,9 @@ int main(int const argc, const char * argv[]) // create socket, bind and listen to it {{{ // ignore SIGPIPE, this can happen on write() if the socket closes connection signal(SIGPIPE, SIG_IGN); + // we don't care for our slaves, so ignore their death + signal(SIGCHLD, SIG_IGN); + int sock = socket(AF_INET6, SOCK_STREAM, 0); if(sock < 0) { @@ -557,179 +727,47 @@ int main(int const argc, const char * argv[]) std::clog << "Serving ANY file on port: " << port << std::endl; - listen(sock, 1); + int const slaves = _config->FindB("aptwebserver::slaves", SOMAXCONN); + listen(sock, slaves); /*}}}*/ _config->CndSet("aptwebserver::response-header::Server", "APT webserver"); _config->CndSet("aptwebserver::response-header::Accept-Ranges", "bytes"); _config->CndSet("aptwebserver::directoryindex", "index.html"); - std::vector messages; - int client; - while ((client = accept(sock, NULL, NULL)) != -1) - { - std::clog << "ACCEPT client " << client - << " on socket " << sock << std::endl; + std::list accepted_clients; - while (ReadMessages(client, messages)) + while (true) + { + int client = accept(sock, NULL, NULL); + if (client == -1) { - bool closeConnection = false; - for (std::vector::const_iterator m = messages.begin(); - m != messages.end() && closeConnection == false; ++m) { - std::clog << ">>> REQUEST >>>>" << std::endl << *m - << std::endl << "<<<<<<<<<<<<<<<<" << std::endl; - std::list headers; - std::string filename; - bool sendContent = true; - if (parseFirstLine(client, *m, filename, sendContent, closeConnection) == false) - continue; - - // special webserver command request - if (filename.length() > 1 && filename[0] == '_') - { - std::vector parts = VectorizeString(filename, '/'); - if (parts[0] == "_config") - { - handleOnTheFlyReconfiguration(client, *m, parts); - continue; - } - } - - // string replacements in the requested filename - ::Configuration::Item const *Replaces = _config->Tree("aptwebserver::redirect::replace"); - if (Replaces != NULL) - { - std::string redirect = "/" + filename; - for (::Configuration::Item *I = Replaces->Child; I != NULL; I = I->Next) - redirect = SubstVar(redirect, I->Tag, I->Value); - redirect.erase(0,1); - if (redirect != filename) - { - sendRedirect(client, 301, redirect, *m, sendContent); - continue; - } - } - - ::Configuration::Item const *Overwrite = _config->Tree("aptwebserver::overwrite"); - if (Overwrite != NULL) - { - for (::Configuration::Item *I = Overwrite->Child; I != NULL; I = I->Next) - { - regex_t *pattern = new regex_t; - int const res = regcomp(pattern, I->Tag.c_str(), REG_EXTENDED | REG_ICASE | REG_NOSUB); - if (res != 0) - { - char error[300]; - regerror(res, pattern, error, sizeof(error)); - sendError(client, 500, *m, sendContent, error); - continue; - } - if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0) - { - filename = _config->Find("aptwebserver::overwrite::" + I->Tag + "::filename", filename); - if (filename[0] == '/') - filename.erase(0,1); - regfree(pattern); - break; - } - regfree(pattern); - } - } - - // deal with the request - if (RealFileExists(filename) == true) - { - FileFd data(filename, FileFd::ReadOnly); - std::string condition = LookupTag(*m, "If-Modified-Since", ""); - if (condition.empty() == false) - { - time_t cache; - if (RFC1123StrToTime(condition.c_str(), cache) == true && - cache >= data.ModificationTime()) - { - sendHead(client, 304, headers); - continue; - } - } - - if (_config->FindB("aptwebserver::support::range", true) == true) - condition = LookupTag(*m, "Range", ""); - else - condition.clear(); - if (condition.empty() == false && strncmp(condition.c_str(), "bytes=", 6) == 0) - { - time_t cache; - std::string ifrange; - if (_config->FindB("aptwebserver::support::if-range", true) == true) - ifrange = LookupTag(*m, "If-Range", ""); - bool validrange = (ifrange.empty() == true || - (RFC1123StrToTime(ifrange.c_str(), cache) == true && - cache <= data.ModificationTime())); - - // FIXME: support multiple byte-ranges (APT clients do not do this) - if (condition.find(',') == std::string::npos) - { - size_t start = 6; - unsigned long long filestart = strtoull(condition.c_str() + start, NULL, 10); - // FIXME: no support for last-byte-pos being not the end of the file (APT clients do not do this) - size_t dash = condition.find('-') + 1; - unsigned long long fileend = strtoull(condition.c_str() + dash, NULL, 10); - unsigned long long filesize = data.FileSize(); - if ((fileend == 0 || (fileend == filesize && fileend >= filestart)) && - validrange == true) - { - if (filesize > filestart) - { - data.Skip(filestart); - std::ostringstream contentlength; - contentlength << "Content-Length: " << (filesize - filestart); - headers.push_back(contentlength.str()); - std::ostringstream contentrange; - contentrange << "Content-Range: bytes " << filestart << "-" - << filesize - 1 << "/" << filesize; - headers.push_back(contentrange.str()); - sendHead(client, 206, headers); - if (sendContent == true) - sendFile(client, data); - continue; - } - else - { - headers.push_back("Content-Length: 0"); - std::ostringstream contentrange; - contentrange << "Content-Range: bytes */" << filesize; - headers.push_back(contentrange.str()); - sendHead(client, 416, headers); - continue; - } - } - } - } + if (errno == EINTR) + continue; + _error->Errno("accept", "Couldn't accept client on socket %d", sock); + _error->DumpErrors(std::cerr); + return 6; + } - addFileHeaders(headers, data); - sendHead(client, 200, headers); - if (sendContent == true) - sendFile(client, data); - } - else if (DirectoryExists(filename) == true) - { - if (filename[filename.length()-1] == '/') - sendDirectoryListing(client, filename, *m, sendContent); - else - sendRedirect(client, 301, filename.append("/"), *m, sendContent); - } - else - sendError(client, 404, *m, sendContent); - } + pthread_attr_t attr; + if (pthread_attr_init(&attr) != 0 || pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) + { + _error->Errno("pthread_attr", "Couldn't set detach attribute for a fresh thread to handle client %d on socket %d", client, sock); _error->DumpErrors(std::cerr); - messages.clear(); - if (closeConnection == true) - break; + close(client); + continue; } - std::clog << "CLOSE client " << client - << " on socket " << sock << std::endl; - close(client); + pthread_t tid; + // thats rather dirty, but we need to store the client socket somewhere safe + accepted_clients.push_front(client); + if (pthread_create(&tid, &attr, &handleClient, &(*accepted_clients.begin())) != 0) + { + _error->Errno("pthread_create", "Couldn't create a fresh thread to handle client %d on socket %d", client, sock); + _error->DumpErrors(std::cerr); + close(client); + continue; + } } pidfile.Close(); -- cgit v1.2.3 From f2c0ec8bdb00b44de240190dae39fa255b6c85de Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 23 Oct 2013 16:32:48 +0200 Subject: tests: add a webserverconfig method to abstract config The URI to use to set a config option is a bit arcane to write/remember and checking if the setting was successful doubly so. Git-Dch: Ignore --- test/interactive-helper/aptwebserver.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/interactive-helper/aptwebserver.cc') diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 7d3589c9d..b7663a76a 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -546,7 +546,7 @@ void * handleClient(void * voidclient) /*{{{*/ { FileFd data(filename, FileFd::ReadOnly); std::string condition = LookupTag(*m, "If-Modified-Since", ""); - if (condition.empty() == false) + if (_config->FindB("aptwebserver::support::modified-since", true) == true && condition.empty() == false) { time_t cache; if (RFC1123StrToTime(condition.c_str(), cache) == true && -- cgit v1.2.3
#NameSizeLast-Modified
dParent Directory--