summaryrefslogtreecommitdiff
path: root/methods
diff options
context:
space:
mode:
Diffstat (limited to 'methods')
-rw-r--r--methods/cdrom.cc5
-rw-r--r--methods/copy.cc20
-rw-r--r--methods/file.cc87
-rw-r--r--methods/ftp.cc18
-rw-r--r--methods/gpgv.cc31
-rw-r--r--methods/gzip.cc2
-rw-r--r--methods/http.cc32
-rw-r--r--methods/http.h2
-rw-r--r--methods/https.cc184
-rw-r--r--methods/https.h30
-rw-r--r--methods/rred.cc130
-rw-r--r--methods/rsh.cc2
-rw-r--r--methods/server.cc82
-rw-r--r--methods/server.h15
14 files changed, 413 insertions, 227 deletions
diff --git a/methods/cdrom.cc b/methods/cdrom.cc
index 74e2ecc6b..67265cfa3 100644
--- a/methods/cdrom.cc
+++ b/methods/cdrom.cc
@@ -260,13 +260,14 @@ bool CDROMMethod::Fetch(FetchItem *Itm)
struct stat Buf;
if (stat(Res.Filename.c_str(),&Buf) != 0)
return _error->Error(_("File not found"));
-
+
+ URIStart(Res);
if (NewID.empty() == false)
CurrentID = NewID;
Res.LastModified = Buf.st_mtime;
Res.Size = Buf.st_size;
- Hashes Hash;
+ Hashes Hash(Itm->ExpectedHashes);
FileFd Fd(Res.Filename, FileFd::ReadOnly);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);
diff --git a/methods/copy.cc b/methods/copy.cc
index a23c0316c..a8e289df5 100644
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -28,16 +28,16 @@
class CopyMethod : public pkgAcqMethod
{
virtual bool Fetch(FetchItem *Itm);
- void CalculateHashes(FetchResult &Res);
+ void CalculateHashes(FetchItem const * const Itm, FetchResult &Res);
public:
CopyMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {};
};
-void CopyMethod::CalculateHashes(FetchResult &Res)
+void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
{
- Hashes Hash;
+ Hashes Hash(Itm->ExpectedHashes);
FileFd::CompressMode CompressMode = FileFd::None;
if (_config->FindB("Acquire::GzipIndexes", false) == true)
CompressMode = FileFd::Extension;
@@ -65,21 +65,13 @@ bool CopyMethod::Fetch(FetchItem *Itm)
Res.Size = Buf.st_size;
Res.Filename = Itm->DestFile;
Res.LastModified = Buf.st_mtime;
- Res.IMSHit = false;
+ Res.IMSHit = false;
URIStart(Res);
- // when the files are identical, just compute the hashes
- if(File == Itm->DestFile)
- {
- CalculateHashes(Res);
- URIDone(Res);
- return true;
- }
-
// just calc the hashes if the source and destination are identical
if (File == Itm->DestFile)
{
- CalculateHashes(Res);
+ CalculateHashes(Itm, Res);
URIDone(Res);
return true;
}
@@ -112,7 +104,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
if (utimes(Res.Filename.c_str(), times) != 0)
return _error->Errno("utimes",_("Failed to set modification time"));
- CalculateHashes(Res);
+ CalculateHashes(Itm, Res);
URIDone(Res);
return true;
diff --git a/methods/file.cc b/methods/file.cc
index 12db62203..5c76ec122 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -16,6 +16,7 @@
#include <config.h>
#include <apt-pkg/acquire-method.h>
+#include <apt-pkg/aptconfiguration.h>
#include <apt-pkg/error.h>
#include <apt-pkg/hashes.h>
#include <apt-pkg/fileutl.h>
@@ -33,7 +34,7 @@ class FileMethod : public pkgAcqMethod
public:
- FileMethod() : pkgAcqMethod("1.0",SingleInstance | LocalOnly) {};
+ FileMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig | LocalOnly) {};
};
// FileMethod::Fetch - Fetch a file /*{{{*/
@@ -47,8 +48,27 @@ bool FileMethod::Fetch(FetchItem *Itm)
if (Get.Host.empty() == false)
return _error->Error(_("Invalid URI, local URIS must not start with //"));
- // See if the file exists
struct stat Buf;
+ // deal with destination files which might linger around
+ if (lstat(Itm->DestFile.c_str(), &Buf) == 0)
+ {
+ if ((Buf.st_mode & S_IFREG) != 0)
+ {
+ if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+ {
+ HashStringList const hsl = Itm->ExpectedHashes;
+ if (Itm->ExpectedHashes.VerifyFile(File))
+ {
+ Res.Filename = Itm->DestFile;
+ Res.IMSHit = true;
+ }
+ }
+ }
+ }
+ if (Res.IMSHit != true)
+ unlink(Itm->DestFile.c_str());
+
+ // See if the file exists
if (stat(File.c_str(),&Buf) == 0)
{
Res.Size = Buf.st_size;
@@ -56,37 +76,50 @@ bool FileMethod::Fetch(FetchItem *Itm)
Res.LastModified = Buf.st_mtime;
Res.IMSHit = false;
if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
- Res.IMSHit = true;
+ {
+ unsigned long long const filesize = Itm->ExpectedHashes.FileSize();
+ if (filesize != 0 && filesize == Res.Size)
+ Res.IMSHit = true;
+ }
+
+ Hashes Hash(Itm->ExpectedHashes);
+ FileFd Fd(File, FileFd::ReadOnly);
+ Hash.AddFD(Fd);
+ Res.TakeHashes(Hash);
}
-
- // See if we can compute a file without a .gz exentsion
- std::string::size_type Pos = File.rfind(".gz");
- if (Pos + 3 == File.length())
+ if (Res.IMSHit == false)
+ URIStart(Res);
+
+ // See if the uncompressed file exists and reuse it
+ FetchResult AltRes;
+ AltRes.Filename.clear();
+ std::vector<std::string> extensions = APT::Configuration::getCompressorExtensions();
+ for (std::vector<std::string>::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext)
{
- File = std::string(File,0,Pos);
- if (stat(File.c_str(),&Buf) == 0)
+ if (APT::String::Endswith(File, *ext) == true)
{
- FetchResult AltRes;
- AltRes.Size = Buf.st_size;
- AltRes.Filename = File;
- AltRes.LastModified = Buf.st_mtime;
- AltRes.IMSHit = false;
- if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
- AltRes.IMSHit = true;
-
- URIDone(Res,&AltRes);
- return true;
- }
+ std::string const unfile = File.substr(0, File.length() - ext->length() - 1);
+ if (stat(unfile.c_str(),&Buf) == 0)
+ {
+ AltRes.Size = Buf.st_size;
+ AltRes.Filename = unfile;
+ AltRes.LastModified = Buf.st_mtime;
+ AltRes.IMSHit = false;
+ if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+ AltRes.IMSHit = true;
+ break;
+ }
+ // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
+ }
}
-
- if (Res.Filename.empty() == true)
+
+ if (AltRes.Filename.empty() == false)
+ URIDone(Res,&AltRes);
+ else if (Res.Filename.empty() == false)
+ URIDone(Res);
+ else
return _error->Error(_("File not found"));
- Hashes Hash;
- FileFd Fd(Res.Filename, FileFd::ReadOnly);
- Hash.AddFD(Fd);
- Res.TakeHashes(Hash);
- URIDone(Res);
return true;
}
/*}}}*/
diff --git a/methods/ftp.cc b/methods/ftp.cc
index 0504e5872..92d8573f1 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -259,19 +259,21 @@ bool FTPConn::Login()
{
if (Opts->Value.empty() == true)
continue;
-
+
// Substitute the variables into the command
- char SitePort[20];
- if (ServerName.Port != 0)
- sprintf(SitePort,"%u",ServerName.Port);
- else
- strcpy(SitePort,"21");
string Tmp = Opts->Value;
Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
- Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
+ if (ServerName.Port != 0)
+ {
+ std::string SitePort;
+ strprintf(SitePort, "%u", ServerName.Port);
+ Tmp = SubstVar(Tmp,"$(SITE_PORT)", SitePort);
+ }
+ else
+ Tmp = SubstVar(Tmp,"$(SITE_PORT)", "21");
Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
// Send the command
@@ -1062,7 +1064,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
}
// Open the file
- Hashes Hash;
+ Hashes Hash(Itm->ExpectedHashes);
{
FileFd Fd(Itm->DestFile,FileFd::WriteAny);
if (_error->PendingError() == true)
diff --git a/methods/gpgv.cc b/methods/gpgv.cc
index 488c16826..41f138be6 100644
--- a/methods/gpgv.cc
+++ b/methods/gpgv.cc
@@ -86,33 +86,12 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile,
FILE *pipein = fdopen(fd[0], "r");
// Loop over the output of apt-key (which really is gnupg), and check the signatures.
- size_t buffersize = 64;
- char *buffer = (char *) malloc(buffersize);
- size_t bufferoff = 0;
+ size_t buffersize = 0;
+ char *buffer = NULL;
while (1)
{
- int c;
-
- // Read a line. Sigh.
- while ((c = getc(pipein)) != EOF && c != '\n')
- {
- if (bufferoff == buffersize)
- {
- char* newBuffer = (char *) realloc(buffer, buffersize *= 2);
- if (newBuffer == NULL)
- {
- free(buffer);
- return "Couldn't allocate a buffer big enough for reading";
- }
- buffer = newBuffer;
- }
- *(buffer+bufferoff) = c;
- bufferoff++;
- }
- if (bufferoff == 0 && c == EOF)
- break;
- *(buffer+bufferoff) = '\0';
- bufferoff = 0;
+ if (getline(&buffer, &buffersize, pipein) == -1)
+ break;
if (Debug == true)
std::clog << "Read: " << buffer << std::endl;
@@ -126,7 +105,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile,
std::clog << "Got BADSIG! " << std::endl;
BadSigners.push_back(string(buffer+sizeof(GNUPGPREFIX)));
}
-
+
if (strncmp(buffer, GNUPGNOPUBKEY, sizeof(GNUPGNOPUBKEY)-1) == 0)
{
if (Debug == true)
diff --git a/methods/gzip.cc b/methods/gzip.cc
index 387c05f2e..65519633c 100644
--- a/methods/gzip.cc
+++ b/methods/gzip.cc
@@ -91,7 +91,7 @@ bool GzipMethod::Fetch(FetchItem *Itm)
return false;
// Read data from source, generate checksums and write
- Hashes Hash;
+ Hashes Hash(Itm->ExpectedHashes);
bool Failed = false;
while (1)
{
diff --git a/methods/http.cc b/methods/http.cc
index a5de13511..ce697a338 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -64,8 +64,8 @@ const unsigned int CircleBuf::BW_HZ=10;
// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
// ---------------------------------------------------------------------
/* */
-CircleBuf::CircleBuf(unsigned long long Size)
- : Size(Size), Hash(0), TotalWriten(0)
+CircleBuf::CircleBuf(unsigned long long Size)
+ : Size(Size), Hash(NULL), TotalWriten(0)
{
Buf = new unsigned char[Size];
Reset();
@@ -84,10 +84,10 @@ void CircleBuf::Reset()
TotalWriten = 0;
MaxGet = (unsigned long long)-1;
OutQueue = string();
- if (Hash != 0)
+ if (Hash != NULL)
{
delete Hash;
- Hash = new Hashes;
+ Hash = NULL;
}
}
/*}}}*/
@@ -222,7 +222,7 @@ bool CircleBuf::Write(int Fd)
TotalWriten += Res;
- if (Hash != 0)
+ if (Hash != NULL)
Hash->Add(Buf + (OutP%Size),Res);
OutP += Res;
@@ -442,10 +442,12 @@ bool HttpServerState::RunData(FileFd * const File)
{
/* Closes encoding is used when the server did not specify a size, the
loss of the connection means we are done */
- if (Encoding == Closes)
+ if (Persistent == false)
In.Limit(-1);
+ else if (JunkSize != 0)
+ In.Limit(JunkSize);
else
- In.Limit(Size - StartPos);
+ In.Limit(DownloadSize);
// Just transfer the whole block.
do
@@ -482,16 +484,14 @@ APT_PURE bool HttpServerState::IsOpen() /*{{{*/
return (ServerFd != -1);
}
/*}}}*/
-bool HttpServerState::InitHashes(FileFd &File) /*{{{*/
+bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
{
delete In.Hash;
- In.Hash = new Hashes;
-
- // Set the expected size and read file for the hashes
- File.Truncate(StartPos);
- return In.Hash->AddFD(File, StartPos);
+ In.Hash = new Hashes(ExpectedHashes);
+ return true;
}
/*}}}*/
+
APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/
{
return In.Hash;
@@ -522,7 +522,7 @@ bool HttpServerState::Die(FileFd &File)
// See if this is because the server finished the data stream
if (In.IsLimit() == false && State != HttpServerState::Header &&
- Encoding != HttpServerState::Closes)
+ Persistent == true)
{
Close();
if (LErrno == 0)
@@ -569,7 +569,7 @@ bool HttpServerState::Flush(FileFd * const File)
return true;
}
- if (In.IsLimit() == true || Encoding == ServerState::Closes)
+ if (In.IsLimit() == true || Persistent == false)
return true;
}
return false;
@@ -770,8 +770,6 @@ bool HttpMethod::Configuration(string Message)
if (ServerMethod::Configuration(Message) == false)
return false;
- DropPrivsOrDie();
-
AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
PipelineDepth);
diff --git a/methods/http.h b/methods/http.h
index 40a88a7be..e73871931 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -111,7 +111,7 @@ struct HttpServerState: public ServerState
virtual bool Open();
virtual bool IsOpen();
virtual bool Close();
- virtual bool InitHashes(FileFd &File);
+ virtual bool InitHashes(HashStringList const &ExpectedHashes);
virtual Hashes * GetHashes();
virtual bool Die(FileFd &File);
virtual bool Flush(FileFd * const File);
diff --git a/methods/https.cc b/methods/https.cc
index 366148e19..d2ddf6fcf 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -37,21 +37,19 @@
/*}}}*/
using namespace std;
-bool HttpsMethod::Configuration(std::string Message)
-{
- if (pkgAcqMethod::Configuration(Message) == false)
- return false;
-
- DropPrivsOrDie();
-
- return true;
-}
+struct APT_HIDDEN CURLUserPointer {
+ HttpsMethod * const https;
+ HttpsMethod::FetchResult * const Res;
+ HttpsMethod::FetchItem const * const Itm;
+ CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res,
+ HttpsMethod::FetchItem const * const Itm) : https(https), Res(Res), Itm(Itm) {}
+};
size_t
HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
{
size_t len = size * nmemb;
- HttpsMethod *me = (HttpsMethod *)userp;
+ CURLUserPointer *me = (CURLUserPointer *)userp;
std::string line((char*) buffer, len);
for (--len; len > 0; --len)
if (isspace(line[len]) == 0)
@@ -63,20 +61,52 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
if (line.empty() == true)
{
- if (me->Server->Result != 416 && me->Server->StartPos != 0)
+ if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0)
;
- else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize())
+ else if (me->https->Server->Result == 416)
{
- me->Server->Result = 200;
- me->Server->StartPos = me->Server->Size;
+ bool partialHit = false;
+ if (me->Itm->ExpectedHashes.usable() == true)
+ {
+ Hashes resultHashes(me->Itm->ExpectedHashes);
+ FileFd file(me->Itm->DestFile, FileFd::ReadOnly);
+ me->https->Server->TotalFileSize = file.FileSize();
+ me->https->Server->Date = file.ModificationTime();
+ resultHashes.AddFD(file);
+ HashStringList const hashList = resultHashes.GetHashStringList();
+ partialHit = (me->Itm->ExpectedHashes == hashList);
+ }
+ else if (me->https->Server->Result == 416 && me->https->Server->TotalFileSize == me->https->File->FileSize())
+ partialHit = true;
+
+ if (partialHit == true)
+ {
+ me->https->Server->Result = 200;
+ me->https->Server->StartPos = me->https->Server->TotalFileSize;
+ // the actual size is not important for https as curl will deal with it
+ // by itself and e.g. doesn't bother us with transport-encoding…
+ me->https->Server->JunkSize = std::numeric_limits<unsigned long long>::max();
+ }
+ else
+ me->https->Server->StartPos = 0;
}
else
- me->Server->StartPos = 0;
+ me->https->Server->StartPos = 0;
+
+ me->Res->LastModified = me->https->Server->Date;
+ me->Res->Size = me->https->Server->TotalFileSize;
+ me->Res->ResumePoint = me->https->Server->StartPos;
- me->File->Truncate(me->Server->StartPos);
- me->File->Seek(me->Server->StartPos);
+ // we expect valid data, so tell our caller we get the file now
+ if (me->https->Server->Result >= 200 && me->https->Server->Result < 300)
+ {
+ if (me->https->Server->JunkSize == 0 && me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
+ me->https->URIStart(*me->Res);
+ if (me->https->Server->AddPartialFileToHashes(*(me->https->File)) == false)
+ return 0;
+ }
}
- else if (me->Server->HeaderLine(line) == false)
+ else if (me->https->Server->HeaderLine(line) == false)
return 0;
return size*nmemb;
@@ -86,41 +116,54 @@ size_t
HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
{
HttpsMethod *me = (HttpsMethod *)userp;
+ size_t buffer_size = size * nmemb;
+ // we don't need to count the junk here, just drop anything we get as
+ // we don't always know how long it would be, e.g. in chunked encoding.
+ if (me->Server->JunkSize != 0)
+ return buffer_size;
- if (me->Res.Size == 0)
- me->URIStart(me->Res);
- if(me->File->Write(buffer, size*nmemb) != true)
- return false;
+ if(me->File->Write(buffer, buffer_size) != true)
+ return 0;
- if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize)
+ if(me->Queue->MaximumSize > 0)
{
- me->SetFailReason("MaximumSizeExceeded");
- return _error->Error("Writing more data than expected (%llu > %llu)",
- me->TotalWritten, me->Queue->MaximumSize);
+ unsigned long long const TotalWritten = me->File->Tell();
+ if (TotalWritten > me->Queue->MaximumSize)
+ {
+ me->SetFailReason("MaximumSizeExceeded");
+ _error->Error("Writing more data than expected (%llu > %llu)",
+ TotalWritten, me->Queue->MaximumSize);
+ return 0;
+ }
}
- return size*nmemb;
-}
-int
-HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
- double /*ultotal*/, double /*ulnow*/)
-{
- HttpsMethod *me = (HttpsMethod *)clientp;
- if(dltotal > 0 && me->Res.Size == 0) {
- me->Res.Size = (unsigned long long)dltotal;
- }
- return 0;
+ if (me->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false)
+ return 0;
+
+ return buffer_size;
}
// HttpsServerState::HttpsServerState - Constructor /*{{{*/
-HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL)
+HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner), Hash(NULL)
{
TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
Reset();
}
/*}}}*/
+bool HttpsServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
+{
+ delete Hash;
+ Hash = new Hashes(ExpectedHashes);
+ return true;
+}
+ /*}}}*/
+APT_PURE Hashes * HttpsServerState::GetHashes() /*{{{*/
+{
+ return Hash;
+}
+ /*}}}*/
-void HttpsMethod::SetupProxy() /*{{{*/
+void HttpsMethod::SetupProxy() /*{{{*/
{
URI ServerName = Queue->Uri;
@@ -183,7 +226,7 @@ void HttpsMethod::SetupProxy() /*{{{*/
bool HttpsMethod::Fetch(FetchItem *Itm)
{
struct stat SBuf;
- struct curl_slist *headers=NULL;
+ struct curl_slist *headers=NULL;
char curl_errorstr[CURL_ERROR_SIZE];
URI Uri = Itm->Uri;
string remotehost = Uri.Host;
@@ -198,16 +241,16 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
+ FetchResult Res;
+ CURLUserPointer userp(this, &Res, Itm);
// callbacks
curl_easy_setopt(curl, CURLOPT_URL, static_cast<string>(Uri).c_str());
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
- curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this);
+ curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &userp);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
- curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
- curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
// options
- curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
curl_easy_setopt(curl, CURLOPT_FILETIME, true);
// only allow curl to handle https, not the other stuff it supports
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
@@ -316,13 +359,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout);
// set redirect options and default to 10 redirects
- bool const AllowRedirect = _config->FindB("Acquire::https::AllowRedirect",
- _config->FindB("Acquire::http::AllowRedirect",true));
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10);
// debug
- if(_config->FindB("Debug::Acquire::https", false))
+ if (Debug == true)
curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
// error handling
@@ -359,7 +400,9 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
// go for it - if the file exists, append on it
File = new FileFd(Itm->DestFile, FileFd::WriteAny);
- Server = new HttpsServerState(Itm->Uri, this);
+ Server = CreateServerState(Itm->Uri);
+ if (Server->InitHashes(Itm->ExpectedHashes) == false)
+ return false;
// keep apt updated
Res.Filename = Itm->DestFile;
@@ -379,7 +422,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
if (success != 0)
{
_error->Error("%s", curl_errorstr);
- unlink(File->Name().c_str());
return false;
}
@@ -402,30 +444,29 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
char err[255];
snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result);
SetFailReason(err);
- _error->Error("%s", err);
+ _error->Error("%i %s", Server->Result, Server->Code);
// unlink, no need keep 401/404 page content in partial/
unlink(File->Name().c_str());
return false;
}
- struct stat resultStat;
- if (unlikely(stat(File->Name().c_str(), &resultStat) != 0))
- {
- _error->Errno("stat", "Unable to access file %s", File->Name().c_str());
- return false;
- }
- Res.Size = resultStat.st_size;
-
// invalid range-request
if (Server->Result == 416)
{
unlink(File->Name().c_str());
- Res.Size = 0;
delete File;
Redirect(Itm->Uri);
return true;
}
+ struct stat resultStat;
+ if (unlikely(stat(File->Name().c_str(), &resultStat) != 0))
+ {
+ _error->Errno("stat", "Unable to access file %s", File->Name().c_str());
+ return false;
+ }
+ Res.Size = resultStat.st_size;
+
// Timestamp
curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified);
if (Res.LastModified != -1)
@@ -440,20 +481,35 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
Res.LastModified = resultStat.st_mtime;
// take hashes
- Hashes Hash;
- FileFd Fd(Res.Filename, FileFd::ReadOnly);
- Hash.AddFD(Fd);
- Res.TakeHashes(Hash);
+ Res.TakeHashes(*(Server->GetHashes()));
// keep apt updated
URIDone(Res);
// cleanup
- Res.Size = 0;
delete File;
return true;
}
+ /*}}}*/
+// HttpsMethod::Configuration - Handle a configuration message /*{{{*/
+bool HttpsMethod::Configuration(string Message)
+{
+ if (ServerMethod::Configuration(Message) == false)
+ return false;
+
+ AllowRedirect = _config->FindB("Acquire::https::AllowRedirect",
+ _config->FindB("Acquire::http::AllowRedirect", true));
+ Debug = _config->FindB("Debug::Acquire::https",false);
+
+ return true;
+}
+ /*}}}*/
+ServerState * HttpsMethod::CreateServerState(URI uri) /*{{{*/
+{
+ return new HttpsServerState(uri, this);
+}
+ /*}}}*/
int main()
{
diff --git a/methods/https.h b/methods/https.h
index 9df18e83a..57fc292ee 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -29,6 +29,8 @@ class FileFd;
class HttpsServerState : public ServerState
{
+ Hashes * Hash;
+
protected:
virtual bool ReadHeaderLines(std::string &/*Data*/) { return false; }
virtual bool LoadNextResponse(bool const /*ToFile*/, FileFd * const /*File*/) { return false; }
@@ -42,8 +44,8 @@ class HttpsServerState : public ServerState
virtual bool Open() { return false; }
virtual bool IsOpen() { return false; }
virtual bool Close() { return false; }
- virtual bool InitHashes(FileFd &/*File*/) { return false; }
- virtual Hashes * GetHashes() { return NULL; }
+ virtual bool InitHashes(HashStringList const &ExpectedHashes);
+ virtual Hashes * GetHashes();
virtual bool Die(FileFd &/*File*/) { return false; }
virtual bool Flush(FileFd * const /*File*/) { return false; }
virtual bool Go(bool /*ToFile*/, FileFd * const /*File*/) { return false; }
@@ -52,28 +54,34 @@ class HttpsServerState : public ServerState
virtual ~HttpsServerState() {Close();};
};
-class HttpsMethod : public pkgAcqMethod
+class HttpsMethod : public ServerMethod
{
// minimum speed in bytes/se that triggers download timeout handling
static const int DL_MIN_SPEED = 10;
virtual bool Fetch(FetchItem *);
- virtual bool Configuration(std::string Message);
static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp);
static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
- static int progress_callback(void *clientp, double dltotal, double dlnow,
- double ultotal, double ulnow);
+ static int progress_callback(void *clientp, double dltotal, double dlnow,
+ double ultotal, double ulnow);
void SetupProxy();
CURL *curl;
- FetchResult Res;
- HttpsServerState *Server;
- unsigned long long TotalWritten;
+ ServerState *Server;
+
+ // Used by ServerMethods unused by https
+ virtual void SendReq(FetchItem *) { exit(42); }
+ virtual void RotateDNS() { exit(42); }
public:
FileFd *File;
-
- HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), Server(NULL), TotalWritten(0), File(NULL)
+
+ virtual bool Configuration(std::string Message);
+ virtual ServerState * CreateServerState(URI uri);
+ using pkgAcqMethod::FetchResult;
+ using pkgAcqMethod::FetchItem;
+
+ HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL)
{
curl = curl_easy_init();
};
diff --git a/methods/rred.cc b/methods/rred.cc
index cabb3c456..54123ab9c 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -21,6 +21,7 @@
#include <vector>
#include <assert.h>
+#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -35,7 +36,7 @@ class MemBlock {
char *start;
size_t size;
char *free;
- struct MemBlock *next;
+ MemBlock *next;
MemBlock(size_t size) : size(size), next(NULL)
{
@@ -116,7 +117,7 @@ struct Change {
size_t add_len; /* bytes */
char *add;
- Change(int off)
+ Change(size_t off)
{
offset = off;
del_cnt = add_cnt = add_len = 0;
@@ -150,11 +151,11 @@ class FileChanges {
std::list<struct Change>::iterator where;
size_t pos; // line number is as far left of iterator as possible
- bool pos_is_okay(void)
+ bool pos_is_okay(void) const
{
#ifdef POSDEBUG
size_t cpos = 0;
- std::list<struct Change>::iterator x;
+ std::list<struct Change>::const_iterator x;
for (x = changes.begin(); x != where; ++x) {
assert(x != changes.end());
cpos += x->offset + x->add_cnt;
@@ -388,28 +389,37 @@ class Patch {
public:
- void read_diff(FileFd &f)
+ bool read_diff(FileFd &f, Hashes * const h)
{
char buffer[BLOCK_SIZE];
bool cmdwanted = true;
- Change ch(0);
- while(f.ReadLine(buffer, sizeof(buffer)))
- {
+ Change ch(std::numeric_limits<size_t>::max());
+ if (f.ReadLine(buffer, sizeof(buffer)) == NULL)
+ return _error->Error("Reading first line of patchfile %s failed", f.Name().c_str());
+ do {
+ if (h != NULL)
+ h->Add(buffer);
if (cmdwanted) {
char *m, *c;
size_t s, e;
- s = strtol(buffer, &m, 10);
- if (m == buffer) {
- s = e = ch.offset + ch.add_cnt;
- c = buffer;
- } else if (*m == ',') {
- m++;
+ errno = 0;
+ s = strtoul(buffer, &m, 10);
+ if (unlikely(m == buffer || s == std::numeric_limits<unsigned long>::max() || errno != 0))
+ return _error->Error("Parsing patchfile %s failed: Expected an effected line start", f.Name().c_str());
+ else if (*m == ',') {
+ ++m;
e = strtol(m, &c, 10);
+ if (unlikely(m == c || e == std::numeric_limits<unsigned long>::max() || errno != 0))
+ return _error->Error("Parsing patchfile %s failed: Expected an effected line end", f.Name().c_str());
+ if (unlikely(e < s))
+ return _error->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f.Name().c_str(), e, s);
} else {
e = s;
c = m;
}
+ if (s > ch.offset)
+ return _error->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f.Name().c_str());
switch(*c) {
case 'a':
cmdwanted = false;
@@ -420,6 +430,8 @@ class Patch {
ch.del_cnt = 0;
break;
case 'c':
+ if (unlikely(s == 0))
+ return _error->Error("Parsing patchfile %s failed: Change command can't effect line zero", f.Name().c_str());
cmdwanted = false;
ch.add = NULL;
ch.add_cnt = 0;
@@ -428,6 +440,8 @@ class Patch {
ch.del_cnt = e - s + 1;
break;
case 'd':
+ if (unlikely(s == 0))
+ return _error->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f.Name().c_str());
ch.offset = s - 1;
ch.del_cnt = e - s + 1;
ch.add = NULL;
@@ -435,9 +449,11 @@ class Patch {
ch.add_len = 0;
filechanges.add_change(ch);
break;
+ default:
+ return _error->Error("Parsing patchfile %s failed: Unknown command", f.Name().c_str());
}
} else { /* !cmdwanted */
- if (buffer[0] == '.' && buffer[1] == '\n') {
+ if (strcmp(buffer, ".\n") == 0) {
cmdwanted = true;
filechanges.add_change(ch);
} else {
@@ -463,7 +479,8 @@ class Patch {
}
}
}
- }
+ } while(f.ReadLine(buffer, sizeof(buffer)));
+ return true;
}
void write_diff(FILE *f)
@@ -519,8 +536,29 @@ class RredMethod : public pkgAcqMethod {
private:
bool Debug;
+ struct PDiffFile {
+ std::string FileName;
+ HashStringList ExpectedHashes;
+ PDiffFile(std::string const &FileName, HashStringList const &ExpectedHashes) :
+ FileName(FileName), ExpectedHashes(ExpectedHashes) {}
+ };
+
+ HashStringList ReadExpectedHashesForPatch(unsigned int const patch, std::string const &Message)
+ {
+ HashStringList ExpectedHashes;
+ for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+ {
+ std::string tagname;
+ strprintf(tagname, "Patch-%d-%s-Hash", patch, *type);
+ std::string const hashsum = LookupTag(Message, tagname.c_str());
+ if (hashsum.empty() == false)
+ ExpectedHashes.push_back(HashString(*type, hashsum));
+ }
+ return ExpectedHashes;
+ }
+
protected:
- virtual bool Fetch(FetchItem *Itm) {
+ virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) {
Debug = _config->FindB("Debug::pkgAcquire::RRed", false);
URI Get = Itm->Uri;
std::string Path = Get.Host + Get.Path; // rred:/path - no host
@@ -534,11 +572,17 @@ class RredMethod : public pkgAcqMethod {
} else
URIStart(Res);
- std::vector<std::string> patchpaths;
+ std::vector<PDiffFile> patchfiles;
Patch patch;
if (FileExists(Path + ".ed") == true)
- patchpaths.push_back(Path + ".ed");
+ {
+ HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(0, Message);
+ std::string const FileName = Path + ".ed";
+ if (ExpectedHashes.usable() == false)
+ return _error->Error("No hashes found for uncompressed patch: %s", FileName.c_str());
+ patchfiles.push_back(PDiffFile(FileName, ExpectedHashes));
+ }
else
{
_error->PushToStack();
@@ -546,31 +590,44 @@ class RredMethod : public pkgAcqMethod {
_error->RevertToStack();
std::string const baseName = Path + ".ed.";
+ unsigned int seen_patches = 0;
for (std::vector<std::string>::const_iterator p = patches.begin();
p != patches.end(); ++p)
+ {
if (p->compare(0, baseName.length(), baseName) == 0)
- patchpaths.push_back(*p);
+ {
+ HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(seen_patches, Message);
+ if (ExpectedHashes.usable() == false)
+ return _error->Error("No hashes found for uncompressed patch %d: %s", seen_patches, p->c_str());
+ patchfiles.push_back(PDiffFile(*p, ExpectedHashes));
+ ++seen_patches;
+ }
+ }
}
std::string patch_name;
- for (std::vector<std::string>::iterator I = patchpaths.begin();
- I != patchpaths.end();
+ for (std::vector<PDiffFile>::iterator I = patchfiles.begin();
+ I != patchfiles.end();
++I)
{
- patch_name = *I;
+ patch_name = I->FileName;
if (Debug == true)
std::clog << "Patching " << Path << " with " << patch_name
<< std::endl;
FileFd p;
+ Hashes patch_hash(I->ExpectedHashes);
// all patches are compressed, even if the name doesn't reflect it
- if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false) {
- std::cerr << "Could not open patch file " << patch_name << std::endl;
+ if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false ||
+ patch.read_diff(p, &patch_hash) == false)
+ {
_error->DumpErrors(std::cerr);
- abort();
+ return false;
}
- patch.read_diff(p);
p.Close();
+ HashStringList const hsl = patch_hash.GetHashStringList();
+ if (hsl != I->ExpectedHashes)
+ return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
}
if (Debug == true)
@@ -581,8 +638,7 @@ class RredMethod : public pkgAcqMethod {
FILE *inp = fopen(Path.c_str(), "r");
FILE *out = fopen(Itm->DestFile.c_str(), "w");
- Hashes hash;
-
+ Hashes hash(Itm->ExpectedHashes);
patch.apply_against_file(out, inp, &hash);
fclose(out);
@@ -615,6 +671,16 @@ class RredMethod : public pkgAcqMethod {
return true;
}
+ bool Configuration(std::string Message)
+ {
+ if (pkgAcqMethod::Configuration(Message) == false)
+ return false;
+
+ DropPrivsOrDie();
+
+ return true;
+ }
+
public:
RredMethod() : pkgAcqMethod("2.0",SingleInstance | SendConfig), Debug(false) {}
};
@@ -643,7 +709,11 @@ int main(int argc, char **argv)
_error->DumpErrors(std::cerr);
exit(1);
}
- patch.read_diff(p);
+ if (patch.read_diff(p, NULL) == false)
+ {
+ _error->DumpErrors(std::cerr);
+ exit(2);
+ }
}
if (just_diff) {
diff --git a/methods/rsh.cc b/methods/rsh.cc
index 0e949160b..52349c61c 100644
--- a/methods/rsh.cc
+++ b/methods/rsh.cc
@@ -477,7 +477,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
}
// Open the file
- Hashes Hash;
+ Hashes Hash(Itm->ExpectedHashes);
{
FileFd Fd(Itm->DestFile,FileFd::WriteAny);
if (_error->PendingError() == true)
diff --git a/methods/server.cc b/methods/server.cc
index c4689ff12..f61a6fedb 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -54,7 +54,8 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
Major = 0;
Minor = 0;
Result = 0;
- Size = 0;
+ TotalFileSize = 0;
+ JunkSize = 0;
StartPos = 0;
Encoding = Closes;
HaveContent = false;
@@ -128,7 +129,7 @@ bool ServerState::HeaderLine(string Line)
if (elements == 3)
{
Code[0] = '\0';
- if (Owner->Debug == true)
+ if (Owner != NULL && Owner->Debug == true)
clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
}
else if (elements != 4)
@@ -163,15 +164,22 @@ bool ServerState::HeaderLine(string Line)
Encoding = Stream;
HaveContent = true;
- // The length is already set from the Content-Range header
- if (StartPos != 0)
- return true;
+ unsigned long long * DownloadSizePtr = &DownloadSize;
+ if (Result == 416)
+ DownloadSizePtr = &JunkSize;
- Size = strtoull(Val.c_str(), NULL, 10);
- if (Size >= std::numeric_limits<unsigned long long>::max())
+ *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
+ if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
- else if (Size == 0)
+ else if (*DownloadSizePtr == 0)
HaveContent = false;
+
+ // On partial content (206) the Content-Length less than the real
+ // size, so do not set it here but leave that to the Content-Range
+ // header instead
+ if(Result != 206 && TotalFileSize == 0)
+ TotalFileSize = DownloadSize;
+
return true;
}
@@ -186,15 +194,15 @@ bool ServerState::HeaderLine(string Line)
HaveContent = true;
// §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
- if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
- {
- StartPos = 1; // ignore Content-Length, it would override Size
- HaveContent = false;
- }
- else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
+ if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
+ ; // we got the expected filesize which is all we wanted
+ else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
- if ((unsigned long long)StartPos > Size)
+ if ((unsigned long long)StartPos > TotalFileSize)
return _error->Error(_("This HTTP server has broken range support"));
+
+ // figure out what we will download
+ DownloadSize = TotalFileSize - StartPos;
return true;
}
@@ -237,10 +245,21 @@ ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOu
Reset();
}
/*}}}*/
+bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
+{
+ File.Truncate(StartPos);
+ return GetHashes()->AddFD(File, StartPos);
+}
+ /*}}}*/
bool ServerMethod::Configuration(string Message) /*{{{*/
{
- return pkgAcqMethod::Configuration(Message);
+ if (pkgAcqMethod::Configuration(Message) == false)
+ return false;
+
+ DropPrivsOrDie();
+
+ return true;
}
/*}}}*/
@@ -260,7 +279,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
Res.LastModified = Queue->LastModified;
return IMS_HIT;
}
-
+
/* Redirect
*
* Note that it is only OK for us to treat all redirection the same
@@ -305,12 +324,31 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
struct stat SBuf;
if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
{
- if ((unsigned long long)SBuf.st_size == Server->Size)
+ bool partialHit = false;
+ if (Queue->ExpectedHashes.usable() == true)
+ {
+ Hashes resultHashes(Queue->ExpectedHashes);
+ FileFd file(Queue->DestFile, FileFd::ReadOnly);
+ Server->TotalFileSize = file.FileSize();
+ Server->Date = file.ModificationTime();
+ resultHashes.AddFD(file);
+ HashStringList const hashList = resultHashes.GetHashStringList();
+ partialHit = (Queue->ExpectedHashes == hashList);
+ }
+ else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
+ partialHit = true;
+ if (partialHit == true)
{
// the file is completely downloaded, but was not moved
- Server->StartPos = Server->Size;
- Server->Result = 200;
+ if (Server->HaveContent == true)
+ {
+ // Send to error page to dev/null
+ FileFd DevNull("/dev/null",FileFd::WriteExists);
+ Server->RunData(&DevNull);
+ }
Server->HaveContent = false;
+ Server->StartPos = Server->TotalFileSize;
+ Server->Result = 200;
}
else if (unlink(Queue->DestFile.c_str()) == 0)
{
@@ -335,7 +373,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
// This is some sort of 2xx 'data follows' reply
Res.LastModified = Server->Date;
- Res.Size = Server->Size;
+ Res.Size = Server->TotalFileSize;
// Open the file
delete File;
@@ -348,7 +386,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
FailFd = File->Fd();
FailTime = Server->Date;
- if (Server->InitHashes(*File) == false)
+ if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
{
_error->Errno("read",_("Problem hashing file"));
return ERROR_NOT_FROM_SERVER;
diff --git a/methods/server.h b/methods/server.h
index 7d5198478..8d7d33ee6 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,8 +34,16 @@ struct ServerState
char Code[360];
// These are some statistics from the last parsed header lines
- unsigned long long Size;
+
+ // total size of the usable content (aka: the file)
+ unsigned long long TotalFileSize;
+ // size we actually download (can be smaller than Size if we have partial content)
+ unsigned long long DownloadSize;
+ // size of junk content (aka: server error pages)
+ unsigned long long JunkSize;
+ // The start of the data (for partial content)
unsigned long long StartPos;
+
time_t Date;
bool HaveContent;
enum {Chunked,Stream,Closes} Encoding;
@@ -71,9 +79,10 @@ struct ServerState
};
/** \brief Get the headers before the data */
RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
+ bool AddPartialFileToHashes(FileFd &File);
bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
- virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+ virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; TotalFileSize = 0; JunkSize = 0;
StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;};
virtual bool WriteResponse(std::string const &Data) = 0;
@@ -84,7 +93,7 @@ struct ServerState
virtual bool Open() = 0;
virtual bool IsOpen() = 0;
virtual bool Close() = 0;
- virtual bool InitHashes(FileFd &File) = 0;
+ virtual bool InitHashes(HashStringList const &ExpectedHashes) = 0;
virtual Hashes * GetHashes() = 0;
virtual bool Die(FileFd &File) = 0;
virtual bool Flush(FileFd * const File) = 0;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 23:13:57 +0000