summaryrefslogtreecommitdiff
path: root/methods
diff options
context:
space:
mode:
Diffstat (limited to 'methods')
-rw-r--r--methods/connect.cc6
-rw-r--r--methods/copy.cc20
-rw-r--r--methods/file.cc2
-rw-r--r--methods/ftp.cc53
-rw-r--r--methods/ftp.h2
-rw-r--r--methods/gpgv.cc1
-rw-r--r--methods/gzip.cc24
-rw-r--r--methods/http.cc983
-rw-r--r--methods/http.h139
-rw-r--r--methods/https.cc151
-rw-r--r--methods/https.h33
-rw-r--r--methods/makefile6
-rw-r--r--methods/mirror.cc14
-rw-r--r--methods/mirror.h2
-rw-r--r--methods/rfc2553emu.h2
-rw-r--r--methods/rred.cc1185
-rw-r--r--methods/rsh.cc38
-rw-r--r--methods/server.cc668
-rw-r--r--methods/server.h144
19 files changed, 1929 insertions, 1544 deletions
diff --git a/methods/connect.cc b/methods/connect.cc
index fc7a72ee9..d9c9a1dd4 100644
--- a/methods/connect.cc
+++ b/methods/connect.cc
@@ -142,9 +142,9 @@ bool Connect(std::string Host,int Port,const char *Service,int DefPort,int &Fd,
// Convert the port name/number
char ServStr[300];
if (Port != 0)
- snprintf(ServStr,sizeof(ServStr),"%u",Port);
+ snprintf(ServStr,sizeof(ServStr),"%i", Port);
else
- snprintf(ServStr,sizeof(ServStr),"%s",Service);
+ snprintf(ServStr,sizeof(ServStr),"%s", Service);
/* We used a cached address record.. Yes this is against the spec but
the way we have setup our rotating dns suggests that this is more
@@ -190,7 +190,7 @@ bool Connect(std::string Host,int Port,const char *Service,int DefPort,int &Fd,
{
if (DefPort != 0)
{
- snprintf(ServStr,sizeof(ServStr),"%u",DefPort);
+ snprintf(ServStr, sizeof(ServStr), "%i", DefPort);
DefPort = 0;
continue;
}
diff --git a/methods/copy.cc b/methods/copy.cc
index e81d0022b..f2a8f9ed8 100644
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -18,7 +18,7 @@
#include <apt-pkg/hashes.h>
#include <sys/stat.h>
-#include <utime.h>
+#include <sys/time.h>
#include <unistd.h>
#include <apti18n.h>
/*}}}*/
@@ -72,17 +72,15 @@ bool CopyMethod::Fetch(FetchItem *Itm)
From.Close();
To.Close();
-
+
// Transfer the modification times
- struct utimbuf TimeBuf;
- TimeBuf.actime = Buf.st_atime;
- TimeBuf.modtime = Buf.st_mtime;
- if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0)
- {
- To.OpFail();
- return _error->Errno("utime",_("Failed to set modification time"));
- }
-
+ struct timeval times[2];
+ times[0].tv_sec = Buf.st_atime;
+ times[1].tv_sec = Buf.st_mtime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ if (utimes(Res.Filename.c_str(), times) != 0)
+ return _error->Errno("utimes",_("Failed to set modification time"));
+
Hashes Hash;
FileFd Fd(Res.Filename, FileFd::ReadOnly);
Hash.AddFD(Fd);
diff --git a/methods/file.cc b/methods/file.cc
index 7ed4e6f60..3d0687c5b 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -5,7 +5,7 @@
File URI method for APT
- This simply checks that the file specified exists, if so the relevent
+ This simply checks that the file specified exists, if so the relevant
information is returned. If a .gz filename is specified then the file
name with .gz removed will also be checked and information about it
will be returned in Alt-*
diff --git a/methods/ftp.cc b/methods/ftp.cc
index 979adca62..621f48476 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -3,7 +3,7 @@
// $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
/* ######################################################################
- FTP Aquire Method - This is the FTP aquire method for APT.
+ FTP Acquire Method - This is the FTP acquire method for APT.
This is a very simple implementation that does not try to optimize
at all. Commands are sent syncronously with the FTP server (as the
@@ -26,7 +26,6 @@
#include <sys/stat.h>
#include <sys/time.h>
-#include <utime.h>
#include <unistd.h>
#include <signal.h>
#include <stdio.h>
@@ -947,20 +946,22 @@ FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
/*}}}*/
// FtpMethod::SigTerm - Handle a fatal signal /*{{{*/
// ---------------------------------------------------------------------
-/* This closes and timestamps the open file. This is neccessary to get
+/* This closes and timestamps the open file. This is necessary to get
resume behavoir on user abort */
void FtpMethod::SigTerm(int)
{
if (FailFd == -1)
_exit(100);
- close(FailFd);
-
+
// Timestamp
- struct utimbuf UBuf;
- UBuf.actime = FailTime;
- UBuf.modtime = FailTime;
- utime(FailFile.c_str(),&UBuf);
-
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+
+ close(FailFd);
+
_exit(100);
}
/*}}}*/
@@ -1059,13 +1060,14 @@ bool FtpMethod::Fetch(FetchItem *Itm)
if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false)
{
Fd.Close();
-
+
// Timestamp
- struct utimbuf UBuf;
- UBuf.actime = FailTime;
- UBuf.modtime = FailTime;
- utime(FailFile.c_str(),&UBuf);
-
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+
// If the file is missing we hard fail and delete the destfile
// otherwise transient fail
if (Missing == true) {
@@ -1077,20 +1079,21 @@ bool FtpMethod::Fetch(FetchItem *Itm)
}
Res.Size = Fd.Size();
+
+ // Timestamp
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(Fd.Name().c_str(), times);
+ FailFd = -1;
}
-
+
Res.LastModified = FailTime;
Res.TakeHashes(Hash);
-
- // Timestamp
- struct utimbuf UBuf;
- UBuf.actime = FailTime;
- UBuf.modtime = FailTime;
- utime(Queue->DestFile.c_str(),&UBuf);
- FailFd = -1;
URIDone(Res);
-
+
return true;
}
/*}}}*/
diff --git a/methods/ftp.h b/methods/ftp.h
index 2634f0732..8055c389f 100644
--- a/methods/ftp.h
+++ b/methods/ftp.h
@@ -3,7 +3,7 @@
// $Id: ftp.h,v 1.4 2001/03/06 07:15:29 jgg Exp $
/* ######################################################################
- FTP Aquire Method - This is the FTP aquire method for APT.
+ FTP Acquire Method - This is the FTP acquire method for APT.
##################################################################### */
/*}}}*/
diff --git a/methods/gpgv.cc b/methods/gpgv.cc
index ea8a26fd4..25bf64ddd 100644
--- a/methods/gpgv.cc
+++ b/methods/gpgv.cc
@@ -8,7 +8,6 @@
#include <apt-pkg/configuration.h>
#include <apt-pkg/gpgv.h>
-#include <utime.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
diff --git a/methods/gzip.cc b/methods/gzip.cc
index 48c8e9892..a2844e969 100644
--- a/methods/gzip.cc
+++ b/methods/gzip.cc
@@ -18,8 +18,8 @@
#include <apt-pkg/hashes.h>
#include <sys/stat.h>
+#include <sys/time.h>
#include <unistd.h>
-#include <utime.h>
#include <stdio.h>
#include <errno.h>
#include <apti18n.h>
@@ -94,32 +94,28 @@ bool GzipMethod::Fetch(FetchItem *Itm)
}
From.Close();
+ Res.Size = To.FileSize();
To.Close();
-
+
if (Failed == true)
return false;
-
+
// Transfer the modification times
struct stat Buf;
if (stat(Path.c_str(),&Buf) != 0)
return _error->Errno("stat",_("Failed to stat"));
- struct utimbuf TimeBuf;
- TimeBuf.actime = Buf.st_atime;
- TimeBuf.modtime = Buf.st_mtime;
- if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0)
- return _error->Errno("utime",_("Failed to set modification time"));
+ struct timeval times[2];
+ times[0].tv_sec = Buf.st_atime;
+ Res.LastModified = times[1].tv_sec = Buf.st_mtime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ if (utimes(Itm->DestFile.c_str(), times) != 0)
+ return _error->Errno("utimes",_("Failed to set modification time"));
- if (stat(Itm->DestFile.c_str(),&Buf) != 0)
- return _error->Errno("stat",_("Failed to stat"));
-
// Return a Done response
- Res.LastModified = Buf.st_mtime;
- Res.Size = Buf.st_size;
Res.TakeHashes(Hash);
URIDone(Res);
-
return true;
}
/*}}}*/
diff --git a/methods/http.cc b/methods/http.cc
index 278ddb290..42b31beeb 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -3,7 +3,7 @@
// $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
/* ######################################################################
- HTTP Acquire Method - This is the HTTP aquire method for APT.
+ HTTP Acquire Method - This is the HTTP acquire method for APT.
It uses HTTP/1.1 and many of the fancy options there-in, such as
pipelining, range, if-range and so on.
@@ -36,7 +36,6 @@
#include <sys/stat.h>
#include <sys/time.h>
-#include <utime.h>
#include <unistd.h>
#include <signal.h>
#include <stdio.h>
@@ -58,15 +57,6 @@
/*}}}*/
using namespace std;
-string HttpMethod::FailFile;
-int HttpMethod::FailFd = -1;
-time_t HttpMethod::FailTime = 0;
-unsigned long PipelineDepth = 0;
-unsigned long TimeOut = 120;
-bool AllowRedirect = false;
-bool Debug = false;
-URI Proxy;
-
unsigned long long CircleBuf::BwReadLimit=0;
unsigned long long CircleBuf::BwTickReadData=0;
struct timeval CircleBuf::BwReadTick={0,0};
@@ -106,8 +96,6 @@ void CircleBuf::Reset()
is non-blocking.. */
bool CircleBuf::Read(int Fd)
{
- unsigned long long BwReadMax;
-
while (1)
{
// Woops, buffer is full
@@ -115,7 +103,7 @@ bool CircleBuf::Read(int Fd)
return true;
// what's left to read in this tick
- BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
+ unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
if(CircleBuf::BwReadLimit) {
struct timeval now;
@@ -296,20 +284,17 @@ CircleBuf::~CircleBuf()
delete Hash;
}
-// ServerState::ServerState - Constructor /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
- In(64*1024), Out(4*1024),
- ServerName(Srv)
+// HttpServerState::HttpServerState - Constructor /*{{{*/
+HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024)
{
+ TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
Reset();
}
/*}}}*/
-// ServerState::Open - Open a connection to the server /*{{{*/
+// HttpServerState::Open - Open a connection to the server /*{{{*/
// ---------------------------------------------------------------------
/* This opens a connection to the server. */
-bool ServerState::Open()
+bool HttpServerState::Open()
{
// Use the already open connection if possible.
if (ServerFd != -1)
@@ -373,72 +358,18 @@ bool ServerState::Open()
return true;
}
/*}}}*/
-// ServerState::Close - Close a connection to the server /*{{{*/
+// HttpServerState::Close - Close a connection to the server /*{{{*/
// ---------------------------------------------------------------------
/* */
-bool ServerState::Close()
+bool HttpServerState::Close()
{
close(ServerFd);
ServerFd = -1;
return true;
}
/*}}}*/
-// ServerState::RunHeaders - Get the headers before the data /*{{{*/
-// ---------------------------------------------------------------------
-/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
- parse error occurred */
-ServerState::RunHeadersResult ServerState::RunHeaders()
-{
- State = Header;
-
- Owner->Status(_("Waiting for headers"));
-
- Major = 0;
- Minor = 0;
- Result = 0;
- Size = 0;
- StartPos = 0;
- Encoding = Closes;
- HaveContent = false;
- time(&Date);
-
- do
- {
- string Data;
- if (In.WriteTillEl(Data) == false)
- continue;
-
- if (Debug == true)
- clog << Data;
-
- for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
- {
- string::const_iterator J = I;
- for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
- if (HeaderLine(string(I,J)) == false)
- return RUN_HEADERS_PARSE_ERROR;
- I = J;
- }
-
- // 100 Continue is a Nop...
- if (Result == 100)
- continue;
-
- // Tidy up the connection persistance state.
- if (Encoding == Closes && HaveContent == true)
- Persistent = false;
-
- return RUN_HEADERS_OK;
- }
- while (Owner->Go(false,this) == true);
-
- return RUN_HEADERS_IO_ERROR;
-}
- /*}}}*/
-// ServerState::RunData - Transfer the data from the socket /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool ServerState::RunData()
+// HttpServerState::RunData - Transfer the data from the socket /*{{{*/
+bool HttpServerState::RunData(FileFd * const File)
{
State = Data;
@@ -456,7 +387,7 @@ bool ServerState::RunData()
if (In.WriteTillEl(Data,true) == true)
break;
}
- while ((Last = Owner->Go(false,this)) == true);
+ while ((Last = Go(false, File)) == true);
if (Last == false)
return false;
@@ -474,7 +405,7 @@ bool ServerState::RunData()
if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
break;
}
- while ((Last = Owner->Go(false,this)) == true);
+ while ((Last = Go(false, File)) == true);
if (Last == false)
return false;
return !_error->PendingError();
@@ -482,7 +413,7 @@ bool ServerState::RunData()
// Transfer the block
In.Limit(Len);
- while (Owner->Go(true,this) == true)
+ while (Go(true, File) == true)
if (In.IsLimit() == true)
break;
@@ -498,7 +429,7 @@ bool ServerState::RunData()
if (In.WriteTillEl(Data,true) == true)
break;
}
- while ((Last = Owner->Go(false,this)) == true);
+ while ((Last = Go(false, File)) == true);
if (Last == false)
return false;
}
@@ -521,139 +452,217 @@ bool ServerState::RunData()
In.Limit(-1);
return !_error->PendingError();
}
- while (Owner->Go(true,this) == true);
+ while (Go(true, File) == true);
}
- return Owner->Flush(this) && !_error->PendingError();
+ return Owner->Flush() && !_error->PendingError();
}
/*}}}*/
-// ServerState::HeaderLine - Process a header line /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool ServerState::HeaderLine(string Line)
+bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
{
- if (Line.empty() == true)
- return true;
+ return In.WriteTillEl(Data);
+}
+ /*}}}*/
+bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
+{
+ return Go(ToFile, File);
+}
+ /*}}}*/
+bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
+{
+ return Out.Read(Data);
+}
+ /*}}}*/
+bool HttpServerState::IsOpen() /*{{{*/
+{
+ return (ServerFd != -1);
+}
+ /*}}}*/
+bool HttpServerState::InitHashes(FileFd &File) /*{{{*/
+{
+ delete In.Hash;
+ In.Hash = new Hashes;
- string::size_type Pos = Line.find(' ');
- if (Pos == string::npos || Pos+1 > Line.length())
+ // Set the expected size and read file for the hashes
+ if (StartPos >= 0)
{
- // Blah, some servers use "connection:closes", evil.
- Pos = Line.find(':');
- if (Pos == string::npos || Pos + 2 > Line.length())
- return _error->Error(_("Bad header line"));
- Pos++;
+ File.Truncate(StartPos);
+
+ return In.Hash->AddFD(File, StartPos);
}
+ return true;
+}
+ /*}}}*/
+Hashes * HttpServerState::GetHashes() /*{{{*/
+{
+ return In.Hash;
+}
+ /*}}}*/
+// HttpServerState::Die - The server has closed the connection. /*{{{*/
+bool HttpServerState::Die(FileFd &File)
+{
+ unsigned int LErrno = errno;
- // Parse off any trailing spaces between the : and the next word.
- string::size_type Pos2 = Pos;
- while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
- Pos2++;
-
- string Tag = string(Line,0,Pos);
- string Val = string(Line,Pos2);
-
- if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
+ // Dump the buffer to the file
+ if (State == ServerState::Data)
{
- // Evil servers return no version
- if (Line[4] == '/')
- {
- int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
- if (elements == 3)
- {
- Code[0] = '\0';
- if (Debug == true)
- clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
- }
- else if (elements != 4)
- return _error->Error(_("The HTTP server sent an invalid reply header"));
- }
- else
+ // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
+ // can't be set
+ if (File.Name() != "/dev/null")
+ SetNonBlock(File.Fd(),false);
+ while (In.WriteSpace() == true)
{
- Major = 0;
- Minor = 9;
- if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
- return _error->Error(_("The HTTP server sent an invalid reply header"));
- }
+ if (In.Write(File.Fd()) == false)
+ return _error->Errno("write",_("Error writing to the file"));
- /* Check the HTTP response header to get the default persistance
- state. */
- if (Major < 1)
- Persistent = false;
- else
- {
- if (Major == 1 && Minor == 0)
- Persistent = false;
- else
- Persistent = true;
+ // Done
+ if (In.IsLimit() == true)
+ return true;
}
+ }
- return true;
- }
-
- if (stringcasecmp(Tag,"Content-Length:") == 0)
+ // See if this is because the server finished the data stream
+ if (In.IsLimit() == false && State != HttpServerState::Header &&
+ Encoding != HttpServerState::Closes)
{
- if (Encoding == Closes)
- Encoding = Stream;
- HaveContent = true;
-
- // The length is already set from the Content-Range header
- if (StartPos != 0)
- return true;
+ Close();
+ if (LErrno == 0)
+ return _error->Error(_("Error reading from server. Remote end closed connection"));
+ errno = LErrno;
+ return _error->Errno("read",_("Error reading from server"));
+ }
+ else
+ {
+ In.Limit(-1);
- Size = strtoull(Val.c_str(), NULL, 10);
- if (Size >= std::numeric_limits<unsigned long long>::max())
- return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
+ // Nothing left in the buffer
+ if (In.WriteSpace() == false)
+ return false;
+
+ // We may have got multiple responses back in one packet..
+ Close();
return true;
}
- if (stringcasecmp(Tag,"Content-Type:") == 0)
+ return false;
+}
+ /*}}}*/
+// HttpServerState::Flush - Dump the buffer into the file /*{{{*/
+// ---------------------------------------------------------------------
+/* This takes the current input buffer from the Server FD and writes it
+ into the file */
+bool HttpServerState::Flush(FileFd * const File)
+{
+ if (File != NULL)
{
- HaveContent = true;
- return true;
+ // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
+ // can't be set
+ if (File->Name() != "/dev/null")
+ SetNonBlock(File->Fd(),false);
+ if (In.WriteSpace() == false)
+ return true;
+
+ while (In.WriteSpace() == true)
+ {
+ if (In.Write(File->Fd()) == false)
+ return _error->Errno("write",_("Error writing to file"));
+ if (In.IsLimit() == true)
+ return true;
+ }
+
+ if (In.IsLimit() == true || Encoding == ServerState::Closes)
+ return true;
}
+ return false;
+}
+ /*}}}*/
+// HttpServerState::Go - Run a single loop /*{{{*/
+// ---------------------------------------------------------------------
+/* This runs the select loop over the server FDs, Output file FDs and
+ stdin. */
+bool HttpServerState::Go(bool ToFile, FileFd * const File)
+{
+ // Server has closed the connection
+ if (ServerFd == -1 && (In.WriteSpace() == false ||
+ ToFile == false))
+ return false;
+
+ fd_set rfds,wfds;
+ FD_ZERO(&rfds);
+ FD_ZERO(&wfds);
+
+ /* Add the server. We only send more requests if the connection will
+ be persisting */
+ if (Out.WriteSpace() == true && ServerFd != -1
+ && Persistent == true)
+ FD_SET(ServerFd,&wfds);
+ if (In.ReadSpace() == true && ServerFd != -1)
+ FD_SET(ServerFd,&rfds);
+
+ // Add the file
+ int FileFD = -1;
+ if (File != NULL)
+ FileFD = File->Fd();
- if (stringcasecmp(Tag,"Content-Range:") == 0)
+ if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
+ FD_SET(FileFD,&wfds);
+
+ // Add stdin
+ if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
+ FD_SET(STDIN_FILENO,&rfds);
+
+ // Figure out the max fd
+ int MaxFd = FileFD;
+ if (MaxFd < ServerFd)
+ MaxFd = ServerFd;
+
+ // Select
+ struct timeval tv;
+ tv.tv_sec = TimeOut;
+ tv.tv_usec = 0;
+ int Res = 0;
+ if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
{
- HaveContent = true;
-
- if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
- return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
- if ((unsigned long long)StartPos > Size)
- return _error->Error(_("This HTTP server has broken range support"));
- return true;
+ if (errno == EINTR)
+ return true;
+ return _error->Errno("select",_("Select failed"));
}
- if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
+ if (Res == 0)
{
- HaveContent = true;
- if (stringcasecmp(Val,"chunked") == 0)
- Encoding = Chunked;
- return true;
+ _error->Error(_("Connection timed out"));
+ return Die(*File);
}
-
- if (stringcasecmp(Tag,"Connection:") == 0)
+
+ // Handle server IO
+ if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
{
- if (stringcasecmp(Val,"close") == 0)
- Persistent = false;
- if (stringcasecmp(Val,"keep-alive") == 0)
- Persistent = true;
- return true;
+ errno = 0;
+ if (In.Read(ServerFd) == false)
+ return Die(*File);
}
-
- if (stringcasecmp(Tag,"Last-Modified:") == 0)
+
+ if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
{
- if (RFC1123StrToTime(Val.c_str(), Date) == false)
- return _error->Error(_("Unknown date format"));
- return true;
+ errno = 0;
+ if (Out.Write(ServerFd) == false)
+ return Die(*File);
}
- if (stringcasecmp(Tag,"Location:") == 0)
+ // Send data to the file
+ if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
{
- Location = Val;
- return true;
+ if (In.Write(FileFD) == false)
+ return _error->Errno("write",_("Error writing to output file"));
}
+ // Handle commands from APT
+ if (FD_ISSET(STDIN_FILENO,&rfds))
+ {
+ if (Owner->Run(true) != -1)
+ exit(100);
+ }
+
return true;
}
/*}}}*/
@@ -661,7 +670,7 @@ bool ServerState::HeaderLine(string Line)
// HttpMethod::SendReq - Send the HTTP request /*{{{*/
// ---------------------------------------------------------------------
/* This places the http request in the outbound buffer */
-void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
+void HttpMethod::SendReq(FetchItem *Itm)
{
URI Uri = Itm->Uri;
@@ -687,7 +696,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
but while its a must for all servers to accept absolute URIs,
it is assumed clients will sent an absolute path for non-proxies */
std::string requesturi;
- if (Proxy.empty() == true || Proxy.Host.empty())
+ if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
requesturi = Uri.Path;
else
requesturi = Itm->Uri;
@@ -723,7 +732,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
}
// If we ask for uncompressed files servers might respond with content-
- // negotation which lets us end up with compressed files we do not support,
+ // negotiation which lets us end up with compressed files we do not support,
// see 657029, 657560 and co, so if we have no extension on the request
// ask for text only. As a sidenote: If there is nothing to negotate servers
// seem to be nice and ignore it.
@@ -742,7 +751,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
{
// In this case we send an if-range query with a range header
- sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size - 1,
+ sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size,
TimeRFC1123(SBuf.st_mtime).c_str());
Req += Buf;
}
@@ -755,9 +764,9 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
}
}
- if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
+ if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
Req += string("Proxy-Authorization: Basic ") +
- Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
+ Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) + "\r\n";
maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
if (Uri.User.empty() == false || Uri.Password.empty() == false)
@@ -771,344 +780,21 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
if (Debug == true)
cerr << Req << endl;
- Out.Read(Req);
+ Server->WriteResponse(Req);
}
/*}}}*/
-// HttpMethod::Go - Run a single loop /*{{{*/
-// ---------------------------------------------------------------------
-/* This runs the select loop over the server FDs, Output file FDs and
- stdin. */
-bool HttpMethod::Go(bool ToFile,ServerState *Srv)
-{
- // Server has closed the connection
- if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
- ToFile == false))
- return false;
-
- fd_set rfds,wfds;
- FD_ZERO(&rfds);
- FD_ZERO(&wfds);
-
- /* Add the server. We only send more requests if the connection will
- be persisting */
- if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
- && Srv->Persistent == true)
- FD_SET(Srv->ServerFd,&wfds);
- if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
- FD_SET(Srv->ServerFd,&rfds);
-
- // Add the file
- int FileFD = -1;
- if (File != 0)
- FileFD = File->Fd();
-
- if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
- FD_SET(FileFD,&wfds);
-
- // Add stdin
- if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
- FD_SET(STDIN_FILENO,&rfds);
-
- // Figure out the max fd
- int MaxFd = FileFD;
- if (MaxFd < Srv->ServerFd)
- MaxFd = Srv->ServerFd;
-
- // Select
- struct timeval tv;
- tv.tv_sec = TimeOut;
- tv.tv_usec = 0;
- int Res = 0;
- if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
- {
- if (errno == EINTR)
- return true;
- return _error->Errno("select",_("Select failed"));
- }
-
- if (Res == 0)
- {
- _error->Error(_("Connection timed out"));
- return ServerDie(Srv);
- }
-
- // Handle server IO
- if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
- {
- errno = 0;
- if (Srv->In.Read(Srv->ServerFd) == false)
- return ServerDie(Srv);
- }
-
- if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
- {
- errno = 0;
- if (Srv->Out.Write(Srv->ServerFd) == false)
- return ServerDie(Srv);
- }
-
- // Send data to the file
- if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
- {
- if (Srv->In.Write(FileFD) == false)
- return _error->Errno("write",_("Error writing to output file"));
- }
-
- // Handle commands from APT
- if (FD_ISSET(STDIN_FILENO,&rfds))
- {
- if (Run(true) != -1)
- exit(100);
- }
-
- return true;
-}
- /*}}}*/
-// HttpMethod::Flush - Dump the buffer into the file /*{{{*/
-// ---------------------------------------------------------------------
-/* This takes the current input buffer from the Server FD and writes it
- into the file */
-bool HttpMethod::Flush(ServerState *Srv)
-{
- if (File != 0)
- {
- // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
- // can't be set
- if (File->Name() != "/dev/null")
- SetNonBlock(File->Fd(),false);
- if (Srv->In.WriteSpace() == false)
- return true;
-
- while (Srv->In.WriteSpace() == true)
- {
- if (Srv->In.Write(File->Fd()) == false)
- return _error->Errno("write",_("Error writing to file"));
- if (Srv->In.IsLimit() == true)
- return true;
- }
-
- if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
- return true;
- }
- return false;
-}
- /*}}}*/
-// HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool HttpMethod::ServerDie(ServerState *Srv)
-{
- unsigned int LErrno = errno;
-
- // Dump the buffer to the file
- if (Srv->State == ServerState::Data)
- {
- // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
- // can't be set
- if (File->Name() != "/dev/null")
- SetNonBlock(File->Fd(),false);
- while (Srv->In.WriteSpace() == true)
- {
- if (Srv->In.Write(File->Fd()) == false)
- return _error->Errno("write",_("Error writing to the file"));
-
- // Done
- if (Srv->In.IsLimit() == true)
- return true;
- }
- }
-
- // See if this is because the server finished the data stream
- if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
- Srv->Encoding != ServerState::Closes)
- {
- Srv->Close();
- if (LErrno == 0)
- return _error->Error(_("Error reading from server. Remote end closed connection"));
- errno = LErrno;
- return _error->Errno("read",_("Error reading from server"));
- }
- else
- {
- Srv->In.Limit(-1);
-
- // Nothing left in the buffer
- if (Srv->In.WriteSpace() == false)
- return false;
-
- // We may have got multiple responses back in one packet..
- Srv->Close();
- return true;
- }
-
- return false;
-}
- /*}}}*/
-// HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
-// ---------------------------------------------------------------------
-/* We look at the header data we got back from the server and decide what
- to do. Returns DealWithHeadersResult (see http.h for details).
- */
-HttpMethod::DealWithHeadersResult
-HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
-{
- // Not Modified
- if (Srv->Result == 304)
- {
- unlink(Queue->DestFile.c_str());
- Res.IMSHit = true;
- Res.LastModified = Queue->LastModified;
- return IMS_HIT;
- }
-
- /* Redirect
- *
- * Note that it is only OK for us to treat all redirection the same
- * because we *always* use GET, not other HTTP methods. There are
- * three redirection codes for which it is not appropriate that we
- * redirect. Pass on those codes so the error handling kicks in.
- */
- if (AllowRedirect
- && (Srv->Result > 300 && Srv->Result < 400)
- && (Srv->Result != 300 // Multiple Choices
- && Srv->Result != 304 // Not Modified
- && Srv->Result != 306)) // (Not part of HTTP/1.1, reserved)
- {
- if (Srv->Location.empty() == true);
- else if (Srv->Location[0] == '/' && Queue->Uri.empty() == false)
- {
- URI Uri = Queue->Uri;
- if (Uri.Host.empty() == false)
- NextURI = URI::SiteOnly(Uri);
- else
- NextURI.clear();
- NextURI.append(DeQuoteString(Srv->Location));
- return TRY_AGAIN_OR_REDIRECT;
- }
- else
- {
- NextURI = DeQuoteString(Srv->Location);
- URI tmpURI = NextURI;
- // Do not allow a redirection to switch protocol
- if (tmpURI.Access == "http")
- return TRY_AGAIN_OR_REDIRECT;
- }
- /* else pass through for error message */
- }
-
- /* We have a reply we dont handle. This should indicate a perm server
- failure */
- if (Srv->Result < 200 || Srv->Result >= 300)
- {
- char err[255];
- snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result);
- SetFailReason(err);
- _error->Error("%u %s",Srv->Result,Srv->Code);
- if (Srv->HaveContent == true)
- return ERROR_WITH_CONTENT_PAGE;
- return ERROR_UNRECOVERABLE;
- }
-
- // This is some sort of 2xx 'data follows' reply
- Res.LastModified = Srv->Date;
- Res.Size = Srv->Size;
-
- // Open the file
- delete File;
- File = new FileFd(Queue->DestFile,FileFd::WriteAny);
- if (_error->PendingError() == true)
- return ERROR_NOT_FROM_SERVER;
-
- FailFile = Queue->DestFile;
- FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
- FailFd = File->Fd();
- FailTime = Srv->Date;
-
- delete Srv->In.Hash;
- Srv->In.Hash = new Hashes;
-
- // Set the expected size and read file for the hashes
- if (Srv->StartPos >= 0)
- {
- Res.ResumePoint = Srv->StartPos;
- File->Truncate(Srv->StartPos);
-
- if (Srv->In.Hash->AddFD(*File,Srv->StartPos) == false)
- {
- _error->Errno("read",_("Problem hashing file"));
- return ERROR_NOT_FROM_SERVER;
- }
- }
-
- SetNonBlock(File->Fd(),true);
- return FILE_IS_OPEN;
-}
- /*}}}*/
-// HttpMethod::SigTerm - Handle a fatal signal /*{{{*/
-// ---------------------------------------------------------------------
-/* This closes and timestamps the open file. This is neccessary to get
- resume behavoir on user abort */
-void HttpMethod::SigTerm(int)
-{
- if (FailFd == -1)
- _exit(100);
- close(FailFd);
-
- // Timestamp
- struct utimbuf UBuf;
- UBuf.actime = FailTime;
- UBuf.modtime = FailTime;
- utime(FailFile.c_str(),&UBuf);
-
- _exit(100);
-}
- /*}}}*/
-// HttpMethod::Fetch - Fetch an item /*{{{*/
-// ---------------------------------------------------------------------
-/* This adds an item to the pipeline. We keep the pipeline at a fixed
- depth. */
-bool HttpMethod::Fetch(FetchItem *)
-{
- if (Server == 0)
- return true;
-
- // Queue the requests
- int Depth = -1;
- for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
- I = I->Next, Depth++)
- {
- // If pipelining is disabled, we only queue 1 request
- if (Server->Pipeline == false && Depth >= 0)
- break;
-
- // Make sure we stick with the same server
- if (Server->Comp(I->Uri) == false)
- break;
- if (QueueBack == I)
- {
- QueueBack = I->Next;
- SendReq(I,Server->Out);
- continue;
- }
- }
-
- return true;
-};
- /*}}}*/
// HttpMethod::Configuration - Handle a configuration message /*{{{*/
// ---------------------------------------------------------------------
/* We stash the desired pipeline depth */
bool HttpMethod::Configuration(string Message)
{
- if (pkgAcqMethod::Configuration(Message) == false)
+ if (ServerMethod::Configuration(Message) == false)
return false;
-
+
AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
- TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
PipelineDepth);
Debug = _config->FindB("Debug::Acquire::http",false);
- AutoDetectProxyCmd = _config->Find("Acquire::http::ProxyAutoDetect");
// Get the proxy to use
AutoDetectProxy();
@@ -1116,263 +802,16 @@ bool HttpMethod::Configuration(string Message)
return true;
}
/*}}}*/
-// HttpMethod::Loop - Main loop /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-int HttpMethod::Loop()
-{
- typedef vector<string> StringVector;
- typedef vector<string>::iterator StringVectorIterator;
- map<string, StringVector> Redirected;
-
- signal(SIGTERM,SigTerm);
- signal(SIGINT,SigTerm);
-
- Server = 0;
-
- int FailCounter = 0;
- while (1)
- {
- // We have no commands, wait for some to arrive
- if (Queue == 0)
- {
- if (WaitFd(STDIN_FILENO) == false)
- return 0;
- }
-
- /* Run messages, we can accept 0 (no message) if we didn't
- do a WaitFd above.. Otherwise the FD is closed. */
- int Result = Run(true);
- if (Result != -1 && (Result != 0 || Queue == 0))
- {
- if(FailReason.empty() == false ||
- _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
- return 100;
- else
- return 0;
- }
-
- if (Queue == 0)
- continue;
-
- // Connect to the server
- if (Server == 0 || Server->Comp(Queue->Uri) == false)
- {
- delete Server;
- Server = new ServerState(Queue->Uri,this);
- }
- /* If the server has explicitly said this is the last connection
- then we pre-emptively shut down the pipeline and tear down
- the connection. This will speed up HTTP/1.0 servers a tad
- since we don't have to wait for the close sequence to
- complete */
- if (Server->Persistent == false)
- Server->Close();
-
- // Reset the pipeline
- if (Server->ServerFd == -1)
- QueueBack = Queue;
-
- // Connnect to the host
- if (Server->Open() == false)
- {
- Fail(true);
- delete Server;
- Server = 0;
- continue;
- }
-
- // Fill the pipeline.
- Fetch(0);
-
- // Fetch the next URL header data from the server.
- switch (Server->RunHeaders())
- {
- case ServerState::RUN_HEADERS_OK:
- break;
-
- // The header data is bad
- case ServerState::RUN_HEADERS_PARSE_ERROR:
- {
- _error->Error(_("Bad header data"));
- Fail(true);
- RotateDNS();
- continue;
- }
-
- // The server closed a connection during the header get..
- default:
- case ServerState::RUN_HEADERS_IO_ERROR:
- {
- FailCounter++;
- _error->Discard();
- Server->Close();
- Server->Pipeline = false;
-
- if (FailCounter >= 2)
- {
- Fail(_("Connection failed"),true);
- FailCounter = 0;
- }
-
- RotateDNS();
- continue;
- }
- };
-
- // Decide what to do.
- FetchResult Res;
- Res.Filename = Queue->DestFile;
- switch (DealWithHeaders(Res,Server))
- {
- // Ok, the file is Open
- case FILE_IS_OPEN:
- {
- URIStart(Res);
-
- // Run the data
- bool Result = Server->RunData();
-
- /* If the server is sending back sizeless responses then fill in
- the size now */
- if (Res.Size == 0)
- Res.Size = File->Size();
-
- // Close the file, destroy the FD object and timestamp it
- FailFd = -1;
- delete File;
- File = 0;
-
- // Timestamp
- struct utimbuf UBuf;
- time(&UBuf.actime);
- UBuf.actime = Server->Date;
- UBuf.modtime = Server->Date;
- utime(Queue->DestFile.c_str(),&UBuf);
-
- // Send status to APT
- if (Result == true)
- {
- Res.TakeHashes(*Server->In.Hash);
- URIDone(Res);
- }
- else
- {
- if (Server->ServerFd == -1)
- {
- FailCounter++;
- _error->Discard();
- Server->Close();
-
- if (FailCounter >= 2)
- {
- Fail(_("Connection failed"),true);
- FailCounter = 0;
- }
-
- QueueBack = Queue;
- }
- else
- Fail(true);
- }
- break;
- }
-
- // IMS hit
- case IMS_HIT:
- {
- URIDone(Res);
- break;
- }
-
- // Hard server error, not found or something
- case ERROR_UNRECOVERABLE:
- {
- Fail();
- break;
- }
-
- // Hard internal error, kill the connection and fail
- case ERROR_NOT_FROM_SERVER:
- {
- delete File;
- File = 0;
-
- Fail();
- RotateDNS();
- Server->Close();
- break;
- }
-
- // We need to flush the data, the header is like a 404 w/ error text
- case ERROR_WITH_CONTENT_PAGE:
- {
- Fail();
-
- // Send to content to dev/null
- File = new FileFd("/dev/null",FileFd::WriteExists);
- Server->RunData();
- delete File;
- File = 0;
- break;
- }
-
- // Try again with a new URL
- case TRY_AGAIN_OR_REDIRECT:
- {
- // Clear rest of response if there is content
- if (Server->HaveContent)
- {
- File = new FileFd("/dev/null",FileFd::WriteExists);
- Server->RunData();
- delete File;
- File = 0;
- }
-
- /* Detect redirect loops. No more redirects are allowed
- after the same URI is seen twice in a queue item. */
- StringVector &R = Redirected[Queue->DestFile];
- bool StopRedirects = false;
- if (R.empty() == true)
- R.push_back(Queue->Uri);
- else if (R[0] == "STOP" || R.size() > 10)
- StopRedirects = true;
- else
- {
- for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
- if (Queue->Uri == *I)
- {
- R[0] = "STOP";
- break;
- }
-
- R.push_back(Queue->Uri);
- }
-
- if (StopRedirects == false)
- Redirect(NextURI);
- else
- Fail();
-
- break;
- }
-
- default:
- Fail(_("Internal error"));
- break;
- }
-
- FailCounter = 0;
- }
-
- return 0;
-}
- /*}}}*/
// HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/
// ---------------------------------------------------------------------
/* */
bool HttpMethod::AutoDetectProxy()
{
+ // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
+ // name without the dash ("-")
+ AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect",
+ _config->Find("Acquire::http::ProxyAutoDetect"));
+
if (AutoDetectProxyCmd.empty())
return true;
@@ -1420,5 +859,13 @@ bool HttpMethod::AutoDetectProxy()
return true;
}
/*}}}*/
-
-
+ServerState * HttpMethod::CreateServerState(URI uri) /*{{{*/
+{
+ return new HttpServerState(uri, this);
+}
+ /*}}}*/
+void HttpMethod::RotateDNS() /*{{{*/
+{
+ ::RotateDNS();
+}
+ /*}}}*/
diff --git a/methods/http.h b/methods/http.h
index 7446119cd..450a42eed 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -3,7 +3,7 @@
// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $
/* ######################################################################
- HTTP Acquire Method - This is the HTTP aquire method for APT.
+ HTTP Acquire Method - This is the HTTP acquire method for APT.
##################################################################### */
/*}}}*/
@@ -15,6 +15,8 @@
#include <string>
+#include "server.h"
+
using std::cout;
using std::endl;
@@ -31,7 +33,7 @@ class CircleBuf
unsigned long long StrPos;
unsigned long long MaxGet;
struct timeval Start;
-
+
static unsigned long long BwReadLimit;
static unsigned long long BwTickReadData;
static struct timeval BwReadTick;
@@ -54,21 +56,20 @@ class CircleBuf
return Sz;
}
void FillOut();
-
+
public:
-
Hashes *Hash;
-
+
// Read data in
bool Read(int Fd);
bool Read(std::string Data);
-
+
// Write data out
bool Write(int Fd);
bool WriteTillEl(std::string &Data,bool Single = false);
-
+
// Control the write limit
- void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}
+ void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}
bool IsLimit() const {return MaxGet == OutP;};
void Print() const {cout << MaxGet << ',' << OutP << endl;};
@@ -84,114 +85,56 @@ class CircleBuf
~CircleBuf();
};
-struct ServerState
+struct HttpServerState: public ServerState
{
- // This is the last parsed Header Line
- unsigned int Major;
- unsigned int Minor;
- unsigned int Result;
- char Code[360];
-
- // These are some statistics from the last parsed header lines
- unsigned long long Size;
- signed long long StartPos;
- time_t Date;
- bool HaveContent;
- enum {Chunked,Stream,Closes} Encoding;
- enum {Header, Data} State;
- bool Persistent;
- std::string Location;
-
- // This is a Persistent attribute of the server itself.
- bool Pipeline;
-
- HttpMethod *Owner;
-
// This is the connection itself. Output is data FROM the server
CircleBuf In;
CircleBuf Out;
int ServerFd;
- URI ServerName;
-
- bool HeaderLine(std::string Line);
- bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
- void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
- StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
- State = Header; Persistent = false; ServerFd = -1;
- Pipeline = true;};
-
- /** \brief Result of the header acquire */
- enum RunHeadersResult {
- /** \brief Header ok */
- RUN_HEADERS_OK,
- /** \brief IO error while retrieving */
- RUN_HEADERS_IO_ERROR,
- /** \brief Parse error after retrieving */
- RUN_HEADERS_PARSE_ERROR,
- };
- /** \brief Get the headers before the data */
- RunHeadersResult RunHeaders();
- /** \brief Transfer the data from the socket */
- bool RunData();
-
- bool Open();
- bool Close();
-
- ServerState(URI Srv,HttpMethod *Owner);
- ~ServerState() {Close();};
+
+ protected:
+ virtual bool ReadHeaderLines(std::string &Data);
+ virtual bool LoadNextResponse(bool const ToFile, FileFd * const File);
+ virtual bool WriteResponse(std::string const &Data);
+
+ public:
+ virtual void Reset() { ServerState::Reset(); ServerFd = -1; };
+
+ virtual bool RunData(FileFd * const File);
+
+ virtual bool Open();
+ virtual bool IsOpen();
+ virtual bool Close();
+ virtual bool InitHashes(FileFd &File);
+ virtual Hashes * GetHashes();
+ virtual bool Die(FileFd &File);
+ virtual bool Flush(FileFd * const File);
+ virtual bool Go(bool ToFile, FileFd * const File);
+
+ HttpServerState(URI Srv, HttpMethod *Owner);
+ virtual ~HttpServerState() {Close();};
};
-class HttpMethod : public pkgAcqMethod
+class HttpMethod : public ServerMethod
{
- void SendReq(FetchItem *Itm,CircleBuf &Out);
- bool Go(bool ToFile,ServerState *Srv);
- bool Flush(ServerState *Srv);
- bool ServerDie(ServerState *Srv);
-
- /** \brief Result of the header parsing */
- enum DealWithHeadersResult {
- /** \brief The file is open and ready */
- FILE_IS_OPEN,
- /** \brief We got a IMS hit, the file has not changed */
- IMS_HIT,
- /** \brief The server reported a unrecoverable error */
- ERROR_UNRECOVERABLE,
- /** \brief The server reported a error with a error content page */
- ERROR_WITH_CONTENT_PAGE,
- /** \brief An error on the client side */
- ERROR_NOT_FROM_SERVER,
- /** \brief A redirect or retry request */
- TRY_AGAIN_OR_REDIRECT
- };
- /** \brief Handle the retrieved header data */
- DealWithHeadersResult DealWithHeaders(FetchResult &Res,ServerState *Srv);
+ public:
+ virtual void SendReq(FetchItem *Itm);
/** \brief Try to AutoDetect the proxy */
bool AutoDetectProxy();
virtual bool Configuration(std::string Message);
-
- // In the event of a fatal signal this file will be closed and timestamped.
- static std::string FailFile;
- static int FailFd;
- static time_t FailTime;
- static void SigTerm(int);
+
+ virtual ServerState * CreateServerState(URI uri);
+ virtual void RotateDNS();
protected:
- virtual bool Fetch(FetchItem *);
-
- std::string NextURI;
std::string AutoDetectProxyCmd;
public:
- friend struct ServerState;
-
- FileFd *File;
- ServerState *Server;
-
- int Loop();
-
- HttpMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig)
+ friend struct HttpServerState;
+
+ HttpMethod() : ServerMethod("1.2",Pipeline | SendConfig)
{
File = 0;
Server = 0;
diff --git a/methods/https.cc b/methods/https.cc
index 84ce2d68f..febe6a0f0 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -3,7 +3,7 @@
// $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
/* ######################################################################
- HTTPS Acquire Method - This is the HTTPS aquire method for APT.
+ HTTPS Acquire Method - This is the HTTPS acquire method for APT.
It uses libcurl
@@ -21,7 +21,6 @@
#include <sys/stat.h>
#include <sys/time.h>
-#include <utime.h>
#include <unistd.h>
#include <signal.h>
#include <stdio.h>
@@ -36,11 +35,48 @@
/*}}}*/
using namespace std;
+size_t
+HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
+{
+ size_t len = size * nmemb;
+ HttpsMethod *me = (HttpsMethod *)userp;
+ std::string line((char*) buffer, len);
+ for (--len; len > 0; --len)
+ if (isspace(line[len]) == 0)
+ {
+ ++len;
+ break;
+ }
+ line.erase(len);
+
+ if (line.empty() == true)
+ {
+ if (me->Server->Result != 416 && me->Server->StartPos != 0)
+ ;
+ else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize())
+ {
+ me->Server->Result = 200;
+ me->Server->StartPos = me->Server->Size;
+ }
+ else
+ me->Server->StartPos = 0;
+
+ me->File->Truncate(me->Server->StartPos);
+ me->File->Seek(me->Server->StartPos);
+ }
+ else if (me->Server->HeaderLine(line) == false)
+ return 0;
+
+ return size*nmemb;
+}
+
size_t
HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
{
HttpsMethod *me = (HttpsMethod *)userp;
+ if (me->Res.Size == 0)
+ me->URIStart(me->Res);
if(me->File->Write(buffer, size*nmemb) != true)
return false;
@@ -54,11 +90,18 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double dlnow,
HttpsMethod *me = (HttpsMethod *)clientp;
if(dltotal > 0 && me->Res.Size == 0) {
me->Res.Size = (unsigned long long)dltotal;
- me->URIStart(me->Res);
}
return 0;
}
+// HttpsServerState::HttpsServerState - Constructor /*{{{*/
+HttpsServerState::HttpsServerState(URI Srv,HttpsMethod *Owner) : ServerState(Srv, NULL)
+{
+ TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
+ Reset();
+}
+ /*}}}*/
+
void HttpsMethod::SetupProxy() /*{{{*/
{
URI ServerName = Queue->Uri;
@@ -121,7 +164,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
struct stat SBuf;
struct curl_slist *headers=NULL;
char curl_errorstr[CURL_ERROR_SIZE];
- long curl_responsecode;
URI Uri = Itm->Uri;
string remotehost = Uri.Host;
@@ -137,12 +179,18 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
// callbacks
curl_easy_setopt(curl, CURLOPT_URL, static_cast<string>(Uri).c_str());
+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
+ curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
+ // options
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_FILETIME, true);
+ // only allow curl to handle https, not the other stuff it supports
+ curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
+ curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS);
// SSL parameters are set by default to the common (non mirror-specific) value
// if available (or a default one) and gets overload by mirror-specific ones.
@@ -261,7 +309,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
// If we ask for uncompressed files servers might respond with content-
- // negotation which lets us end up with compressed files we do not support,
+ // negotiation which lets us end up with compressed files we do not support,
// see 657029, 657560 and co, so if we have no extension on the request
// ask for text only. As a sidenote: If there is nothing to negotate servers
// seem to be nice and ignore it.
@@ -277,7 +325,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
{
char Buf[1000];
- sprintf(Buf, "Range: bytes=%li-", (long) SBuf.st_size - 1);
+ sprintf(Buf, "Range: bytes=%li-", (long) SBuf.st_size);
headers = curl_slist_append(headers, Buf);
sprintf(Buf, "If-Range: %s", TimeRFC1123(SBuf.st_mtime).c_str());
headers = curl_slist_append(headers, Buf);
@@ -290,18 +338,13 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
// go for it - if the file exists, append on it
File = new FileFd(Itm->DestFile, FileFd::WriteAny);
- if (File->Size() > 0)
- File->Seek(File->Size() - 1);
-
+ Server = new HttpsServerState(Itm->Uri, this);
+
// keep apt updated
Res.Filename = Itm->DestFile;
// get it!
CURLcode success = curl_easy_perform(curl);
- curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &curl_responsecode);
-
- long curl_servdate;
- curl_easy_getinfo(curl, CURLINFO_FILETIME, &curl_servdate);
// If the server returns 200 OK but the If-Modified-Since condition is not
// met, CURLINFO_CONDITION_UNMET will be set to 1
@@ -309,57 +352,84 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &curl_condition_unmet);
File->Close();
+ curl_slist_free_all(headers);
// cleanup
- if(success != 0 || (curl_responsecode != 200 && curl_responsecode != 304))
+ if (success != 0)
{
_error->Error("%s", curl_errorstr);
- // unlink, no need keep 401/404 page content in partial/
unlink(File->Name().c_str());
- Fail();
+ return false;
+ }
+
+ // server says file not modified
+ if (Server->Result == 304 || curl_condition_unmet == 1)
+ {
+ unlink(File->Name().c_str());
+ Res.IMSHit = true;
+ Res.LastModified = Itm->LastModified;
+ Res.Size = 0;
+ URIDone(Res);
return true;
}
+ Res.IMSHit = false;
- // Timestamp
- struct utimbuf UBuf;
- if (curl_servdate != -1) {
- UBuf.actime = curl_servdate;
- UBuf.modtime = curl_servdate;
- utime(File->Name().c_str(),&UBuf);
+ if (Server->Result != 200 && // OK
+ Server->Result != 206 && // Partial
+ Server->Result != 416) // invalid Range
+ {
+ char err[255];
+ snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result);
+ SetFailReason(err);
+ _error->Error("%s", err);
+ // unlink, no need keep 401/404 page content in partial/
+ unlink(File->Name().c_str());
+ return false;
}
- // check the downloaded result
- struct stat Buf;
- if (stat(File->Name().c_str(),&Buf) == 0)
+ struct stat resultStat;
+ if (unlikely(stat(File->Name().c_str(), &resultStat) != 0))
{
- Res.Filename = File->Name();
- Res.LastModified = Buf.st_mtime;
- Res.IMSHit = false;
- if (curl_responsecode == 304 || curl_condition_unmet)
- {
- unlink(File->Name().c_str());
- Res.IMSHit = true;
- Res.LastModified = Itm->LastModified;
- Res.Size = 0;
- URIDone(Res);
- return true;
- }
- Res.Size = Buf.st_size;
+ _error->Errno("stat", "Unable to access file %s", File->Name().c_str());
+ return false;
}
+ Res.Size = resultStat.st_size;
+
+ // invalid range-request
+ if (Server->Result == 416)
+ {
+ unlink(File->Name().c_str());
+ Res.Size = 0;
+ delete File;
+ Redirect(Itm->Uri);
+ return true;
+ }
+
+ // Timestamp
+ curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified);
+ if (Res.LastModified != -1)
+ {
+ struct timeval times[2];
+ times[0].tv_sec = Res.LastModified;
+ times[1].tv_sec = Res.LastModified;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(File->Name().c_str(), times);
+ }
+ else
+ Res.LastModified = resultStat.st_mtime;
// take hashes
Hashes Hash;
FileFd Fd(Res.Filename, FileFd::ReadOnly);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);
-
+
// keep apt updated
URIDone(Res);
// cleanup
Res.Size = 0;
delete File;
- curl_slist_free_all(headers);
return true;
};
@@ -374,4 +444,3 @@ int main()
return Mth.Run();
}
-
diff --git a/methods/https.h b/methods/https.h
index 293e288e0..ab0dd3407 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -3,7 +3,7 @@
// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $
/* ######################################################################
- HTTP Acquire Method - This is the HTTP aquire method for APT.
+ HTTP Acquire Method - This is the HTTP acquire method for APT.
##################################################################### */
/*}}}*/
@@ -14,29 +14,58 @@
#include <iostream>
#include <curl/curl.h>
+#include "server.h"
+
using std::cout;
using std::endl;
class HttpsMethod;
class FileFd;
+class HttpsServerState : public ServerState
+{
+ protected:
+ virtual bool ReadHeaderLines(std::string &Data) { return false; }
+ virtual bool LoadNextResponse(bool const ToFile, FileFd * const File) { return false; }
+
+ public:
+ virtual bool WriteResponse(std::string const &Data) { return false; }
+
+ /** \brief Transfer the data from the socket */
+ virtual bool RunData(FileFd * const File) { return false; }
+
+ virtual bool Open() { return false; }
+ virtual bool IsOpen() { return false; }
+ virtual bool Close() { return false; }
+ virtual bool InitHashes(FileFd &File) { return false; }
+ virtual Hashes * GetHashes() { return NULL; }
+ virtual bool Die(FileFd &File) { return false; }
+ virtual bool Flush(FileFd * const File) { return false; }
+ virtual bool Go(bool ToFile, FileFd * const File) { return false; }
+
+ HttpsServerState(URI Srv, HttpsMethod *Owner);
+ virtual ~HttpsServerState() {Close();};
+};
+
class HttpsMethod : public pkgAcqMethod
{
// minimum speed in bytes/se that triggers download timeout handling
static const int DL_MIN_SPEED = 10;
virtual bool Fetch(FetchItem *);
+ static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp);
static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
static int progress_callback(void *clientp, double dltotal, double dlnow,
double ultotal, double ulnow);
void SetupProxy();
CURL *curl;
FetchResult Res;
+ HttpsServerState *Server;
public:
FileFd *File;
- HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig)
+ HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), File(NULL)
{
File = 0;
curl = curl_easy_init();
diff --git a/methods/makefile b/methods/makefile
index 294c55d23..6b7781294 100644
--- a/methods/makefile
+++ b/methods/makefile
@@ -48,14 +48,14 @@ include $(PROGRAM_H)
PROGRAM=http
SLIBS = -lapt-pkg $(SOCKETLIBS) $(INTLLIBS)
LIB_MAKES = apt-pkg/makefile
-SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc
+SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc server.cc
include $(PROGRAM_H)
# The https method
PROGRAM=https
SLIBS = -lapt-pkg -lcurl $(INTLLIBS)
LIB_MAKES = apt-pkg/makefile
-SOURCE = https.cc
+SOURCE = https.cc server.cc
include $(PROGRAM_H)
# The ftp method
@@ -83,7 +83,7 @@ include $(PROGRAM_H)
PROGRAM=mirror
SLIBS = -lapt-pkg $(SOCKETLIBS)
LIB_MAKES = apt-pkg/makefile
-SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc
+SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc server.cc
include $(PROGRAM_H)
# SSH method symlink
diff --git a/methods/mirror.cc b/methods/mirror.cc
index 854366318..085f3717b 100644
--- a/methods/mirror.cc
+++ b/methods/mirror.cc
@@ -3,7 +3,7 @@
// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
/* ######################################################################
- Mirror Aquire Method - This is the Mirror aquire method for APT.
+ Mirror Acquire Method - This is the Mirror acquire method for APT.
##################################################################### */
/*}}}*/
@@ -49,7 +49,7 @@ using namespace std;
* of the failure that is also send to LP
*
* TODO:
- * - deal with runing as non-root because we can't write to the lists
+ * - deal with running as non-root because we can't write to the lists
dir then -> use the cached mirror file
* - better method to download than having a pkgAcquire interface here
* and better error handling there!
@@ -114,7 +114,7 @@ bool MirrorMethod::Clean(string Dir)
for(I=list.begin(); I != list.end(); ++I)
{
string uri = (*I)->GetURI();
- if(uri.find("mirror://") != 0)
+ if(uri.compare(0, strlen("mirror://"), "mirror://") != 0)
continue;
string BaseUri = uri.substr(0,uri.size()-1);
if (URItoFileName(BaseUri) == Dir->d_name)
@@ -198,9 +198,9 @@ bool MirrorMethod::RandomizeMirrorFile(string mirror_file)
// "stable" on the same machine. this is to avoid running into out-of-sync
// issues (i.e. Release/Release.gpg different on each mirror)
struct utsname buf;
- int seed=1, i;
+ int seed=1;
if(uname(&buf) == 0) {
- for(i=0,seed=1; buf.nodename[i] != 0; i++) {
+ for(int i=0,seed=1; buf.nodename[i] != 0; ++i) {
seed = seed * 31 + buf.nodename[i];
}
}
@@ -290,7 +290,7 @@ bool MirrorMethod::InitMirrors()
// FIXME: make the mirror selection more clever, do not
// just use the first one!
// BUT: we can not make this random, the mirror has to be
- // stable accross session, because otherwise we can
+ // stable across session, because otherwise we can
// get into sync issues (got indexfiles from mirror A,
// but packages from mirror B - one might be out of date etc)
ifstream in(MirrorFile.c_str());
@@ -306,7 +306,7 @@ bool MirrorMethod::InitMirrors()
if (s.size() == 0)
continue;
// ignore non http lines
- if (s.find("http://") != 0)
+ if (s.compare(0, strlen("http://"), "http://") != 0)
continue;
AllMirrors.push_back(s);
diff --git a/methods/mirror.h b/methods/mirror.h
index 81e531e21..1dd9f2ec6 100644
--- a/methods/mirror.h
+++ b/methods/mirror.h
@@ -3,7 +3,7 @@
// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $
/* ######################################################################
- MIRROR Aquire Method - This is the MIRROR aquire method for APT.
+ MIRROR Acquire Method - This is the MIRROR acquire method for APT.
##################################################################### */
/*}}}*/
diff --git a/methods/rfc2553emu.h b/methods/rfc2553emu.h
index b15facb31..ad7ddf48a 100644
--- a/methods/rfc2553emu.h
+++ b/methods/rfc2553emu.h
@@ -75,7 +75,7 @@
#endif
/* If we don't have getaddrinfo then we probably don't have
- sockaddr_storage either (same RFC) so we definately will not be
+ sockaddr_storage either (same RFC) so we definitely will not be
doing any IPv6 stuff. Do not use the members of this structure to
retain portability, cast to a sockaddr. */
#define sockaddr_storage sockaddr_in
diff --git a/methods/rred.cc b/methods/rred.cc
index 7c65f8f92..fe7ef7322 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -1,4 +1,10 @@
-// Includes /*{{{*/
+// Copyright (c) 2014 Anthony Towns
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+
#include <config.h>
#include <apt-pkg/fileutl.h>
@@ -9,563 +15,646 @@
#include <apt-pkg/hashes.h>
#include <apt-pkg/configuration.h>
-#include <sys/stat.h>
-#include <sys/uio.h>
-#include <unistd.h>
-#include <utime.h>
+#include <string>
+#include <list>
+#include <vector>
+#include <iterator>
+
+#include <fcntl.h>
+#include <assert.h>
#include <stdio.h>
-#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+
#include <apti18n.h>
- /*}}}*/
-/** \brief RredMethod - ed-style incremential patch method {{{
- *
- * This method implements a patch functionality similar to "patch --ed" that is
- * used by the "tiffany" incremental packages download stuff. It differs from
- * "ed" insofar that it is way more restricted (and therefore secure).
- * The currently supported ed commands are "<em>c</em>hange", "<em>a</em>dd" and
- * "<em>d</em>elete" (diff doesn't output any other).
- * Additionally the records must be reverse sorted by line number and
- * may not overlap (diff *seems* to produce this kind of output).
- * */
-class RredMethod : public pkgAcqMethod {
- bool Debug;
- // the size of this doesn't really matter (except for performance)
- const static int BUF_SIZE = 1024;
- // the supported ed commands
- enum Mode {MODE_CHANGED='c', MODE_DELETED='d', MODE_ADDED='a'};
- // return values
- enum State {ED_OK, ED_ORDERING, ED_PARSER, ED_FAILURE, MMAP_FAILED};
-
- State applyFile(FileFd &ed_cmds, FileFd &in_file, FileFd &out_file,
- unsigned long &line, char *buffer, Hashes *hash) const;
- void ignoreLineInFile(FileFd &fin, char *buffer) const;
- void copyLinesFromFileToFile(FileFd &fin, FileFd &fout, unsigned int lines,
- Hashes *hash, char *buffer) const;
-
- State patchFile(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const;
- State patchMMap(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const;
-
-protected:
- // the methods main method
- virtual bool Fetch(FetchItem *Itm);
-
-public:
- RredMethod() : pkgAcqMethod("1.1",SingleInstance | SendConfig), Debug(false) {};
+
+#define BLOCK_SIZE (512*1024)
+
+class MemBlock {
+ char *start;
+ size_t size;
+ char *free;
+ struct MemBlock *next;
+
+ MemBlock(size_t size) : size(size), next(NULL)
+ {
+ free = start = new char[size];
+ }
+
+ size_t avail(void) { return size - (free - start); }
+
+ public:
+
+ MemBlock(void) {
+ free = start = new char[BLOCK_SIZE];
+ size = BLOCK_SIZE;
+ next = NULL;
+ }
+
+ ~MemBlock() {
+ delete [] start;
+ delete next;
+ }
+
+ void clear(void) {
+ free = start;
+ if (next)
+ next->clear();
+ }
+
+ char *add_easy(char *src, size_t len, char *last)
+ {
+ if (last) {
+ for (MemBlock *k = this; k; k = k->next) {
+ if (k->free == last) {
+ if (len <= k->avail()) {
+ char *n = k->add(src, len);
+ assert(last == n);
+ if (last == n)
+ return NULL;
+ return n;
+ } else {
+ break;
+ }
+ } else if (last >= start && last < free) {
+ break;
+ }
+ }
+ }
+ return add(src, len);
+ }
+
+ char *add(char *src, size_t len) {
+ if (len > avail()) {
+ if (!next) {
+ if (len > BLOCK_SIZE) {
+ next = new MemBlock(len);
+ } else {
+ next = new MemBlock;
+ }
+ }
+ return next->add(src, len);
+ }
+ char *dst = free;
+ free += len;
+ memcpy(dst, src, len);
+ return dst;
+ }
};
- /*}}}*/
-/** \brief applyFile - in reverse order with a tail recursion {{{
- *
- * As it is expected that the commands are in reversed order in the patch file
- * we check in the first half if the command is valid, but doesn't execute it
- * and move a step deeper. After reaching the end of the file we apply the
- * patches in the correct order: last found command first.
- *
- * \param ed_cmds patch file to apply
- * \param in_file base file we want to patch
- * \param out_file file to write the patched result to
- * \param line of command operation
- * \param buffer internal used read/write buffer
- * \param hash the created file for correctness
- * \return the success State of the ed command executor
- */
-RredMethod::State RredMethod::applyFile(FileFd &ed_cmds, FileFd &in_file, FileFd &out_file,
- unsigned long &line, char *buffer, Hashes *hash) const {
- // get the current command and parse it
- if (ed_cmds.ReadLine(buffer, BUF_SIZE) == NULL) {
- if (Debug == true)
- std::clog << "rred: encounter end of file - we can start patching now." << std::endl;
- line = 0;
- return ED_OK;
- }
-
- // parse in the effected linenumbers
- char* idx;
- errno=0;
- unsigned long const startline = strtol(buffer, &idx, 10);
- if (errno == ERANGE || errno == EINVAL) {
- _error->Errno("rred", "startline is an invalid number");
- return ED_PARSER;
- }
- if (startline > line) {
- _error->Error("rred: The start line (%lu) of the next command is higher than the last line (%lu). This is not allowed.", startline, line);
- return ED_ORDERING;
- }
- unsigned long stopline;
- if (*idx == ',') {
- idx++;
- errno=0;
- stopline = strtol(idx, &idx, 10);
- if (errno == ERANGE || errno == EINVAL) {
- _error->Errno("rred", "stopline is an invalid number");
- return ED_PARSER;
- }
- }
- else {
- stopline = startline;
- }
- line = startline;
-
- // which command to execute on this line(s)?
- switch (*idx) {
- case MODE_CHANGED:
- if (Debug == true)
- std::clog << "Change from line " << startline << " to " << stopline << std::endl;
- break;
- case MODE_ADDED:
- if (Debug == true)
- std::clog << "Insert after line " << startline << std::endl;
- break;
- case MODE_DELETED:
- if (Debug == true)
- std::clog << "Delete from line " << startline << " to " << stopline << std::endl;
- break;
- default:
- _error->Error("rred: Unknown ed command '%c'. Abort.", *idx);
- return ED_PARSER;
- }
- unsigned char mode = *idx;
-
- // save the current position
- unsigned const long long pos = ed_cmds.Tell();
-
- // if this is add or change then go to the next full stop
- unsigned int data_length = 0;
- if (mode == MODE_CHANGED || mode == MODE_ADDED) {
- do {
- ignoreLineInFile(ed_cmds, buffer);
- data_length++;
- }
- while (strncmp(buffer, ".", 1) != 0);
- data_length--; // the dot should not be copied
- }
-
- // do the recursive call - the last command is the one we need to execute at first
- const State child = applyFile(ed_cmds, in_file, out_file, line, buffer, hash);
- if (child != ED_OK) {
- return child;
- }
-
- // change and delete are working on "line" - add is done after "line"
- if (mode != MODE_ADDED)
- line++;
-
- // first wind to the current position and copy over all unchanged lines
- if (line < startline) {
- copyLinesFromFileToFile(in_file, out_file, (startline - line), hash, buffer);
- line = startline;
- }
-
- if (mode != MODE_ADDED)
- line--;
-
- // include data from ed script
- if (mode == MODE_CHANGED || mode == MODE_ADDED) {
- ed_cmds.Seek(pos);
- copyLinesFromFileToFile(ed_cmds, out_file, data_length, hash, buffer);
- }
-
- // ignore the corresponding number of lines from input
- if (mode == MODE_CHANGED || mode == MODE_DELETED) {
- while (line < stopline) {
- ignoreLineInFile(in_file, buffer);
- line++;
- }
- }
- return ED_OK;
-}
- /*}}}*/
-void RredMethod::copyLinesFromFileToFile(FileFd &fin, FileFd &fout, unsigned int lines,/*{{{*/
- Hashes *hash, char *buffer) const {
- while (0 < lines--) {
- do {
- fin.ReadLine(buffer, BUF_SIZE);
- unsigned long long const towrite = strlen(buffer);
- fout.Write(buffer, towrite);
- hash->Add((unsigned char*)buffer, towrite);
- } while (strlen(buffer) == (BUF_SIZE - 1) &&
- buffer[BUF_SIZE - 2] != '\n');
- }
-}
- /*}}}*/
-void RredMethod::ignoreLineInFile(FileFd &fin, char *buffer) const { /*{{{*/
- fin.ReadLine(buffer, BUF_SIZE);
- while (strlen(buffer) == (BUF_SIZE - 1) &&
- buffer[BUF_SIZE - 2] != '\n') {
- fin.ReadLine(buffer, BUF_SIZE);
- buffer[0] = ' ';
- }
-}
- /*}}}*/
-RredMethod::State RredMethod::patchFile(FileFd &Patch, FileFd &From, /*{{{*/
- FileFd &out_file, Hashes *hash) const {
- char buffer[BUF_SIZE];
-
- /* we do a tail recursion to read the commands in the right order */
- unsigned long line = -1; // assign highest possible value
- State const result = applyFile(Patch, From, out_file, line, buffer, hash);
-
- /* read the rest from infile */
- if (result == ED_OK) {
- while (From.ReadLine(buffer, BUF_SIZE) != NULL) {
- unsigned long long const towrite = strlen(buffer);
- out_file.Write(buffer, towrite);
- hash->Add((unsigned char*)buffer, towrite);
+
+struct Change {
+ /* Ordering:
+ *
+ * 1. write out <offset> lines unchanged
+ * 2. skip <del_cnt> lines from source
+ * 3. write out <add_cnt> lines (<add>/<add_len>)
+ */
+ size_t offset;
+ size_t del_cnt;
+ size_t add_cnt; /* lines */
+ size_t add_len; /* bytes */
+ char *add;
+
+ Change(int off)
+ {
+ offset = off;
+ del_cnt = add_cnt = add_len = 0;
+ add = NULL;
+ }
+
+ /* actually, don't write <lines> lines from <add> */
+ void skip_lines(size_t lines)
+ {
+ while (lines > 0) {
+ char *s = (char*) memchr(add, '\n', add_len);
+ assert(s != NULL);
+ s++;
+ add_len -= (s - add);
+ add_cnt--;
+ lines--;
+ if (add_len == 0) {
+ add = NULL;
+ assert(add_cnt == 0);
+ assert(lines == 0);
+ } else {
+ add = s;
+ assert(add_cnt > 0);
+ }
}
}
- return result;
-}
- /*}}}*/
-/* struct EdCommand {{{*/
-#ifdef _POSIX_MAPPED_FILES
-struct EdCommand {
- size_t data_start;
- size_t data_end;
- size_t data_lines;
- size_t first_line;
- size_t last_line;
- char type;
};
-#define IOV_COUNT 1024 /* Don't really want IOV_MAX since it can be arbitrarily large */
-static ssize_t retry_writev(int fd, const struct iovec *iov, int iovcnt) {
- ssize_t Res;
- errno = 0;
- ssize_t i = 0;
- do {
- Res = writev(fd, iov + i, iovcnt);
- if (Res < 0 && errno == EINTR)
- continue;
- if (Res < 0)
- return _error->Errno("writev",_("Write error"));
- iovcnt -= Res;
- i += Res;
- } while (Res > 0 && iovcnt > 0);
- return i;
-}
-#endif
- /*}}}*/
-RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/
- FileFd &out_file, Hashes *hash) const {
-#ifdef _POSIX_MAPPED_FILES
- MMap ed_cmds(Patch, MMap::ReadOnly);
- MMap in_file(From, MMap::ReadOnly);
-
- unsigned long long const ed_size = ed_cmds.Size();
- unsigned long long const in_size = in_file.Size();
- if (ed_size == 0 || in_size == 0)
- return MMAP_FAILED;
-
- EdCommand* commands = 0;
- size_t command_count = 0;
- size_t command_alloc = 0;
-
- const char* begin = (char*) ed_cmds.Data();
- const char* end = begin;
- const char* ed_end = (char*) ed_cmds.Data() + ed_size;
-
- const char* input = (char*) in_file.Data();
- const char* input_end = (char*) in_file.Data() + in_size;
-
- size_t i;
-
- /* 1. Parse entire script. It is executed in reverse order, so we cather it
- * in the `commands' buffer first
- */
-
- for(;;) {
- EdCommand cmd;
- cmd.data_start = 0;
- cmd.data_end = 0;
-
- while(begin != ed_end && *begin == '\n')
- ++begin;
- while(end != ed_end && *end != '\n')
- ++end;
- if(end == ed_end && begin == end)
- break;
-
- /* Determine command range */
- const char* tmp = begin;
-
- for(;;) {
- /* atoll is safe despite lacking NUL-termination; we know there's an
- * alphabetic character at end[-1]
- */
- if(tmp == end) {
- cmd.first_line = atol(begin);
- cmd.last_line = cmd.first_line;
- break;
- }
- if(*tmp == ',') {
- cmd.first_line = atol(begin);
- cmd.last_line = atol(tmp + 1);
- break;
- }
- ++tmp;
- }
-
- // which command to execute on this line(s)?
- switch (end[-1]) {
- case MODE_CHANGED:
- if (Debug == true)
- std::clog << "Change from line " << cmd.first_line << " to " << cmd.last_line << std::endl;
- break;
- case MODE_ADDED:
- if (Debug == true)
- std::clog << "Insert after line " << cmd.first_line << std::endl;
- break;
- case MODE_DELETED:
- if (Debug == true)
- std::clog << "Delete from line " << cmd.first_line << " to " << cmd.last_line << std::endl;
- break;
- default:
- _error->Error("rred: Unknown ed command '%c'. Abort.", end[-1]);
- free(commands);
- return ED_PARSER;
- }
- cmd.type = end[-1];
-
- /* Determine the size of the inserted text, so we don't have to scan this
- * text again later.
- */
- begin = end + 1;
- end = begin;
- cmd.data_lines = 0;
-
- if(cmd.type == MODE_ADDED || cmd.type == MODE_CHANGED) {
- cmd.data_start = begin - (char*) ed_cmds.Data();
- while(end != ed_end) {
- if(*end == '\n') {
- if(end[-1] == '.' && end[-2] == '\n')
- break;
- ++cmd.data_lines;
- }
- ++end;
- }
- cmd.data_end = end - (char*) ed_cmds.Data() - 1;
- begin = end + 1;
- end = begin;
- }
- if(command_count == command_alloc) {
- command_alloc = (command_alloc + 64) * 3 / 2;
- EdCommand* newCommands = (EdCommand*) realloc(commands, command_alloc * sizeof(EdCommand));
- if (newCommands == NULL) {
- free(commands);
- return MMAP_FAILED;
- }
- commands = newCommands;
- }
- commands[command_count++] = cmd;
- }
-
- struct iovec* iov = new struct iovec[IOV_COUNT];
- size_t iov_size = 0;
-
- size_t amount, remaining;
- size_t line = 1;
- EdCommand* cmd;
-
- /* 2. Execute script. We gather writes in a `struct iov' array, and flush
- * using writev to minimize the number of system calls. Data is read
- * directly from the memory mappings of the input file and the script.
- */
-
- for(i = command_count; i-- > 0; ) {
- cmd = &commands[i];
- if(cmd->type == MODE_ADDED)
- amount = cmd->first_line + 1;
- else
- amount = cmd->first_line;
-
- if(line < amount) {
- begin = input;
- while(line != amount) {
- input = (const char*) memchr(input, '\n', input_end - input);
- if(!input)
- break;
- ++line;
- ++input;
- }
-
- iov[iov_size].iov_base = (void*) begin;
- iov[iov_size].iov_len = input - begin;
- hash->Add((const unsigned char*) begin, input - begin);
-
- if(++iov_size == IOV_COUNT) {
- retry_writev(out_file.Fd(), iov, IOV_COUNT);
- iov_size = 0;
- }
- }
-
- if(cmd->type == MODE_DELETED || cmd->type == MODE_CHANGED) {
- remaining = (cmd->last_line - cmd->first_line) + 1;
- line += remaining;
- while(remaining) {
- input = (const char*) memchr(input, '\n', input_end - input);
- if(!input)
- break;
- --remaining;
- ++input;
- }
- }
-
- if(cmd->type == MODE_CHANGED || cmd->type == MODE_ADDED) {
- if(cmd->data_end != cmd->data_start) {
- iov[iov_size].iov_base = (void*) ((char*)ed_cmds.Data() + cmd->data_start);
- iov[iov_size].iov_len = cmd->data_end - cmd->data_start;
- hash->Add((const unsigned char*) ((char*)ed_cmds.Data() + cmd->data_start),
- iov[iov_size].iov_len);
-
- if(++iov_size == IOV_COUNT) {
- retry_writev(out_file.Fd(), iov, IOV_COUNT);
- iov_size = 0;
- }
- }
- }
- }
-
- if(input != input_end) {
- iov[iov_size].iov_base = (void*) input;
- iov[iov_size].iov_len = input_end - input;
- hash->Add((const unsigned char*) input, input_end - input);
- ++iov_size;
- }
-
- if(iov_size) {
- retry_writev(out_file.Fd(), iov, iov_size);
- iov_size = 0;
- }
-
- for(i = 0; i < iov_size; i += IOV_COUNT) {
- if(iov_size - i < IOV_COUNT)
- retry_writev(out_file.Fd(), iov + i, iov_size - i);
- else
- retry_writev(out_file.Fd(), iov + i, IOV_COUNT);
- }
-
- delete [] iov;
- free(commands);
-
- return ED_OK;
+
+class FileChanges {
+ std::list<struct Change> changes;
+ std::list<struct Change>::iterator where;
+ size_t pos; // line number is as far left of iterator as possible
+
+ bool pos_is_okay(void)
+ {
+#ifdef POSDEBUG
+ size_t cpos = 0;
+ std::list<struct Change>::iterator x;
+ for (x = changes.begin(); x != where; ++x) {
+ assert(x != changes.end());
+ cpos += x->offset + x->add_cnt;
+ }
+ return cpos == pos;
#else
- return MMAP_FAILED;
+ return true;
#endif
-}
- /*}}}*/
-bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/
-{
- Debug = _config->FindB("Debug::pkgAcquire::RRed", false);
- URI Get = Itm->Uri;
- std::string Path = Get.Host + Get.Path; // To account for relative paths
-
- FetchResult Res;
- Res.Filename = Itm->DestFile;
- if (Itm->Uri.empty() == true) {
- Path = Itm->DestFile;
- Itm->DestFile.append(".result");
- } else
- URIStart(Res);
-
- if (Debug == true)
- std::clog << "Patching " << Path << " with " << Path
- << ".ed and putting result into " << Itm->DestFile << std::endl;
- // Open the source and destination files (the d'tor of FileFd will do
- // the cleanup/closing of the fds)
- FileFd From(Path,FileFd::ReadOnly);
- FileFd Patch(Path+".ed",FileFd::ReadOnly, FileFd::Gzip);
- FileFd To(Itm->DestFile,FileFd::WriteAtomic);
- To.EraseOnFailure();
- if (_error->PendingError() == true)
- return false;
-
- Hashes Hash;
- // now do the actual patching
- State const result = patchMMap(Patch, From, To, &Hash);
- if (result == MMAP_FAILED) {
- // retry with patchFile
- Patch.Seek(0);
- From.Seek(0);
- To.Open(Itm->DestFile,FileFd::WriteAtomic);
- if (_error->PendingError() == true)
- return false;
- if (patchFile(Patch, From, To, &Hash) != ED_OK) {
- return _error->WarningE("rred", _("Could not patch %s with mmap and with file operation usage - the patch seems to be corrupt."), Path.c_str());
- } else if (Debug == true) {
- std::clog << "rred: finished file patching of " << Path << " after mmap failed." << std::endl;
+ }
+
+ public:
+ FileChanges() {
+ where = changes.end();
+ pos = 0;
+ }
+
+ std::list<struct Change>::iterator begin(void) { return changes.begin(); }
+ std::list<struct Change>::iterator end(void) { return changes.end(); }
+
+ std::list<struct Change>::reverse_iterator rbegin(void) { return changes.rbegin(); }
+ std::list<struct Change>::reverse_iterator rend(void) { return changes.rend(); }
+
+ void add_change(Change c) {
+ assert(pos_is_okay());
+ go_to_change_for(c.offset);
+ assert(pos + where->offset == c.offset);
+ if (c.del_cnt > 0)
+ delete_lines(c.del_cnt);
+ assert(pos + where->offset == c.offset);
+ if (c.add_len > 0) {
+ assert(pos_is_okay());
+ if (where->add_len > 0)
+ new_change();
+ assert(where->add_len == 0 && where->add_cnt == 0);
+
+ where->add_len = c.add_len;
+ where->add_cnt = c.add_cnt;
+ where->add = c.add;
}
- } else if (result != ED_OK) {
- return _error->Errno("rred", _("Could not patch %s with mmap (but no mmap specific fail) - the patch seems to be corrupt."), Path.c_str());
- } else if (Debug == true) {
- std::clog << "rred: finished mmap patching of " << Path << std::endl;
+ assert(pos_is_okay());
+ merge();
+ assert(pos_is_okay());
}
- // write out the result
- From.Close();
- Patch.Close();
- To.Close();
-
- /* Transfer the modification times from the patch file
- to be able to see in which state the file should be
- and use the access time from the "old" file */
- struct stat BufBase, BufPatch;
- if (stat(Path.c_str(),&BufBase) != 0 ||
- stat(std::string(Path+".ed").c_str(),&BufPatch) != 0)
- return _error->Errno("stat",_("Failed to stat"));
-
- struct utimbuf TimeBuf;
- TimeBuf.actime = BufBase.st_atime;
- TimeBuf.modtime = BufPatch.st_mtime;
- if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0)
- return _error->Errno("utime",_("Failed to set modification time"));
-
- if (stat(Itm->DestFile.c_str(),&BufBase) != 0)
- return _error->Errno("stat",_("Failed to stat"));
-
- // return done
- Res.LastModified = BufBase.st_mtime;
- Res.Size = BufBase.st_size;
- Res.TakeHashes(Hash);
- URIDone(Res);
-
- return true;
-}
- /*}}}*/
-/** \brief Wrapper class for testing rred */ /*{{{*/
-class TestRredMethod : public RredMethod {
-public:
- /** \brief Run rred in debug test mode
- *
- * This method can be used to run the rred method outside
- * of the "normal" acquire environment for easier testing.
- *
- * \param base basename of all files involved in this rred test
- */
- bool Run(char const *base) {
- _config->CndSet("Debug::pkgAcquire::RRed", "true");
- FetchItem *test = new FetchItem;
- test->DestFile = base;
- return Fetch(test);
- }
+ private:
+ void merge(void)
+ {
+ while (where->offset == 0 && where != changes.begin()) {
+ left();
+ }
+ std::list<struct Change>::iterator next = where;
+ ++next;
+
+ while (next != changes.end() && next->offset == 0) {
+ where->del_cnt += next->del_cnt;
+ next->del_cnt = 0;
+ if (next->add == NULL) {
+ next = changes.erase(next);
+ } else if (where->add == NULL) {
+ where->add = next->add;
+ where->add_len = next->add_len;
+ where->add_cnt = next->add_cnt;
+ next = changes.erase(next);
+ } else {
+ ++next;
+ }
+ }
+ }
+
+ void go_to_change_for(size_t line)
+ {
+ while(where != changes.end()) {
+ if (line < pos) {
+ left();
+ continue;
+ }
+ if (pos + where->offset + where->add_cnt <= line) {
+ right();
+ continue;
+ }
+ // line is somewhere in this slot
+ if (line < pos + where->offset) {
+ break;
+ } else if (line == pos + where->offset) {
+ return;
+ } else {
+ split(line - pos);
+ right();
+ return;
+ }
+ }
+ /* it goes before this patch */
+ insert(line-pos);
+ }
+
+ void new_change(void) { insert(where->offset); }
+
+ void insert(size_t offset)
+ {
+ assert(pos_is_okay());
+ assert(where == changes.end() || offset <= where->offset);
+ if (where != changes.end())
+ where->offset -= offset;
+ changes.insert(where, Change(offset));
+ --where;
+ assert(pos_is_okay());
+ }
+
+ void split(size_t offset)
+ {
+ assert(pos_is_okay());
+
+ assert(where->offset < offset);
+ assert(offset < where->offset + where->add_cnt);
+
+ size_t keep_lines = offset - where->offset;
+
+ Change before(*where);
+
+ where->del_cnt = 0;
+ where->offset = 0;
+ where->skip_lines(keep_lines);
+
+ before.add_cnt = keep_lines;
+ before.add_len -= where->add_len;
+
+ changes.insert(where, before);
+ --where;
+ assert(pos_is_okay());
+ }
+
+ void delete_lines(size_t cnt)
+ {
+ std::list<struct Change>::iterator x = where;
+ assert(pos_is_okay());
+ while (cnt > 0)
+ {
+ size_t del;
+ del = x->add_cnt;
+ if (del > cnt)
+ del = cnt;
+ x->skip_lines(del);
+ cnt -= del;
+
+ ++x;
+ if (x == changes.end()) {
+ del = cnt;
+ } else {
+ del = x->offset;
+ if (del > cnt)
+ del = cnt;
+ x->offset -= del;
+ }
+ where->del_cnt += del;
+ cnt -= del;
+ }
+ assert(pos_is_okay());
+ }
+
+ void left(void) {
+ assert(pos_is_okay());
+ --where;
+ pos -= where->offset + where->add_cnt;
+ assert(pos_is_okay());
+ }
+
+ void right(void) {
+ assert(pos_is_okay());
+ pos += where->offset + where->add_cnt;
+ ++where;
+ assert(pos_is_okay());
+ }
+};
+
+class Patch {
+ FileChanges filechanges;
+ MemBlock add_text;
+
+ static bool retry_fwrite(char *b, size_t l, FILE *f, Hashes *hash)
+ {
+ size_t r = 1;
+ while (r > 0 && l > 0)
+ {
+ r = fwrite(b, 1, l, f);
+ if (hash)
+ hash->Add((unsigned char*)b, r);
+ l -= r;
+ b += r;
+ }
+ return l == 0;
+ }
+
+ static void dump_rest(FILE *o, FILE *i, Hashes *hash)
+ {
+ char buffer[BLOCK_SIZE];
+ size_t l;
+ while (0 < (l = fread(buffer, 1, sizeof(buffer), i))) {
+ if (!retry_fwrite(buffer, l, o, hash))
+ break;
+ }
+ }
+
+ static void dump_lines(FILE *o, FILE *i, size_t n, Hashes *hash)
+ {
+ char buffer[BLOCK_SIZE];
+ while (n > 0) {
+ if (fgets(buffer, sizeof(buffer), i) == 0)
+ buffer[0] = '\0';
+ size_t const l = strlen(buffer);
+ if (l == 0 || buffer[l-1] == '\n')
+ n--;
+ retry_fwrite(buffer, l, o, hash);
+ }
+ }
+
+ static void skip_lines(FILE *i, int n)
+ {
+ char buffer[BLOCK_SIZE];
+ while (n > 0) {
+ if (fgets(buffer, sizeof(buffer), i) == 0)
+ buffer[0] = '\0';
+ size_t const l = strlen(buffer);
+ if (l == 0 || buffer[l-1] == '\n')
+ n--;
+ }
+ }
+
+ static void dump_mem(FILE *o, char *p, size_t s, Hashes *hash) {
+ retry_fwrite(p, s, o, hash);
+ }
+
+ public:
+
+ void read_diff(FileFd &f)
+ {
+ char buffer[BLOCK_SIZE];
+ bool cmdwanted = true;
+
+ Change ch(0);
+ while(f.ReadLine(buffer, sizeof(buffer)))
+ {
+ if (cmdwanted) {
+ char *m, *c;
+ size_t s, e;
+ s = strtol(buffer, &m, 10);
+ if (m == buffer) {
+ s = e = ch.offset + ch.add_cnt;
+ c = buffer;
+ } else if (*m == ',') {
+ m++;
+ e = strtol(m, &c, 10);
+ } else {
+ e = s;
+ c = m;
+ }
+ switch(*c) {
+ case 'a':
+ cmdwanted = false;
+ ch.add = NULL;
+ ch.add_cnt = 0;
+ ch.add_len = 0;
+ ch.offset = s;
+ ch.del_cnt = 0;
+ break;
+ case 'c':
+ cmdwanted = false;
+ ch.add = NULL;
+ ch.add_cnt = 0;
+ ch.add_len = 0;
+ ch.offset = s - 1;
+ ch.del_cnt = e - s + 1;
+ break;
+ case 'd':
+ ch.offset = s - 1;
+ ch.del_cnt = e - s + 1;
+ ch.add = NULL;
+ ch.add_cnt = 0;
+ ch.add_len = 0;
+ filechanges.add_change(ch);
+ break;
+ }
+ } else { /* !cmdwanted */
+ if (buffer[0] == '.' && buffer[1] == '\n') {
+ cmdwanted = true;
+ filechanges.add_change(ch);
+ } else {
+ char *last = NULL;
+ char *add;
+ size_t l;
+ if (ch.add)
+ last = ch.add + ch.add_len;
+ l = strlen(buffer);
+ add = add_text.add_easy(buffer, l, last);
+ if (!add) {
+ ch.add_len += l;
+ ch.add_cnt++;
+ } else {
+ if (ch.add) {
+ filechanges.add_change(ch);
+ ch.del_cnt = 0;
+ }
+ ch.offset += ch.add_cnt;
+ ch.add = add;
+ ch.add_len = l;
+ ch.add_cnt = 1;
+ }
+ }
+ }
+ }
+ }
+
+ void write_diff(FILE *f)
+ {
+ size_t line = 0;
+ std::list<struct Change>::reverse_iterator ch;
+ for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
+ line += ch->offset + ch->del_cnt;
+ }
+
+ for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
+ std::list<struct Change>::reverse_iterator mg_i, mg_e = ch;
+ while (ch->del_cnt == 0 && ch->offset == 0)
+ ++ch;
+ line -= ch->del_cnt;
+ if (ch->add_cnt > 0) {
+ if (ch->del_cnt == 0) {
+ fprintf(f, "%lua\n", line);
+ } else if (ch->del_cnt == 1) {
+ fprintf(f, "%luc\n", line+1);
+ } else {
+ fprintf(f, "%lu,%luc\n", line+1, line+ch->del_cnt);
+ }
+
+ mg_i = ch;
+ do {
+ dump_mem(f, mg_i->add, mg_i->add_len, NULL);
+ } while (mg_i-- != mg_e);
+
+ fprintf(f, ".\n");
+ } else if (ch->del_cnt == 1) {
+ fprintf(f, "%lud\n", line+1);
+ } else if (ch->del_cnt > 1) {
+ fprintf(f, "%lu,%lud\n", line+1, line+ch->del_cnt);
+ }
+ line -= ch->offset;
+ }
+ }
+
+ void apply_against_file(FILE *out, FILE *in, Hashes *hash = NULL)
+ {
+ std::list<struct Change>::iterator ch;
+ for (ch = filechanges.begin(); ch != filechanges.end(); ++ch) {
+ dump_lines(out, in, ch->offset, hash);
+ skip_lines(in, ch->del_cnt);
+ dump_mem(out, ch->add, ch->add_len, hash);
+ }
+ dump_rest(out, in, hash);
+ }
};
- /*}}}*/
-/** \brief Starter for the rred method (or its test method) {{{
- *
- * Used without parameters is the normal behavior for methods for
- * the APT acquire system. While this works great for the acquire system
- * it is very hard to test the method and therefore the method also
- * accepts one parameter which will switch it directly to debug test mode:
- * The test mode expects that if "Testfile" is given as parameter
- * the file "Testfile" should be ed-style patched with "Testfile.ed"
- * and will write the result to "Testfile.result".
- */
-int main(int argc, char *argv[]) {
- if (argc <= 1) {
- RredMethod Mth;
- return Mth.Run();
- } else {
- TestRredMethod Mth;
- bool result = Mth.Run(argv[1]);
- _error->DumpErrors();
- return result;
- }
+
+class RredMethod : public pkgAcqMethod {
+ private:
+ bool Debug;
+
+ protected:
+ virtual bool Fetch(FetchItem *Itm) {
+ Debug = _config->FindB("Debug::pkgAcquire::RRed", false);
+ URI Get = Itm->Uri;
+ std::string Path = Get.Host + Get.Path; // rred:/path - no host
+
+ FetchResult Res;
+ Res.Filename = Itm->DestFile;
+ if (Itm->Uri.empty())
+ {
+ Path = Itm->DestFile;
+ Itm->DestFile.append(".result");
+ } else
+ URIStart(Res);
+
+ std::vector<std::string> patchpaths;
+ Patch patch;
+
+ if (FileExists(Path + ".ed") == true)
+ patchpaths.push_back(Path + ".ed");
+ else
+ {
+ _error->PushToStack();
+ std::vector<std::string> patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false);
+ _error->RevertToStack();
+
+ std::string const baseName = Path + ".ed.";
+ for (std::vector<std::string>::const_iterator p = patches.begin();
+ p != patches.end(); ++p)
+ if (p->compare(0, baseName.length(), baseName) == 0)
+ patchpaths.push_back(*p);
+ }
+
+ std::string patch_name;
+ for (std::vector<std::string>::iterator I = patchpaths.begin();
+ I != patchpaths.end();
+ ++I)
+ {
+ patch_name = *I;
+ if (Debug == true)
+ std::clog << "Patching " << Path << " with " << patch_name
+ << std::endl;
+
+ FileFd p;
+ // all patches are compressed, even if the name doesn't reflect it
+ if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false) {
+ std::cerr << "Could not open patch file " << patch_name << std::endl;
+ _error->DumpErrors(std::cerr);
+ abort();
+ }
+ patch.read_diff(p);
+ p.Close();
+ }
+
+ if (Debug == true)
+ std::clog << "Applying patches against " << Path
+ << " and writing results to " << Itm->DestFile
+ << std::endl;
+
+ FILE *inp = fopen(Path.c_str(), "r");
+ FILE *out = fopen(Itm->DestFile.c_str(), "w");
+
+ Hashes hash;
+
+ patch.apply_against_file(out, inp, &hash);
+
+ fclose(out);
+ fclose(inp);
+
+ if (Debug == true) {
+ std::clog << "rred: finished file patching of " << Path << "." << std::endl;
+ }
+
+ struct stat bufbase, bufpatch;
+ if (stat(Path.c_str(), &bufbase) != 0 ||
+ stat(patch_name.c_str(), &bufpatch) != 0)
+ return _error->Errno("stat", _("Failed to stat"));
+
+ struct timeval times[2];
+ times[0].tv_sec = bufbase.st_atime;
+ times[1].tv_sec = bufpatch.st_mtime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ if (utimes(Itm->DestFile.c_str(), times) != 0)
+ return _error->Errno("utimes",_("Failed to set modification time"));
+
+ if (stat(Itm->DestFile.c_str(), &bufbase) != 0)
+ return _error->Errno("stat", _("Failed to stat"));
+
+ Res.LastModified = bufbase.st_mtime;
+ Res.Size = bufbase.st_size;
+ Res.TakeHashes(hash);
+ URIDone(Res);
+
+ return true;
+ }
+
+ public:
+ RredMethod() : pkgAcqMethod("2.0",SingleInstance | SendConfig), Debug(false) {}
+};
+
+int main(int argc, char **argv)
+{
+ int i;
+ bool just_diff = true;
+ Patch patch;
+
+ if (argc <= 1) {
+ RredMethod Mth;
+ return Mth.Run();
+ }
+
+ if (argc > 1 && strcmp(argv[1], "-f") == 0) {
+ just_diff = false;
+ i = 2;
+ } else {
+ i = 1;
+ }
+
+ for (; i < argc; i++) {
+ FileFd p;
+ if (p.Open(argv[i], FileFd::ReadOnly) == false) {
+ _error->DumpErrors(std::cerr);
+ exit(1);
+ }
+ patch.read_diff(p);
+ }
+
+ if (just_diff) {
+ patch.write_diff(stdout);
+ } else {
+ FILE *out, *inp;
+ out = stdout;
+ inp = stdin;
+
+ patch.apply_against_file(out, inp);
+ }
+ return 0;
}
- /*}}}*/
diff --git a/methods/rsh.cc b/methods/rsh.cc
index d76dca6ef..550f77eca 100644
--- a/methods/rsh.cc
+++ b/methods/rsh.cc
@@ -20,7 +20,6 @@
#include <sys/stat.h>
#include <sys/time.h>
-#include <utime.h>
#include <unistd.h>
#include <signal.h>
#include <stdio.h>
@@ -256,7 +255,7 @@ bool RSHConn::WriteMsg(std::string &Text,bool Sync,const char *Fmt,...)
/*}}}*/
// RSHConn::Size - Return the size of the file /*{{{*/
// ---------------------------------------------------------------------
-/* Right now for successfull transfer the file size must be known in
+/* Right now for successful transfer the file size must be known in
advance. */
bool RSHConn::Size(const char *Path,unsigned long long &Size)
{
@@ -395,13 +394,14 @@ void RSHMethod::SigTerm(int sig)
{
if (FailFd == -1)
_exit(100);
- close(FailFd);
- // Timestamp
- struct utimbuf UBuf;
- UBuf.actime = FailTime;
- UBuf.modtime = FailTime;
- utime(FailFile.c_str(),&UBuf);
+ // Transfer the modification times
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+ close(FailFd);
_exit(100);
}
@@ -488,10 +488,11 @@ bool RSHMethod::Fetch(FetchItem *Itm)
Fd.Close();
// Timestamp
- struct utimbuf UBuf;
- UBuf.actime = FailTime;
- UBuf.modtime = FailTime;
- utime(FailFile.c_str(),&UBuf);
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
// If the file is missing we hard fail otherwise transient fail
if (Missing == true)
@@ -501,18 +502,17 @@ bool RSHMethod::Fetch(FetchItem *Itm)
}
Res.Size = Fd.Size();
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(Fd.Name().c_str(), times);
+ FailFd = -1;
}
Res.LastModified = FailTime;
Res.TakeHashes(Hash);
- // Timestamp
- struct utimbuf UBuf;
- UBuf.actime = FailTime;
- UBuf.modtime = FailTime;
- utime(Queue->DestFile.c_str(),&UBuf);
- FailFd = -1;
-
URIDone(Res);
return true;
diff --git a/methods/server.cc b/methods/server.cc
new file mode 100644
index 000000000..ef90c809c
--- /dev/null
+++ b/methods/server.cc
@@ -0,0 +1,668 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ HTTP and HTTPS share a lot of common code and these classes are
+ exactly the dumping ground for this common code
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/acquire-method.h>
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/netrc.h>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <signal.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <climits>
+#include <iostream>
+#include <map>
+
+// Internet stuff
+#include <netdb.h>
+
+#include "config.h"
+#include "connect.h"
+#include "rfc2553emu.h"
+#include "http.h"
+
+#include <apti18n.h>
+ /*}}}*/
+using namespace std;
+
+string ServerMethod::FailFile;
+int ServerMethod::FailFd = -1;
+time_t ServerMethod::FailTime = 0;
+
+// ServerState::RunHeaders - Get the headers before the data /*{{{*/
+// ---------------------------------------------------------------------
+/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
+ parse error occurred */
+ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File)
+{
+ State = Header;
+
+ Owner->Status(_("Waiting for headers"));
+
+ Major = 0;
+ Minor = 0;
+ Result = 0;
+ Size = 0;
+ StartPos = 0;
+ Encoding = Closes;
+ HaveContent = false;
+ time(&Date);
+
+ do
+ {
+ string Data;
+ if (ReadHeaderLines(Data) == false)
+ continue;
+
+ if (Owner->Debug == true)
+ clog << Data;
+
+ for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
+ {
+ string::const_iterator J = I;
+ for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
+ if (HeaderLine(string(I,J)) == false)
+ return RUN_HEADERS_PARSE_ERROR;
+ I = J;
+ }
+
+ // 100 Continue is a Nop...
+ if (Result == 100)
+ continue;
+
+ // Tidy up the connection persistence state.
+ if (Encoding == Closes && HaveContent == true)
+ Persistent = false;
+
+ return RUN_HEADERS_OK;
+ }
+ while (LoadNextResponse(false, File) == true);
+
+ return RUN_HEADERS_IO_ERROR;
+}
+ /*}}}*/
+// ServerState::HeaderLine - Process a header line /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool ServerState::HeaderLine(string Line)
+{
+ if (Line.empty() == true)
+ return true;
+
+ string::size_type Pos = Line.find(' ');
+ if (Pos == string::npos || Pos+1 > Line.length())
+ {
+ // Blah, some servers use "connection:closes", evil.
+ Pos = Line.find(':');
+ if (Pos == string::npos || Pos + 2 > Line.length())
+ return _error->Error(_("Bad header line"));
+ Pos++;
+ }
+
+ // Parse off any trailing spaces between the : and the next word.
+ string::size_type Pos2 = Pos;
+ while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
+ Pos2++;
+
+ string Tag = string(Line,0,Pos);
+ string Val = string(Line,Pos2);
+
+ if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
+ {
+ // Evil servers return no version
+ if (Line[4] == '/')
+ {
+ int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
+ if (elements == 3)
+ {
+ Code[0] = '\0';
+ if (Owner->Debug == true)
+ clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
+ }
+ else if (elements != 4)
+ return _error->Error(_("The HTTP server sent an invalid reply header"));
+ }
+ else
+ {
+ Major = 0;
+ Minor = 9;
+ if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
+ return _error->Error(_("The HTTP server sent an invalid reply header"));
+ }
+
+ /* Check the HTTP response header to get the default persistence
+ state. */
+ if (Major < 1)
+ Persistent = false;
+ else
+ {
+ if (Major == 1 && Minor == 0)
+ Persistent = false;
+ else
+ Persistent = true;
+ }
+
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Content-Length:") == 0)
+ {
+ if (Encoding == Closes)
+ Encoding = Stream;
+ HaveContent = true;
+
+ // The length is already set from the Content-Range header
+ if (StartPos != 0)
+ return true;
+
+ Size = strtoull(Val.c_str(), NULL, 10);
+ if (Size >= std::numeric_limits<unsigned long long>::max())
+ return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
+ else if (Size == 0)
+ HaveContent = false;
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Content-Type:") == 0)
+ {
+ HaveContent = true;
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Content-Range:") == 0)
+ {
+ HaveContent = true;
+
+ // ยง14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
+ if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
+ {
+ StartPos = 1; // ignore Content-Length, it would override Size
+ HaveContent = false;
+ }
+ else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
+ return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
+ if ((unsigned long long)StartPos > Size)
+ return _error->Error(_("This HTTP server has broken range support"));
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
+ {
+ HaveContent = true;
+ if (stringcasecmp(Val,"chunked") == 0)
+ Encoding = Chunked;
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Connection:") == 0)
+ {
+ if (stringcasecmp(Val,"close") == 0)
+ Persistent = false;
+ if (stringcasecmp(Val,"keep-alive") == 0)
+ Persistent = true;
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Last-Modified:") == 0)
+ {
+ if (RFC1123StrToTime(Val.c_str(), Date) == false)
+ return _error->Error(_("Unknown date format"));
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Location:") == 0)
+ {
+ Location = Val;
+ return true;
+ }
+
+ return true;
+}
+ /*}}}*/
+// ServerState::ServerState - Constructor /*{{{*/
+ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
+{
+ Reset();
+}
+ /*}}}*/
+
+bool ServerMethod::Configuration(string Message) /*{{{*/
+{
+ return pkgAcqMethod::Configuration(Message);
+}
+ /*}}}*/
+
+// ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
+// ---------------------------------------------------------------------
+/* We look at the header data we got back from the server and decide what
+ to do. Returns DealWithHeadersResult (see http.h for details).
+ */
+ServerMethod::DealWithHeadersResult
+ServerMethod::DealWithHeaders(FetchResult &Res)
+{
+ // Not Modified
+ if (Server->Result == 304)
+ {
+ unlink(Queue->DestFile.c_str());
+ Res.IMSHit = true;
+ Res.LastModified = Queue->LastModified;
+ return IMS_HIT;
+ }
+
+ /* Redirect
+ *
+ * Note that it is only OK for us to treat all redirection the same
+ * because we *always* use GET, not other HTTP methods. There are
+ * three redirection codes for which it is not appropriate that we
+ * redirect. Pass on those codes so the error handling kicks in.
+ */
+ if (AllowRedirect
+ && (Server->Result > 300 && Server->Result < 400)
+ && (Server->Result != 300 // Multiple Choices
+ && Server->Result != 304 // Not Modified
+ && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
+ {
+ if (Server->Location.empty() == true);
+ else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
+ {
+ URI Uri = Queue->Uri;
+ if (Uri.Host.empty() == false)
+ NextURI = URI::SiteOnly(Uri);
+ else
+ NextURI.clear();
+ NextURI.append(DeQuoteString(Server->Location));
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ else
+ {
+ NextURI = DeQuoteString(Server->Location);
+ URI tmpURI = NextURI;
+ URI Uri = Queue->Uri;
+ // same protocol redirects are okay
+ if (tmpURI.Access == Uri.Access)
+ return TRY_AGAIN_OR_REDIRECT;
+ // as well as http to https
+ else if (Uri.Access == "http" && tmpURI.Access == "https")
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ /* else pass through for error message */
+ }
+ // retry after an invalid range response without partial data
+ else if (Server->Result == 416)
+ {
+ struct stat SBuf;
+ if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
+ {
+ if ((unsigned long long)SBuf.st_size == Server->Size)
+ {
+ // the file is completely downloaded, but was not moved
+ Server->StartPos = Server->Size;
+ Server->Result = 200;
+ Server->HaveContent = false;
+ }
+ else if (unlink(Queue->DestFile.c_str()) == 0)
+ {
+ NextURI = Queue->Uri;
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ }
+ }
+
+ /* We have a reply we dont handle. This should indicate a perm server
+ failure */
+ if (Server->Result < 200 || Server->Result >= 300)
+ {
+ char err[255];
+ snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result);
+ SetFailReason(err);
+ _error->Error("%u %s",Server->Result,Server->Code);
+ if (Server->HaveContent == true)
+ return ERROR_WITH_CONTENT_PAGE;
+ return ERROR_UNRECOVERABLE;
+ }
+
+ // This is some sort of 2xx 'data follows' reply
+ Res.LastModified = Server->Date;
+ Res.Size = Server->Size;
+
+ // Open the file
+ delete File;
+ File = new FileFd(Queue->DestFile,FileFd::WriteAny);
+ if (_error->PendingError() == true)
+ return ERROR_NOT_FROM_SERVER;
+
+ FailFile = Queue->DestFile;
+ FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
+ FailFd = File->Fd();
+ FailTime = Server->Date;
+
+ if (Server->InitHashes(*File) == false)
+ {
+ _error->Errno("read",_("Problem hashing file"));
+ return ERROR_NOT_FROM_SERVER;
+ }
+ if (Server->StartPos > 0)
+ Res.ResumePoint = Server->StartPos;
+
+ SetNonBlock(File->Fd(),true);
+ return FILE_IS_OPEN;
+}
+ /*}}}*/
+// ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
+// ---------------------------------------------------------------------
+/* This closes and timestamps the open file. This is necessary to get
+ resume behavoir on user abort */
+void ServerMethod::SigTerm(int)
+{
+ if (FailFd == -1)
+ _exit(100);
+
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+ close(FailFd);
+
+ _exit(100);
+}
+ /*}}}*/
+// ServerMethod::Fetch - Fetch an item /*{{{*/
+// ---------------------------------------------------------------------
+/* This adds an item to the pipeline. We keep the pipeline at a fixed
+ depth. */
+bool ServerMethod::Fetch(FetchItem *)
+{
+ if (Server == 0)
+ return true;
+
+ // Queue the requests
+ int Depth = -1;
+ for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
+ I = I->Next, Depth++)
+ {
+ // If pipelining is disabled, we only queue 1 request
+ if (Server->Pipeline == false && Depth >= 0)
+ break;
+
+ // Make sure we stick with the same server
+ if (Server->Comp(I->Uri) == false)
+ break;
+ if (QueueBack == I)
+ {
+ QueueBack = I->Next;
+ SendReq(I);
+ continue;
+ }
+ }
+
+ return true;
+};
+ /*}}}*/
+// ServerMethod::Loop - Main loop /*{{{*/
+int ServerMethod::Loop()
+{
+ typedef vector<string> StringVector;
+ typedef vector<string>::iterator StringVectorIterator;
+ map<string, StringVector> Redirected;
+
+ signal(SIGTERM,SigTerm);
+ signal(SIGINT,SigTerm);
+
+ Server = 0;
+
+ int FailCounter = 0;
+ while (1)
+ {
+ // We have no commands, wait for some to arrive
+ if (Queue == 0)
+ {
+ if (WaitFd(STDIN_FILENO) == false)
+ return 0;
+ }
+
+ /* Run messages, we can accept 0 (no message) if we didn't
+ do a WaitFd above.. Otherwise the FD is closed. */
+ int Result = Run(true);
+ if (Result != -1 && (Result != 0 || Queue == 0))
+ {
+ if(FailReason.empty() == false ||
+ _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
+ return 100;
+ else
+ return 0;
+ }
+
+ if (Queue == 0)
+ continue;
+
+ // Connect to the server
+ if (Server == 0 || Server->Comp(Queue->Uri) == false)
+ {
+ delete Server;
+ Server = CreateServerState(Queue->Uri);
+ }
+ /* If the server has explicitly said this is the last connection
+ then we pre-emptively shut down the pipeline and tear down
+ the connection. This will speed up HTTP/1.0 servers a tad
+ since we don't have to wait for the close sequence to
+ complete */
+ if (Server->Persistent == false)
+ Server->Close();
+
+ // Reset the pipeline
+ if (Server->IsOpen() == false)
+ QueueBack = Queue;
+
+ // Connnect to the host
+ if (Server->Open() == false)
+ {
+ Fail(true);
+ delete Server;
+ Server = 0;
+ continue;
+ }
+
+ // Fill the pipeline.
+ Fetch(0);
+
+ // Fetch the next URL header data from the server.
+ switch (Server->RunHeaders(File))
+ {
+ case ServerState::RUN_HEADERS_OK:
+ break;
+
+ // The header data is bad
+ case ServerState::RUN_HEADERS_PARSE_ERROR:
+ {
+ _error->Error(_("Bad header data"));
+ Fail(true);
+ RotateDNS();
+ continue;
+ }
+
+ // The server closed a connection during the header get..
+ default:
+ case ServerState::RUN_HEADERS_IO_ERROR:
+ {
+ FailCounter++;
+ _error->Discard();
+ Server->Close();
+ Server->Pipeline = false;
+
+ if (FailCounter >= 2)
+ {
+ Fail(_("Connection failed"),true);
+ FailCounter = 0;
+ }
+
+ RotateDNS();
+ continue;
+ }
+ };
+
+ // Decide what to do.
+ FetchResult Res;
+ Res.Filename = Queue->DestFile;
+ switch (DealWithHeaders(Res))
+ {
+ // Ok, the file is Open
+ case FILE_IS_OPEN:
+ {
+ URIStart(Res);
+
+ // Run the data
+ bool Result = true;
+ if (Server->HaveContent)
+ Result = Server->RunData(File);
+
+ /* If the server is sending back sizeless responses then fill in
+ the size now */
+ if (Res.Size == 0)
+ Res.Size = File->Size();
+
+ // Close the file, destroy the FD object and timestamp it
+ FailFd = -1;
+ delete File;
+ File = 0;
+
+ // Timestamp
+ struct timeval times[2];
+ times[0].tv_sec = times[1].tv_sec = Server->Date;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(Queue->DestFile.c_str(), times);
+
+ // Send status to APT
+ if (Result == true)
+ {
+ Res.TakeHashes(*Server->GetHashes());
+ URIDone(Res);
+ }
+ else
+ {
+ if (Server->IsOpen() == false)
+ {
+ FailCounter++;
+ _error->Discard();
+ Server->Close();
+
+ if (FailCounter >= 2)
+ {
+ Fail(_("Connection failed"),true);
+ FailCounter = 0;
+ }
+
+ QueueBack = Queue;
+ }
+ else
+ Fail(true);
+ }
+ break;
+ }
+
+ // IMS hit
+ case IMS_HIT:
+ {
+ URIDone(Res);
+ break;
+ }
+
+ // Hard server error, not found or something
+ case ERROR_UNRECOVERABLE:
+ {
+ Fail();
+ break;
+ }
+
+ // Hard internal error, kill the connection and fail
+ case ERROR_NOT_FROM_SERVER:
+ {
+ delete File;
+ File = 0;
+
+ Fail();
+ RotateDNS();
+ Server->Close();
+ break;
+ }
+
+ // We need to flush the data, the header is like a 404 w/ error text
+ case ERROR_WITH_CONTENT_PAGE:
+ {
+ Fail();
+
+ // Send to content to dev/null
+ File = new FileFd("/dev/null",FileFd::WriteExists);
+ Server->RunData(File);
+ delete File;
+ File = 0;
+ break;
+ }
+
+ // Try again with a new URL
+ case TRY_AGAIN_OR_REDIRECT:
+ {
+ // Clear rest of response if there is content
+ if (Server->HaveContent)
+ {
+ File = new FileFd("/dev/null",FileFd::WriteExists);
+ Server->RunData(File);
+ delete File;
+ File = 0;
+ }
+
+ /* Detect redirect loops. No more redirects are allowed
+ after the same URI is seen twice in a queue item. */
+ StringVector &R = Redirected[Queue->DestFile];
+ bool StopRedirects = false;
+ if (R.empty() == true)
+ R.push_back(Queue->Uri);
+ else if (R[0] == "STOP" || R.size() > 10)
+ StopRedirects = true;
+ else
+ {
+ for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
+ if (Queue->Uri == *I)
+ {
+ R[0] = "STOP";
+ break;
+ }
+
+ R.push_back(Queue->Uri);
+ }
+
+ if (StopRedirects == false)
+ Redirect(NextURI);
+ else
+ Fail();
+
+ break;
+ }
+
+ default:
+ Fail(_("Internal error"));
+ break;
+ }
+
+ FailCounter = 0;
+ }
+
+ return 0;
+}
+ /*}}}*/
diff --git a/methods/server.h b/methods/server.h
new file mode 100644
index 000000000..2b81e6173
--- /dev/null
+++ b/methods/server.h
@@ -0,0 +1,144 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Classes dealing with the abstraction of talking to a end via a text
+ protocol like HTTP (which is used by the http and https methods)
+
+ ##################################################################### */
+ /*}}}*/
+
+#ifndef APT_SERVER_H
+#define APT_SERVER_H
+
+#include <apt-pkg/strutl.h>
+
+#include <string>
+
+using std::cout;
+using std::endl;
+
+class Hashes;
+class ServerMethod;
+class FileFd;
+
+struct ServerState
+{
+ // This is the last parsed Header Line
+ unsigned int Major;
+ unsigned int Minor;
+ unsigned int Result;
+ char Code[360];
+
+ // These are some statistics from the last parsed header lines
+ unsigned long long Size;
+ signed long long StartPos;
+ time_t Date;
+ bool HaveContent;
+ enum {Chunked,Stream,Closes} Encoding;
+ enum {Header, Data} State;
+ bool Persistent;
+ std::string Location;
+
+ // This is a Persistent attribute of the server itself.
+ bool Pipeline;
+ URI ServerName;
+ URI Proxy;
+ unsigned long TimeOut;
+
+ protected:
+ ServerMethod *Owner;
+
+ virtual bool ReadHeaderLines(std::string &Data) = 0;
+ virtual bool LoadNextResponse(bool const ToFile, FileFd * const File) = 0;
+
+ public:
+ bool HeaderLine(std::string Line);
+
+ /** \brief Result of the header acquire */
+ enum RunHeadersResult {
+ /** \brief Header ok */
+ RUN_HEADERS_OK,
+ /** \brief IO error while retrieving */
+ RUN_HEADERS_IO_ERROR,
+ /** \brief Parse error after retrieving */
+ RUN_HEADERS_PARSE_ERROR,
+ };
+ /** \brief Get the headers before the data */
+ RunHeadersResult RunHeaders(FileFd * const File);
+
+ bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
+ virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+ StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
+ State = Header; Persistent = false; Pipeline = true;};
+ virtual bool WriteResponse(std::string const &Data) = 0;
+
+ /** \brief Transfer the data from the socket */
+ virtual bool RunData(FileFd * const File) = 0;
+
+ virtual bool Open() = 0;
+ virtual bool IsOpen() = 0;
+ virtual bool Close() = 0;
+ virtual bool InitHashes(FileFd &File) = 0;
+ virtual Hashes * GetHashes() = 0;
+ virtual bool Die(FileFd &File) = 0;
+ virtual bool Flush(FileFd * const File) = 0;
+ virtual bool Go(bool ToFile, FileFd * const File) = 0;
+
+ ServerState(URI Srv, ServerMethod *Owner);
+ virtual ~ServerState() {};
+};
+
+class ServerMethod : public pkgAcqMethod
+{
+ protected:
+ virtual bool Fetch(FetchItem *);
+
+ ServerState *Server;
+ std::string NextURI;
+ FileFd *File;
+
+ unsigned long PipelineDepth;
+ bool AllowRedirect;
+
+ public:
+ bool Debug;
+
+ /** \brief Result of the header parsing */
+ enum DealWithHeadersResult {
+ /** \brief The file is open and ready */
+ FILE_IS_OPEN,
+ /** \brief We got a IMS hit, the file has not changed */
+ IMS_HIT,
+ /** \brief The server reported a unrecoverable error */
+ ERROR_UNRECOVERABLE,
+ /** \brief The server reported a error with a error content page */
+ ERROR_WITH_CONTENT_PAGE,
+ /** \brief An error on the client side */
+ ERROR_NOT_FROM_SERVER,
+ /** \brief A redirect or retry request */
+ TRY_AGAIN_OR_REDIRECT
+ };
+ /** \brief Handle the retrieved header data */
+ DealWithHeadersResult DealWithHeaders(FetchResult &Res);
+
+ // In the event of a fatal signal this file will be closed and timestamped.
+ static std::string FailFile;
+ static int FailFd;
+ static time_t FailTime;
+ static void SigTerm(int);
+
+ virtual bool Configuration(std::string Message);
+ virtual bool Flush() { return Server->Flush(File); };
+
+ int Loop();
+
+ virtual void SendReq(FetchItem *Itm) = 0;
+ virtual ServerState * CreateServerState(URI uri) = 0;
+ virtual void RotateDNS() = 0;
+
+ ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(0), AllowRedirect(false), Debug(false) {};
+ virtual ~ServerMethod() {};
+};
+
+#endif
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-01 21:27:34 +0000