From 8b89e57fa2ae7d34b055b8f804cee0c2c194043b Mon Sep 17 00:00:00 2001 From: Arch Librarian Date: Mon, 20 Sep 2004 16:51:12 +0000 Subject: Stable acquire code Author: jgg Date: 1998-10-26 07:11:43 GMT Stable acquire code --- apt-pkg/acquire-item.cc | 86 ++++++++++++++++++++++++++++++++++++++++------ apt-pkg/acquire-item.h | 9 +++-- apt-pkg/acquire-worker.cc | 43 ++++++++++++++++++++--- apt-pkg/acquire-worker.h | 5 ++- apt-pkg/acquire.cc | 12 ++++++- apt-pkg/acquire.h | 3 +- apt-pkg/contrib/fileutl.cc | 4 +-- apt-pkg/contrib/fileutl.h | 4 +-- doc/examples/apt.conf | 4 ++- methods/copy.cc | 3 +- methods/gzip.cc | 14 ++++++-- 11 files changed, 157 insertions(+), 30 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index e6eee197b..4435e2553 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire-item.cc,v 1.4 1998/10/24 04:57:56 jgg Exp $ +// $Id: acquire-item.cc,v 1.5 1998/10/26 07:11:43 jgg Exp $ /* ###################################################################### Acquire Item - Item to acquire @@ -65,6 +65,22 @@ void pkgAcquire::Item::Done(string,unsigned long,string) Owner->Dequeue(this); } /*}}}*/ +// Acquire::Item::Rename - Rename a file /*{{{*/ +// --------------------------------------------------------------------- +/* This helper function is used by alot of item methods as thier final + step */ +void pkgAcquire::Item::Rename(string From,string To) +{ + if (rename(From.c_str(),To.c_str()) != 0) + { + char S[300]; + sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno), + From.c_str(),To.c_str()); + Status = StatError; + ErrorText = S; + } +} + /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -73,6 +89,8 @@ void pkgAcquire::Item::Done(string,unsigned long,string) pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) : Item(Owner), Location(Location) { + Decompression = false; + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(Location->PackagesURI()); @@ -97,6 +115,57 @@ string pkgAcqIndex::Custom600Headers() return "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ +// AcqIndex::Done - Finished a fetch /*{{{*/ +// --------------------------------------------------------------------- +/* This goes through a number of states.. On the initial fetch the + method could possibly return an alternate filename which points + to the uncompressed version of the file. If this is so the file + is copied into the partial directory. In all other cases the file + is decompressed with a gzip uri. */ +void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5) +{ + Item::Done(Message,Size,MD5); + + if (Decompression == true) + { + // Done, move it into position + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(Location->PackagesURI()); + Rename(DestFile,FinalFile); + return; + } + + // Handle the unzipd case + string FileName = LookupTag(Message,"Alt-Filename"); + if (FileName.empty() == false) + { + // The files timestamp matches + if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true) + return; + + Decompression = true; + DestFile += ".decomp"; + QueueURI("copy:" + FileName,string()); + return; + } + + FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + } + + // The files timestamp matches + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + return; + + Decompression = true; + DestFile += ".decomp"; + QueueURI("gzip:" + FileName,string()); +} + /*}}}*/ + // AcqIndexRel::pkgAcqIndexRel - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The Release file is added to the queue */ @@ -139,7 +208,12 @@ void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5) { Status = StatError; ErrorText = "Method gave a blank filename"; + return; } + + // The files timestamp matches + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + return; // We have to copy it into place if (FileName != DestFile) @@ -151,14 +225,6 @@ void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5) // Done, move it into position string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(Location->ReleaseURI()); - - if (rename(DestFile.c_str(),FinalFile.c_str()) != 0) - { - char S[300]; - sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno), - DestFile.c_str(),FinalFile.c_str()); - Status = StatError; - ErrorText = S; - } + Rename(DestFile,FinalFile); } /*}}}*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 352fc3e24..bdd4d3581 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire-item.h,v 1.3 1998/10/24 04:57:57 jgg Exp $ +// $Id: acquire-item.h,v 1.4 1998/10/26 07:11:44 jgg Exp $ /* ###################################################################### Acquire Item - Item to acquire @@ -33,6 +33,8 @@ class pkgAcquire::Item inline void QueueURI(string URI,string Description) {Owner->Enqueue(this,URI,Description);}; + void Rename(string From,string To); + public: // State of the item @@ -60,9 +62,11 @@ class pkgAcqIndex : public pkgAcquire::Item protected: const pkgSourceList::Item *Location; + bool Decompression; public: + virtual void Done(string Message,unsigned long Size,string Md5Hash); virtual string Custom600Headers(); pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location); @@ -77,8 +81,7 @@ class pkgAcqIndexRel : public pkgAcquire::Item public: - virtual void Done(string Message,unsigned long Size,string Md5Hash); - + virtual void Done(string Message,unsigned long Size,string Md5Hash); virtual string Custom600Headers(); pkgAcqIndexRel(pkgAcquire *Owner,const pkgSourceList::Item *Location); diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 936d469c3..392c8ca47 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire-worker.cc,v 1.6 1998/10/24 04:57:58 jgg Exp $ +// $Id: acquire-worker.cc,v 1.7 1998/10/26 07:11:45 jgg Exp $ /* ###################################################################### Acquire Worker @@ -106,7 +106,9 @@ bool pkgAcquire::Worker::Start() close(Pipes[I]); return false; } - + for (int I = 0; I != 4; I++) + SetCloseExec(Pipes[0],true); + // Fork off the process Process = fork(); if (Process < 0) @@ -122,8 +124,6 @@ bool pkgAcquire::Worker::Start() dup2(Pipes[1],STDOUT_FILENO); dup2(Pipes[2],STDIN_FILENO); dup2(((filebuf *)clog.rdbuf())->fd(),STDERR_FILENO); - for (int I = 0; I != 4; I++) - close(Pipes[I]); SetCloseExec(STDOUT_FILENO,false); SetCloseExec(STDIN_FILENO,false); SetCloseExec(STDERR_FILENO,false); @@ -152,7 +152,8 @@ bool pkgAcquire::Worker::Start() return _error->Error("Method %s did not start correctly",Method.c_str()); RunMessages(); - SendConfiguration(); + if (OwnerQ != 0) + SendConfiguration(); return true; } @@ -416,3 +417,35 @@ bool pkgAcquire::Worker::MethodFailure() return false; } /*}}}*/ + +// InjectConfiguration - Configuration aid for methods /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgInjectConfiguration(string &Message,Configuration &Cnf) +{ + const char *I = Message.begin(); + + unsigned int Length = strlen("Config-Item"); + for (; I + Length < Message.end(); I++) + { + // Not a config item + if (I[Length] != ':' || stringcasecmp(I,I+Length,"Config-Item") != 0) + continue; + + I += Length + 1; + + for (; I < Message.end() && *I == ' '; I++); + const char *Equals = I; + for (; Equals < Message.end() && *Equals != '='; Equals++); + const char *End = Equals; + for (; End < Message.end() && *End != '\n'; End++); + if (End == Equals) + return false; + + Cnf.Set(string(I,Equals-I),string(Equals+1,End-Equals-1)); + I = End; + } + + return true; +} + /*}}}*/ diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h index eb04485b9..b0acde3e3 100644 --- a/apt-pkg/acquire-worker.h +++ b/apt-pkg/acquire-worker.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire-worker.h,v 1.4 1998/10/24 04:57:59 jgg Exp $ +// $Id: acquire-worker.h,v 1.5 1998/10/26 07:11:46 jgg Exp $ /* ###################################################################### Acquire Worker - Worker process manager @@ -13,6 +13,7 @@ #define PKGLIB_ACQUIRE_WORKER_H #include +#include #ifdef __GNUG__ #pragma interface "apt-pkg/acquire-worker.h" @@ -80,4 +81,6 @@ class pkgAcquire::Worker ~Worker(); }; +bool pkgInjectConfiguration(string &Message,Configuration &Cnf); + #endif diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 21aade75d..4ed21831d 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire.cc,v 1.4 1998/10/24 04:58:01 jgg Exp $ +// $Id: acquire.cc,v 1.5 1998/10/26 07:11:47 jgg Exp $ /* ###################################################################### Acquire - File Acquiration @@ -33,6 +33,7 @@ pkgAcquire::pkgAcquire() Configs = 0; Workers = 0; ToFetch = 0; + Running = false; string Mode = _config->Find("Acquire::Queue-Mode","host"); if (strcasecmp(Mode.c_str(),"host") == 0) @@ -133,6 +134,9 @@ void pkgAcquire::Enqueue(Item *Itm,string URI,string Description) // Queue it into the named queue I->Enqueue(Itm,URI,Description); ToFetch++; + + if (Running == true) + I->Startup(); // Some trace stuff if (Debug == true) @@ -249,6 +253,8 @@ void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet) manage the actual fetch. */ bool pkgAcquire::Run() { + Running = true; + for (Queue *I = Queues; I != 0; I = I->Next) I->Startup(); @@ -263,7 +269,10 @@ bool pkgAcquire::Run() SetFds(Highest,&RFds,&WFds); if (select(Highest+1,&RFds,&WFds,0,0) <= 0) + { + Running = false; return _error->Errno("select","Select has failed"); + } RunFds(&RFds,&WFds); } @@ -271,6 +280,7 @@ bool pkgAcquire::Run() for (Queue *I = Queues; I != 0; I = I->Next) I->Shutdown(); + Running = false; return true; } /*}}}*/ diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index d4dbed5f6..2e111cccd 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire.h,v 1.4 1998/10/24 04:58:02 jgg Exp $ +// $Id: acquire.h,v 1.5 1998/10/26 07:11:48 jgg Exp $ /* ###################################################################### Acquire - File Acquiration @@ -66,6 +66,7 @@ class pkgAcquire // Configurable parameters for the schedular enum {QueueHost,QueueAccess} QueueMode; bool Debug; + bool Running; void Add(Item *Item); void Remove(Item *Item); diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index ad1fcf184..77e846117 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: fileutl.cc,v 1.12 1998/10/24 22:15:41 jgg Exp $ +// $Id: fileutl.cc,v 1.13 1998/10/26 07:11:49 jgg Exp $ /* ###################################################################### File Utilities @@ -29,7 +29,7 @@ // CopyFile - Buffered copy of a file /*{{{*/ // --------------------------------------------------------------------- /* The caller is expected to set things so that failure causes erasure */ -bool CopyFile(FileFd From,FileFd To) +bool CopyFile(FileFd &From,FileFd &To) { if (From.IsOpen() == false || To.IsOpen() == false) return false; diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 7a72df09f..01dc46b13 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: fileutl.h,v 1.7 1998/10/20 02:39:29 jgg Exp $ +// $Id: fileutl.h,v 1.8 1998/10/26 07:11:50 jgg Exp $ /* ###################################################################### File Utilities @@ -59,7 +59,7 @@ class FileFd virtual ~FileFd(); }; -bool CopyFile(FileFd From,FileFd To); +bool CopyFile(FileFd &From,FileFd &To); int GetLock(string File,bool Errors = true); bool FileExists(string File); string SafeGetCWD(); diff --git a/doc/examples/apt.conf b/doc/examples/apt.conf index 04d4fbc85..dc2f9cc3a 100644 --- a/doc/examples/apt.conf +++ b/doc/examples/apt.conf @@ -1,4 +1,4 @@ -// $Id: apt.conf,v 1.5 1998/10/24 04:58:10 jgg Exp $ +// $Id: apt.conf,v 1.6 1998/10/26 07:11:51 jgg Exp $ /* This file is an index of all APT configuration directives. It should NOT actually be used as a real config file, though it is a completely valid file. @@ -60,3 +60,5 @@ Debug { pkgAcquire "false"; pkgAcquire::Worker "true"; } + +dir::state::lists "/tmp/lists/"; diff --git a/methods/copy.cc b/methods/copy.cc index c1cc26a69..b1c0fe360 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: copy.cc,v 1.2 1998/10/25 07:07:29 jgg Exp $ +// $Id: copy.cc,v 1.3 1998/10/26 07:11:52 jgg Exp $ /* ###################################################################### Copy URI - This method takes a uri like a file: uri and copies it @@ -121,6 +121,7 @@ int main() TimeBuf.modtime = Buf.st_mtime; if (utime(Target.c_str(),&TimeBuf) != 0) { + To.OpFail(); _error->Errno("utime","Failed to set modification time"); Fail(URI); continue; diff --git a/methods/gzip.cc b/methods/gzip.cc index 15bff4d83..84ad472e9 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: gzip.cc,v 1.1 1998/10/25 07:07:30 jgg Exp $ +// $Id: gzip.cc,v 1.2 1998/10/26 07:11:53 jgg Exp $ /* ###################################################################### GZip method - Take a file URI in and decompress it into the target @@ -123,20 +123,25 @@ int main() { dup2(From.Fd(),STDIN_FILENO); dup2(To.Fd(),STDOUT_FILENO); + From.Close(); + To.Close(); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDOUT_FILENO,false); const char *Args[3]; - Args[0] = _config->FindFile("Dir::bin::gzip","gzip").c_str(); + Args[0] = _config->Find("Dir::bin::gzip","gzip").c_str(); Args[1] = "-d"; Args[2] = 0; execvp(Args[0],(char **)Args); + exit(100); } From.Close(); - To.Close(); // Wait for gzip to finish int Status; if (waitpid(Process,&Status,0) != Process) { + To.OpFail(); _error->Errno("wait","Waiting for gzip failed"); Fail(URI); continue; @@ -144,11 +149,14 @@ int main() if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0) { + To.OpFail(); _error->Error("gzip failed, perhaps the disk is full or the directory permissions are wrong."); Fail(URI); continue; } + To.Close(); + // Transfer the modification times struct stat Buf; if (stat(File.c_str(),&Buf) != 0) -- cgit v1.2.3