diff options
Diffstat (limited to 'apt-pkg')
42 files changed, 3512 insertions, 382 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index e0e0611f0..966126255 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -24,6 +24,8 @@ #include <apt-pkg/strutl.h> #include <apt-pkg/fileutl.h> #include <apt-pkg/md5.h> +#include <apt-pkg/sha1.h> +#include <apt-pkg/tagfile.h> #include <apti18n.h> @@ -31,6 +33,7 @@ #include <unistd.h> #include <errno.h> #include <string> +#include <sstream> #include <stdio.h> /*}}}*/ @@ -100,7 +103,8 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string, { // We just downloaded something.. string FileName = LookupTag(Message,"Filename"); - if (Complete == false && FileName == DestFile) + // we only inform the Log class if it was actually not a local thing + if (Complete == false && !Local && FileName == DestFile) { if (Owner->Log != 0) Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str())); @@ -131,14 +135,430 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ +// AcqDiffIndex::AcqDiffIndex - Constructor +// --------------------------------------------------------------------- +/* Get the DiffIndex file first and see if there are patches availabe + * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the + * patches. If anything goes wrong in that process, it will fall back to + * the original packages file + */ +pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), Description(URIDesc) +{ + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Description = URIDesc + "/DiffIndex"; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + Desc.URI = URI + ".diff/Index"; + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI) + string(".DiffIndex"); + + if(Debug) + std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; + + // look for the current package file + CurrentPackagesFile = _config->FindDir("Dir::State::lists"); + CurrentPackagesFile += URItoFileName(RealURI); + + // FIXME: this file:/ check is a hack to prevent fetching + // from local sources. this is really silly, and + // should be fixed cleanly as soon as possible + if(!FileExists(CurrentPackagesFile) || + Desc.URI.substr(0,strlen("file:/")) == "file:/") + { + // we don't have a pkg file or we don't want to queue + if(Debug) + std::clog << "No index file, local or canceld by user" << std::endl; + Failed("", NULL); + return; + } + + if(Debug) + std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " + << CurrentPackagesFile << std::endl; + + QueueURI(Desc); + +} + +// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqDiffIndex::Custom600Headers() +{ + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(RealURI) + string(".IndexDiff"); + + if(Debug) + std::clog << "Custom600Header-IMS: " << Final << std::endl; + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true"; + + return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} + + +bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile + << std::endl; + + pkgTagSection Tags; + string ServerSha1; + vector<DiffInfo> available_patches; + + FileFd Fd(IndexDiffFile,FileFd::ReadOnly); + pkgTagFile TF(&Fd); + if (_error->PendingError() == true) + return false; + + if(TF.Step(Tags) == true) + { + string local_sha1; + bool found = false; + DiffInfo d; + string size; + + string tmp = Tags.FindS("SHA1-Current"); + std::stringstream ss(tmp); + ss >> ServerSha1; + + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + local_sha1 = string(SHA1.Result()); + + if(local_sha1 == ServerSha1) + { + // we have the same sha1 as the server + if(Debug) + std::clog << "Package file is up-to-date" << std::endl; + // set found to true, this will queue a pkgAcqIndexDiffs with + // a empty availabe_patches + found = true; + } + else + { + if(Debug) + std::clog << "SHA1-Current: " << ServerSha1 << std::endl; + + // check the historie and see what patches we need + string history = Tags.FindS("SHA1-History"); + std::stringstream hist(history); + while(hist >> d.sha1 >> size >> d.file) + { + d.size = atoi(size.c_str()); + // read until the first match is found + if(d.sha1 == local_sha1) + found=true; + // from that point on, we probably need all diffs + if(found) + { + if(Debug) + std::clog << "Need to get diff: " << d.file << std::endl; + available_patches.push_back(d); + } + } + } + + // no information how to get the patches, bail out + if(!found) + { + if(Debug) + std::clog << "Can't find a patch in the index file" << std::endl; + // Failed will queue a big package file + Failed("", NULL); + } + else + { + // queue the diffs + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + Complete = false; + Status = StatDone; + Dequeue(); + return true; + } + } + + return false; +} + +void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl + << "Falling back to normal index file aquire" << std::endl; + + new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5); + + Complete = false; + Status = StatDone; + Dequeue(); +} + +void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + string FinalFile; + FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); + + // sucess in downloading the index + // rename the index + FinalFile += string(".IndexDiff"); + if(Debug) + std::clog << "Renaming: " << DestFile << " -> " << FinalFile + << std::endl; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + + if(!ParseDiffIndex(DestFile)) + return Failed("", NULL); + + Complete = true; + Status = StatDone; + Dequeue(); + return; +} + + + +// AcqIndexDiffs::AcqIndexDiffs - Constructor +// --------------------------------------------------------------------- +/* The package diff is added to the queue. one object is constructed + * for each diff and the index + */ +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5, vector<DiffInfo> diffs) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), + available_patches(diffs) +{ + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + if(available_patches.size() == 0) + { + // we are done (yeah!) + Finish(true); + } + else + { + // get the next diff + State = StateFetchDiff; + QueueNextDiff(); + } +} + + +void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl + << "Falling back to normal index file aquire" << std::endl; + new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, + ExpectedMD5); + Finish(); +} + + +// helper that cleans the item out of the fetcher queue +void pkgAcqIndexDiffs::Finish(bool allDone) +{ + // we restore the original name, this is required, otherwise + // the file will be cleaned + if(allDone) + { + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + + // do the final md5sum checking + MD5Summation sum; + FileFd Fd(DestFile, FileFd::ReadOnly); + sum.AddFD(Fd.Fd(), Fd.Size()); + Fd.Close(); + string MD5 = (string)sum.Result(); + + if (!ExpectedMD5.empty() && MD5 != ExpectedMD5) + { + Status = StatAuthError; + ErrorText = _("MD5Sum mismatch"); + Rename(DestFile,DestFile + ".FAILED"); + Dequeue(); + return; + } + + // this is for the "real" finish + Complete = true; + Status = StatDone; + Dequeue(); + if(Debug) + std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; + return; + } + + if(Debug) + std::clog << "Finishing: " << Desc.URI << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + return; +} + + + +bool pkgAcqIndexDiffs::QueueNextDiff() +{ + + // calc sha1 of the just patched file + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + FileFd fd(FinalFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + string local_sha1 = string(SHA1.Result()); + if(Debug) + std::clog << "QueueNextDiff: " + << FinalFile << " (" << local_sha1 << ")"<<std::endl; + + // remove all patches until the next matching patch is found + // this requires the Index file to be ordered + for(vector<DiffInfo>::iterator I=available_patches.begin(); + available_patches.size() > 0 && + I != available_patches.end() && + (*I).sha1 != local_sha1; + I++) + { + available_patches.erase(I); + } + + // error checking and falling back if no patch was found + if(available_patches.size() == 0) + { + Failed("", NULL); + return false; + } + + // queue the right diff + Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz"; + Desc.Description = available_patches[0].file + string(".pdiff"); + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file); + + if(Debug) + std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; + + QueueURI(Desc); + + return true; +} + + + +void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + string FinalFile; + FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); + + // sucess in downloading a diff, enter ApplyDiff state + if(State == StateFetchDiff) + { + + if(Debug) + std::clog << "Sending to gzip method: " << FinalFile << std::endl; + + string FileName = LookupTag(Message,"Filename"); + State = StateUnzipDiff; + Local = true; + Desc.URI = "gzip:" + FileName; + DestFile += ".decomp"; + QueueURI(Desc); + Mode = "gzip"; + return; + } + + // sucess in downloading a diff, enter ApplyDiff state + if(State == StateUnzipDiff) + { + + // rred excepts the patch as $FinalFile.ed + Rename(DestFile,FinalFile+".ed"); + + if(Debug) + std::clog << "Sending to rred method: " << FinalFile << std::endl; + + State = StateApplyDiff; + Local = true; + Desc.URI = "rred:" + FinalFile; + QueueURI(Desc); + Mode = "rred"; + return; + } + + + // success in download/apply a diff, queue next (if needed) + if(State == StateApplyDiff) + { + // remove the just applied patch + available_patches.erase(available_patches.begin()); + + // move into place + if(Debug) + { + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << FinalFile << std::endl; + } + Rename(DestFile,FinalFile); + + // see if there is more to download + if(available_patches.size() > 0) { + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + return Finish(); + } else + return Finish(true); + } +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - string ExpectedMD5, string comprExt) : - Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) + string ExpectedMD5, string comprExt) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) { Decompression = false; Erase = false; @@ -184,7 +604,7 @@ string pkgAcqIndex::Custom600Headers() void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { // no .bz2 found, retry with .gz - if(Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1) == "bz2") { + if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") { Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz"; // retry with a gzip one @@ -290,7 +710,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5, else Local = true; - string compExt = Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1); + string compExt = Desc.URI.substr(Desc.URI.size()-3); char *decompProg; if(compExt == "bz2") decompProg = "bzip2"; @@ -344,10 +764,9 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, const vector<IndexTarget*>* IndexTargets, indexRecords* MetaIndexParser) : Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI), - MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc) + MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), + MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) { - this->MetaIndexParser = MetaIndexParser; - this->IndexTargets = IndexTargets; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -370,12 +789,6 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, // File was already in place. It needs to be re-verified // because Release might have changed, so Move it into partial Rename(Final,DestFile); - // unlink the file and do not try to use I-M-S and Last-Modified - // if the users proxy is broken - if(_config->FindB("Acquire::BrokenProxy", false) == true) { - std::cerr << "forcing re-get of the signature file as requested" << std::endl; - unlink(DestFile.c_str()); - } } QueueURI(Desc); @@ -425,18 +838,19 @@ void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5, /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - // Delete any existing sigfile, so that this source isn't - // mistakenly trusted - string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - unlink(Final.c_str()); - // if we get a timeout if fail + // if we get a network error we fail gracefully if(LookupTag(Message,"FailReason") == "Timeout" || - LookupTag(Message,"FailReason") == "TmpResolveFailure") { + LookupTag(Message,"FailReason") == "TmpResolveFailure" || + LookupTag(Message,"FailReason") == "ConnectionRefused") { Item::Failed(Message,Cnf); return; } + // Delete any existing sigfile when the acquire failed + string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + unlink(Final.c_str()); + // queue a pkgAcqMetaIndex with no sigfile new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, "", IndexTargets, MetaIndexParser); @@ -459,11 +873,9 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, string SigFile, const vector<struct IndexTarget*>* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner), RealURI(URI), SigFile(SigFile) + Item(Owner), RealURI(URI), SigFile(SigFile), AuthPass(false), + MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets), IMSHit(false) { - this->AuthPass = false; - this->MetaIndexParser = MetaIndexParser; - this->IndexTargets = IndexTargets; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -555,6 +967,9 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) return; } + // see if the download was a IMSHit + IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false); + Complete = true; string FinalFile = _config->FindDir("Dir::State::lists"); @@ -583,7 +998,7 @@ void pkgAcqMetaIndex::AuthDone(string Message) return; } - if (!VerifyVendor()) + if (!VerifyVendor(Message)) { return; } @@ -635,13 +1050,18 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) } } - // Queue Packages file - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexMD5); + // Queue Packages file (either diff or full packages files, depending + // on the users option) + if(_config->FindB("Acquire::PDiffs",true) == true) + new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexMD5); + else + new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexMD5); } } -bool pkgAcqMetaIndex::VerifyVendor() +bool pkgAcqMetaIndex::VerifyVendor(string Message) { // // Maybe this should be made available from above so we don't have // // to read and parse it every time? @@ -666,6 +1086,22 @@ bool pkgAcqMetaIndex::VerifyVendor() // break; // } // } + string::size_type pos; + + // check for missing sigs (that where not fatal because otherwise we had + // bombed earlier) + string missingkeys; + string msg = _("There are no public key available for the " + "following key IDs:\n"); + pos = Message.find("NO_PUBKEY "); + if (pos != std::string::npos) + { + string::size_type start = pos+strlen("NO_PUBKEY "); + string Fingerprint = Message.substr(start, Message.find("\n")-start); + missingkeys += (Fingerprint); + } + if(!missingkeys.empty()) + _error->Warning("%s", string(msg+missingkeys).c_str()); string Transformed = MetaIndexParser->GetExpectedDist(); @@ -674,7 +1110,7 @@ bool pkgAcqMetaIndex::VerifyVendor() Transformed = "experimental"; } - string::size_type pos = Transformed.rfind('/'); + pos = Transformed.rfind('/'); if (pos != string::npos) { Transformed = Transformed.substr(0, pos); @@ -720,10 +1156,30 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { if (AuthPass == true) { - // gpgv method failed + // if we fail the authentication but got the file via a IMS-Hit + // this means that the file wasn't downloaded and that it might be + // just stale (server problem, proxy etc). we delete what we have + // queue it again without i-m-s + // alternatively we could just unlink the file and let the user try again + if (IMSHit) + { + Complete = false; + Local = false; + AuthPass = false; + unlink(DestFile.c_str()); + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + Desc.URI = RealURI; + QueueURI(Desc); + return; + } + + // gpgv method failed _error->Warning("GPG error: %s: %s", Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); + } // No Release file was present, or verification failed, so fall @@ -799,6 +1255,12 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, } } + // "allow-unauthenticated" restores apts old fetching behaviour + // that means that e.g. unauthenticated file:// uris are higher + // priority than authenticated http:// uris + if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true) + Trusted = false; + // Select a source if (QueueNext() == false && _error->PendingError() == false) _error->Error(_("I wasn't able to locate file for the %s package. " @@ -1031,13 +1493,19 @@ void pkgAcqArchive::Finished() // --------------------------------------------------------------------- /* The file is added to the queue */ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5, - unsigned long Size,string Dsc,string ShortDesc) : + unsigned long Size,string Dsc,string ShortDesc, + const string &DestDir, const string &DestFilename) : Item(Owner), Md5Hash(MD5) { Retries = _config->FindI("Acquire::Retries",0); - DestFile = flNotDir(URI); - + if(!DestFilename.empty()) + DestFile = DestFilename; + else if(!DestDir.empty()) + DestFile = DestDir + "/" + flNotDir(URI); + else + DestFile = flNotDir(URI); + // Create the item Desc.URI = URI; Desc.Description = Dsc; @@ -1057,7 +1525,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5, else PartialSize = Buf.st_size; } - + QueueURI(Desc); } /*}}}*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index ae2fba4d5..217ddb3ef 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -31,67 +31,486 @@ #pragma interface "apt-pkg/acquire-item.h" #endif -// Item to acquire +/** \addtogroup acquire + * @{ + * + * \file acquire-item.h + */ + +/** \brief Represents the process by which a pkgAcquire object should + * retrieve a file or a collection of files. + * + * By convention, Item subclasses should insert themselves into the + * acquire queue when they are created by calling QueueURI(), and + * remove themselves by calling Dequeue() when either Done() or + * Failed() is invoked. Item objects are also responsible for + * notifying the download progress indicator (accessible via + * #Owner->Log) of their status. + * + * \see pkgAcquire + */ class pkgAcquire::Item { protected: - // Some private helper methods for registering URIs + /** \brief The acquire object with which this item is associated. */ pkgAcquire *Owner; + + /** \brief Insert this item into its owner's queue. + * + * \param ItemDesc Metadata about this item (its URI and + * description). + */ inline void QueueURI(ItemDesc &Item) {Owner->Enqueue(Item);}; + + /** \brief Remove this item from its owner's queue. */ inline void Dequeue() {Owner->Dequeue(this);}; - // Safe rename function with timestamp preservation + /** \brief Rename a file without modifying its timestamp. + * + * Many item methods call this as their final action. + * + * \param From The file to be renamed. + * + * \param To The new name of #From. If #To exists it will be + * overwritten. + */ void Rename(string From,string To); public: - // State of the item - enum {StatIdle, StatFetching, StatDone, StatError, StatAuthError} Status; + /** \brief The current status of this item. */ + enum ItemState + { + /** \brief The item is waiting to be downloaded. */ + StatIdle, + + /** \brief The item is currently being downloaded. */ + StatFetching, + + /** \brief The item has been successfully downloaded. */ + StatDone, + + /** \brief An error was encountered while downloading this + * item. + */ + StatError, + + /** \brief The item was downloaded but its authenticity could + * not be verified. + */ + StatAuthError + } Status; + + /** \brief Contains a textual description of the error encountered + * if #Status is #StatError or #StatAuthError. + */ string ErrorText; + + /** \brief The size of the object to fetch. */ unsigned long FileSize; - unsigned long PartialSize; + + /** \brief How much of the object was already fetched. */ + unsigned long PartialSize; + + /** \brief If not \b NULL, contains the name of a subprocess that + * is operating on this object (for instance, "gzip" or "gpgv"). + */ const char *Mode; + + /** \brief A client-supplied unique identifier. + * + * This field is initalized to 0; it is meant to be filled in by + * clients that wish to use it to uniquely identify items. + * + * \todo it's unused in apt itself + */ unsigned long ID; + + /** \brief If \b true, the entire object has been successfully fetched. + * + * Subclasses should set this to \b true when appropriate. + */ bool Complete; + + /** \brief If \b true, the URI of this object is "local". + * + * The only effect of this field is to exclude the object from the + * download progress indicator's overall statistics. + */ bool Local; - // Number of queues we are inserted into + /** \brief The number of fetch queues into which this item has been + * inserted. + * + * There is one queue for each source from which an item could be + * downloaded. + * + * \sa pkgAcquire + */ unsigned int QueueCounter; - // File to write the fetch into + /** \brief The name of the file into which the retrieved object + * will be written. + */ string DestFile; - // Action members invoked by the worker + /** \brief Invoked by the acquire worker when the object couldn't + * be fetched. + * + * This is a branch of the continuation of the fetch process. + * + * \param Message An RFC822-formatted message from the acquire + * method describing what went wrong. Use LookupTag() to parse + * it. + * + * \param Cnf The method via which the worker tried to fetch this object. + * + * \sa pkgAcqMethod + */ virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + + /** \brief Invoked by the acquire worker when the object was + * fetched successfully. + * + * Note that the object might \e not have been written to + * DestFile; check for the presence of an Alt-Filename entry in + * Message to find the file to which it was really written. + * + * Done is often used to switch from one stage of the processing + * to the next (e.g. fetching, unpacking, copying). It is one + * branch of the continuation of the fetch process. + * + * \param Message Data from the acquire method. Use LookupTag() + * to parse it. + * \param Size The size of the object that was fetched. + * \param Md5Hash The MD5Sum of the object that was fetched. + * \param Cnf The method via which the object was fetched. + * + * \sa pkgAcqMethod + */ virtual void Done(string Message,unsigned long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); + + /** \brief Invoked when the worker starts to fetch this object. + * + * \param Message RFC822-formatted data from the worker process. + * Use LookupTag() to parse it. + * + * \param Size The size of the object being fetched. + * + * \sa pkgAcqMethod + */ virtual void Start(string Message,unsigned long Size); + + /** \brief Custom headers to be sent to the fetch process. + * + * \return a string containing RFC822-style headers that are to be + * inserted into the 600 URI Acquire message sent to the fetch + * subprocess. The headers are inserted after a newline-less + * line, so they should (if nonempty) have a leading newline and + * no trailing newline. + */ virtual string Custom600Headers() {return string();}; + + /** \brief A "descriptive" URI-like string. + * + * \return a URI that should be used to describe what is being fetched. + */ virtual string DescURI() = 0; + /** \brief Short item description. + * + * \return a brief description of the object being fetched. + */ virtual string ShortDesc() {return DescURI();} + + /** \brief Invoked by the worker when the download is completely done. */ virtual void Finished() {}; - // Inquire functions + /** \brief MD5Sum. + * + * \return the MD5Sum of this object, if applicable; otherwise, an + * empty string. + */ virtual string MD5Sum() {return string();}; + + /** \return the acquire process with which this item is associated. */ pkgAcquire *GetOwner() {return Owner;}; + + /** \return \b true if this object is being fetched from a trusted source. */ virtual bool IsTrusted() {return false;}; - + + /** \brief Initialize an item. + * + * Adds the item to the list of items known to the acquire + * process, but does not place it into any fetch queues (you must + * manually invoke QueueURI() to do so). + * + * Initializes all fields of the item other than Owner to 0, + * false, or the empty string. + * + * \param Owner The new owner of this item. + */ Item(pkgAcquire *Owner); + + /** \brief Remove this item from its owner's queue by invoking + * pkgAcquire::Remove. + */ virtual ~Item(); }; -// Item class for index files -class pkgAcqIndex : public pkgAcquire::Item +/** \brief Information about an index patch (aka diff). */ +struct DiffInfo { + /** The filename of the diff. */ + string file; + + /** The sha1 hash of the diff. */ + string sha1; + + /** The size of the diff. */ + unsigned long size; +}; + +/** \brief An item that is responsible for fetching an index file of + * package list diffs and starting the package list's download. + * + * This item downloads the Index file and parses it, then enqueues + * additional downloads of either the individual patches (using + * pkgAcqIndexDiffs) or the entire Packages file (using pkgAcqIndex). + * + * \sa pkgAcqIndexDiffs, pkgAcqIndex + */ +class pkgAcqDiffIndex : public pkgAcquire::Item { + protected: + /** \brief If \b true, debugging information will be written to std::clog. */ + bool Debug; + + /** \brief The item that is currently being downloaded. */ + pkgAcquire::ItemDesc Desc; + + /** \brief The URI of the index file to recreate at our end (either + * by downloading it or by applying partial patches). + */ + string RealURI; + + /** \brief The MD5Sum that the real index file should have after + * all patches have been applied. + */ + string ExpectedMD5; + + /** \brief The index file which will be patched to generate the new + * file. + */ + string CurrentPackagesFile; + + /** \brief A description of the Packages file (stored in + * pkgAcquire::ItemDesc::Description). + */ + string Description; + + public: + // Specialized action members + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual string DescURI() {return RealURI + "Index";}; + virtual string Custom600Headers(); + + /** \brief Parse the Index file for a set of Packages diffs. + * + * Parses the Index file and creates additional download items as + * necessary. + * + * \param IndexDiffFile The name of the Index file. + * + * \return \b true if the Index file was successfully parsed, \b + * false otherwise. + */ + bool ParseDiffIndex(string IndexDiffFile); + + + /** \brief Create a new pkgAcqDiffIndex. + * + * \param Owner The Acquire object that owns this item. + * + * \param URI The URI of the list file to download. + * + * \param URIDesc A long description of the list file to download. + * + * \param ShortDesc A short description of the list file to download. + * + * \param ExpectedMD5 The list file's MD5 signature. + */ + pkgAcqDiffIndex(pkgAcquire *Owner,string URI,string URIDesc, + string ShortDesc, string ExpectedMD5); +}; + +/** \brief An item that is responsible for fetching all the patches + * that need to be applied to a given package index file. + * + * After downloading and applying a single patch, this item will + * enqueue a new pkgAcqIndexDiffs to download and apply the remaining + * patches. If no patch can be found that applies to an intermediate + * file or if one of the patches cannot be downloaded, falls back to + * downloading the entire package index file using pkgAcqIndex. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ +class pkgAcqIndexDiffs : public pkgAcquire::Item +{ + private: + + /** \brief Queue up the next diff download. + * + * Search for the next available diff that applies to the file + * that currently exists on disk, and enqueue it by calling + * QueueURI(). + * + * \return \b true if an applicable diff was found, \b false + * otherwise. + */ + bool QueueNextDiff(); + + /** \brief Handle tasks that must be performed after the item + * finishes downloading. + * + * Dequeues the item and checks the resulting file's md5sum + * against ExpectedMD5 after the last patch was applied. + * There is no need to check the md5/sha1 after a "normal" + * patch because QueueNextDiff() will check the sha1 later. + * + * \param allDone If \b true, the file was entirely reconstructed, + * and its md5sum is verified. + */ + void Finish(bool allDone=false); + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ + bool Debug; + + /** \brief A description of the item that is currently being + * downloaded. + */ + pkgAcquire::ItemDesc Desc; + + /** \brief The URI of the package index file that is being + * reconstructed. + */ + string RealURI; + + /** \brief The MD5Sum of the package index file that is being + * reconstructed. + */ + string ExpectedMD5; + + /** A description of the file being downloaded. */ + string Description; + + /** The patches that remain to be downloaded, including the patch + * being downloaded right now. This list should be ordered so + * that each diff appears before any diff that depends on it. + * + * \todo These are indexed by sha1sum; why not use some sort of + * dictionary instead of relying on ordering and stripping them + * off the front? + */ + vector<DiffInfo> available_patches; + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is in an unknown state. */ + StateFetchUnkown, + + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being uncompressed. */ + StateUnzipDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff + } State; + + public: + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + + virtual void Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual string DescURI() {return RealURI + "Index";}; + + /** \brief Create an index diff item. + * + * After filling in its basic fields, this invokes Finish(true) if + * #diffs is empty, or QueueNextDiff() otherwise. + * + * \param Owner The pkgAcquire object that owns this item. + * + * \param URI The URI of the package index file being + * reconstructed. + * + * \param URIDesc A long description of this item. + * + * \param ShortDesc A brief description of this item. + * + * \param ExpectedMD5 The expected md5sum of the completely + * reconstructed package index file; the index file will be tested + * against this value when it is entirely reconstructed. + * + * \param diffs The remaining diffs from the index of diffs. They + * should be ordered so that each diff appears before any diff + * that depends on it. + */ + pkgAcqIndexDiffs(pkgAcquire *Owner,string URI,string URIDesc, + string ShortDesc, string ExpectedMD5, + vector<DiffInfo> diffs=vector<DiffInfo>()); +}; + +/** \brief An acquire item that is responsible for fetching an index + * file (e.g., Packages or Sources). + * + * \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans + * + * \todo Why does pkgAcqIndex have protected members? + */ +class pkgAcqIndex : public pkgAcquire::Item +{ + protected: + + /** \brief If \b true, the index file has been decompressed. */ bool Decompression; + + /** \brief If \b true, the partially downloaded file will be + * removed when the download completes. + */ bool Erase; + + /** \brief The download request that is currently being + * processed. + */ pkgAcquire::ItemDesc Desc; + + /** \brief The object that is actually being fetched (minus any + * compression-related extensions). + */ string RealURI; + + /** \brief The expected md5sum of the decompressed index file. */ string ExpectedMD5; + + /** \brief The compression-related file extension that is being + * added to the downloaded file (e.g., ".gz" or ".bz2"). + */ string CompressionExtension; public: @@ -103,36 +522,120 @@ class pkgAcqIndex : public pkgAcquire::Item virtual string Custom600Headers(); virtual string DescURI() {return RealURI + CompressionExtension;}; + /** \brief Create a pkgAcqIndex. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param URI The URI of the index file that is to be downloaded. + * + * \param URIDesc A "URI-style" description of this index file. + * + * \param ShortDesc A brief description of this index file. + * + * \param ExpectedMD5 The expected md5sum of this index file. + * + * \param compressExt The compression-related extension with which + * this index file should be downloaded, or "" to autodetect + * (".bz2" is used if bzip2 is installed, ".gz" otherwise). + */ pkgAcqIndex(pkgAcquire *Owner,string URI,string URIDesc, - string ShortDesct, string ExpectedMD5, string compressExt=""); + string ShortDesc, string ExpectedMD5, string compressExt=""); }; -// Item class for translated package index files +/** \brief An acquire item that is responsible for fetching a + * translated index file. + * + * The only difference from pkgAcqIndex is that transient failures + * are suppressed: no error occurs if the translated index file is + * missing. + */ class pkgAcqIndexTrans : public pkgAcqIndex { public: virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + + /** \brief Create a pkgAcqIndexTrans. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param URI The URI of the index file that is to be downloaded. + * + * \param URIDesc A "URI-style" description of this index file. + * + * \param ShortDesc A brief description of this index file. + * + * \param ExpectedMD5 The expected md5sum of this index file. + * + * \param compressExt The compression-related extension with which + * this index file should be downloaded, or "" to autodetect + * (".bz2" is used if bzip2 is installed, ".gz" otherwise). + */ pkgAcqIndexTrans(pkgAcquire *Owner,string URI,string URIDesc, - string ShortDesct); + string ShortDesc); }; +/** \brief Information about an index file. */ struct IndexTarget { + /** \brief A URI from which the index file can be downloaded. */ string URI; + + /** \brief A description of the index file. */ string Description; + + /** \brief A shorter description of the index file. */ string ShortDesc; + + /** \brief The key by which this index file should be + * looked up within the meta signature file. + */ string MetaKey; }; -// Item class for index signatures +/** \brief An acquire item that downloads the detached signature + * of a meta-index (Release) file, then queues up the release + * file itself. + * + * \todo Why protected members? + * + * \sa pkgAcqMetaIndex + */ class pkgAcqMetaSig : public pkgAcquire::Item { protected: - + /** \brief The fetch request that is currently being processed. */ pkgAcquire::ItemDesc Desc; - string RealURI,MetaIndexURI,MetaIndexURIDesc,MetaIndexShortDesc; + + /** \brief The URI of the signature file. Unlike Desc.URI, this is + * never modified; it is used to determine the file that is being + * downloaded. + */ + string RealURI; + + /** \brief The URI of the meta-index file to be fetched after the signature. */ + string MetaIndexURI; + + /** \brief A "URI-style" description of the meta-index file to be + * fetched after the signature. + */ + string MetaIndexURIDesc; + + /** \brief A brief description of the meta-index file to be fetched + * after the signature. + */ + string MetaIndexShortDesc; + + /** \brief A package-system-specific parser for the meta-index file. */ indexRecords* MetaIndexParser; + + /** \brief The index files which should be looked up in the meta-index + * and then downloaded. + * + * \todo Why a list of pointers instead of a list of structs? + */ const vector<struct IndexTarget*>* IndexTargets; public: @@ -144,27 +647,90 @@ class pkgAcqMetaSig : public pkgAcquire::Item virtual string Custom600Headers(); virtual string DescURI() {return RealURI; }; + /** \brief Create a new pkgAcqMetaSig. */ pkgAcqMetaSig(pkgAcquire *Owner,string URI,string URIDesc, string ShortDesc, string MetaIndexURI, string MetaIndexURIDesc, string MetaIndexShortDesc, const vector<struct IndexTarget*>* IndexTargets, indexRecords* MetaIndexParser); }; -// Item class for index signatures +/** \brief An item that is responsible for downloading the meta-index + * file (i.e., Release) itself and verifying its signature. + * + * Once the download and verification are complete, the downloads of + * the individual index files are queued up using pkgAcqDiffIndex. + * If the meta-index file had a valid signature, the expected md5sums + * of the index files will be the md5sums listed in the meta-index; + * otherwise, the expected md5sums will be "" (causing the + * authentication of the index files to be bypassed). + */ class pkgAcqMetaIndex : public pkgAcquire::Item { protected: - + /** \brief The fetch command that is currently being processed. */ pkgAcquire::ItemDesc Desc; - string RealURI; // FIXME: is this redundant w/ Desc.URI? + + /** \brief The URI that is actually being downloaded; never + * modified by pkgAcqMetaIndex. + */ + string RealURI; + + /** \brief The file in which the signature for this index was stored. + * + * If empty, the signature and the md5sums of the individual + * indices will not be checked. + */ string SigFile; + + /** \brief The index files to download. */ const vector<struct IndexTarget*>* IndexTargets; + + /** \brief The parser for the meta-index file. */ indexRecords* MetaIndexParser; + + /** \brief If \b true, the index's signature is currently being verified. + */ bool AuthPass; + // required to deal gracefully with problems caused by incorrect ims hits + bool IMSHit; - bool VerifyVendor(); + /** \brief Check that the release file is a release file for the + * correct distribution. + * + * \return \b true if no fatal errors were encountered. + */ + bool VerifyVendor(string Message); + + /** \brief Called when a file is finished being retrieved. + * + * If the file was not downloaded to DestFile, a copy process is + * set up to copy it to DestFile; otherwise, Complete is set to \b + * true and the file is moved to its final location. + * + * \param Message The message block received from the fetch + * subprocess. + */ void RetrievalDone(string Message); + + /** \brief Called when authentication succeeded. + * + * Sanity-checks the authenticated file, queues up the individual + * index files for download, and saves the signature in the lists + * directory next to the authenticated list file. + * + * \param Message The message block received from the fetch + * subprocess. + */ void AuthDone(string Message); + + /** \brief Starts downloading the individual index files. + * + * \param verify If \b true, only indices whose expected md5sum + * can be determined from the meta-index will be downloaded, and + * the md5sums of indices will be checked (reporting + * #StatAuthError if there is a mismatch). If verify is \b false, + * no md5sum checking will be performed. + */ void QueueIndexes(bool verify); public: @@ -176,6 +742,7 @@ class pkgAcqMetaIndex : public pkgAcquire::Item virtual string Custom600Headers(); virtual string DescURI() {return RealURI; }; + /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, string URI,string URIDesc, string ShortDesc, string SigFile, @@ -183,28 +750,58 @@ class pkgAcqMetaIndex : public pkgAcquire::Item indexRecords* MetaIndexParser); }; -// Item class for archive files +/** \brief An item that is responsible for fetching a package file. + * + * If the package file already exists in the cache, nothing will be + * done. + */ class pkgAcqArchive : public pkgAcquire::Item { protected: - - // State information for the retry mechanism + /** \brief The package version being fetched. */ pkgCache::VerIterator Version; + + /** \brief The fetch command that is currently being processed. */ pkgAcquire::ItemDesc Desc; + + /** \brief The list of sources from which to pick archives to + * download this package from. + */ pkgSourceList *Sources; + + /** \brief A package records object, used to look up the file + * corresponding to each version of the package. + */ pkgRecords *Recs; + + /** \brief The md5sum of this package. */ string MD5; + + /** \brief A location in which the actual filename of the package + * should be stored. + */ string &StoreFilename; + + /** \brief The next file for this version to try to download. */ pkgCache::VerFileIterator Vf; + + /** \brief How many (more) times to try to find a new source from + * which to download this package version if it fails. + * + * Set from Acquire::Retries. + */ unsigned int Retries; + + /** \brief \b true if this version file is being downloaded from a + * trusted source. + */ bool Trusted; - // Queue the next available file for download. + /** \brief Queue up the next available file for this version. */ bool QueueNext(); public: - // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); virtual void Done(string Message,unsigned long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); @@ -212,18 +809,49 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string DescURI() {return Desc.URI;}; virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); + virtual bool IsTrusted(); + /** \brief Create a new pkgAcqArchive. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param Sources The sources from which to download version + * files. + * + * \param Recs A package records object, used to look up the file + * corresponding to each version of the package. + * + * \param Version The package version to download. + * + * \param StoreFilename A location in which the actual filename of + * the package should be stored. It will be set to a guessed + * basename in the constructor, and filled in with a fully + * qualified filename once the download finishes. + */ pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, string &StoreFilename); }; -// Fetch a generic file to the current directory +/** \brief Retrieve an arbitrary file to the current directory. + * + * The file is retrieved even if it is accessed via a URL type that + * normally is a NOP, such as "file". If the download fails, the + * partial file is renamed to get a ".FAILED" extension. + */ class pkgAcqFile : public pkgAcquire::Item { + /** \brief The currently active download process. */ pkgAcquire::ItemDesc Desc; + + /** \brief The md5sum of the file to download, if it is known. */ string Md5Hash; + + /** \brief How many times to retry the download, set from + * Acquire::Retries. + */ unsigned int Retries; public: @@ -234,9 +862,41 @@ class pkgAcqFile : public pkgAcquire::Item pkgAcquire::MethodConfig *Cnf); virtual string MD5Sum() {return Md5Hash;}; virtual string DescURI() {return Desc.URI;}; - - pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,unsigned long Size, - string Desc,string ShortDesc); + + /** \brief Create a new pkgAcqFile object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * + * \param URI The URI to download. + * + * \param MD5 The md5sum of the file to download, if it is known; + * otherwise "". + * + * \param Size The size of the file to download, if it is known; + * otherwise 0. + * + * \param Desc A description of the file being downloaded. + * + * \param ShortDesc A brief description of the file being + * downloaded. + * + * \param DestDir The directory the file should be downloaded into. + * + * \param DestFilename The filename+path the file is downloaded to. + * + * + * If DestFilename is empty, download to DestDir/<basename> if + * DestDir is non-empty, $CWD/<basename> otherwise. If + * DestFilename is NOT empty, DestDir is ignored and DestFilename + * is the absolute name to which the file should be downloaded. + */ + + pkgAcqFile(pkgAcquire *Owner, string URI, string MD5, unsigned long Size, + string Desc, string ShortDesc, + const string &DestDir="", const string &DestFilename=""); }; +/** @} */ + #endif diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index f46209d12..4f08a43ae 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -10,6 +10,13 @@ ##################################################################### */ /*}}}*/ + +/** \addtogroup acquire + * @{ + * + * \file acquire-method.h + */ + #ifndef PKGLIB_ACQUIRE_METHOD_H #define PKGLIB_ACQUIRE_METHOD_H @@ -86,4 +93,6 @@ class pkgAcqMethod virtual ~pkgAcqMethod() {}; }; +/** @} */ + #endif diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h index 6e1952202..1f6bcc05f 100644 --- a/apt-pkg/acquire-worker.h +++ b/apt-pkg/acquire-worker.h @@ -9,6 +9,13 @@ ##################################################################### */ /*}}}*/ + +/** \addtogroup acquire + * @{ + * + * \file acquire-worker.h + */ + #ifndef PKGLIB_ACQUIRE_WORKER_H #define PKGLIB_ACQUIRE_WORKER_H @@ -18,7 +25,25 @@ #pragma interface "apt-pkg/acquire-worker.h" #endif -// Interfacing to the method process +/** \brief A fetch subprocess. + * + * A worker process is responsible for one stage of the fetch. This + * class encapsulates the communications protocol between the master + * process and the worker, from the master end. + * + * Each worker is intrinsically placed on two linked lists. The + * Queue list (maintained in the #NextQueue variable) is maintained + * by the pkgAcquire::Queue class; it represents the set of workers + * assigned to a particular queue. The Acquire list (maintained in + * the #NextAcquire variable) is maintained by the pkgAcquire class; + * it represents the set of active workers for a particular + * pkgAcquire object. + * + * \todo Like everything else in the Acquire system, this has way too + * many protected items. + * + * \sa pkgAcqMethod, pkgAcquire::Item, pkgAcquire + */ class pkgAcquire::Worker { friend class pkgAcquire; @@ -26,64 +51,274 @@ class pkgAcquire::Worker protected: friend class Queue; - /* Linked list starting at a Queue and a linked list starting - at Acquire */ + /** \brief The next link on the Queue list. + * + * \todo This is always NULL; is it just for future use? + */ Worker *NextQueue; + + /** \brief The next link on the Acquire list. */ Worker *NextAcquire; - // The access association + /** \brief The Queue with which this worker is associated. */ Queue *OwnerQ; + + /** \brief The download progress indicator to which progress + * messages should be sent. + */ pkgAcquireStatus *Log; + + /** \brief The configuration of this method. On startup, the + * target of this pointer is filled in with basic data about the + * method, as reported by the worker. + */ MethodConfig *Config; + + /** \brief The access method to be used by this worker. + * + * \todo Doesn't this duplicate Config->Access? + */ string Access; - // This is the subprocess IPC setup + /** \brief The PID of the subprocess. */ pid_t Process; + + /** \brief A file descriptor connected to the standard output of + * the subprocess. + * + * Used to read messages and data from the subprocess. + */ int InFd; + + /** \brief A file descriptor connected to the standard input of the + * subprocess. + * + * Used to send commands and configuration data to the subprocess. + */ int OutFd; + + /** \brief Set to \b true if the worker is in a state in which it + * might generate data or command responses. + * + * \todo Is this right? It's a guess. + */ bool InReady; + + /** \brief Set to \b true if the worker is in a state in which it + * is legal to send commands to it. + * + * \todo Is this right? + */ bool OutReady; - // Various internal things + /** If \b true, debugging output will be sent to std::clog. */ bool Debug; + + /** \brief The raw text values of messages received from the + * worker, in sequence. + */ vector<string> MessageQueue; + + /** \brief Buffers pending writes to the subprocess. + * + * \todo Wouldn't a std::dequeue be more appropriate? + */ string OutQueue; - // Private constructor helper + /** \brief Common code for the constructor. + * + * Initializes NextQueue and NextAcquire to NULL; Process, InFd, + * and OutFd to -1, OutReady and InReady to \b false, and Debug + * from _config. + */ void Construct(); - // Message handling things + /** \brief Retrieve any available messages from the subprocess. + * + * The messages are retrieved as in ::ReadMessages(), and + * MessageFailure() is invoked if an error occurs; in particular, + * if the pipe to the subprocess dies unexpectedly while a message + * is being read. + * + * \return \b true if the messages were successfully read, \b + * false otherwise. + */ bool ReadMessages(); + + /** \brief Parse and dispatch pending messages. + * + * This dispatches the message in a manner appropriate for its + * type. + * + * \todo Several message types lack separate handlers. + * + * \sa Capabilities(), SendConfiguration(), MediaChange() + */ bool RunMessages(); + + /** \brief Read and dispatch any pending messages from the + * subprocess. + * + * \return \b false if the subprocess died unexpectedly while a + * message was being transmitted. + */ bool InFdReady(); + + /** \brief Send any pending commands to the subprocess. + * + * This method will fail if there is no pending output. + * + * \return \b true if all commands were succeeded, \b false if an + * error occurred (in which case MethodFailure() will be invoked). + */ bool OutFdReady(); - // The message handlers + /** \brief Handle a 100 Capabilities response from the subprocess. + * + * \param Message the raw text of the message from the subprocess. + * + * The message will be parsed and its contents used to fill + * #Config. If #Config is NULL, this routine is a NOP. + * + * \return \b true. + */ bool Capabilities(string Message); + + /** \brief Send a 601 Configuration message (containing the APT + * configuration) to the subprocess. + * + * The APT configuration will be send to the subprocess in a + * message of the following form: + * + * <pre> + * 601 Configuration + * Config-Item: Fully-Qualified-Item=Val + * Config-Item: Fully-Qualified-Item=Val + * ... + * </pre> + * + * \return \b true if the command was successfully sent, \b false + * otherwise. + */ bool SendConfiguration(); + + /** \brief Handle a 403 Media Change message. + * + * \param Message the raw text of the message; the Media field + * indicates what type of media should be changed, and the Drive + * field indicates where the media is located. + * + * Invokes pkgAcquireStatus::MediaChange(Media, Drive) to ask the + * user to swap disks; informs the subprocess of the result (via + * 603 Media Changed, with the Failed field set to \b true if the + * user cancelled the media change). + */ bool MediaChange(string Message); + /** \brief Invoked when the worked process dies unexpectedly. + * + * Waits for the subprocess to terminate and generates an error if + * it terminated abnormally, then closes and blanks out all file + * descriptors. Discards all pending messages from the + * subprocess. + * + * \return \b false. + */ bool MethodFailure(); + + /** \brief Invoked when a fetch job is completed, either + * successfully or unsuccessfully. + * + * Resets the status information for the worker process. + */ void ItemDone(); public: - // The curent method state + /** \brief The queue entry that is currently being downloaded. */ pkgAcquire::Queue::QItem *CurrentItem; + + /** \brief The most recent status string received from the + * subprocess. + */ string Status; + + /** \brief How many bytes of the file have been downloaded. Zero + * if the current progress of the file cannot be determined. + */ unsigned long CurrentSize; + + /** \brief The total number of bytes to be downloaded. Zero if the + * total size of the final is unknown. + */ unsigned long TotalSize; + + /** \brief How much of the file was already downloaded prior to + * starting this worker. + */ unsigned long ResumePoint; - // Load the method and do the startup + /** \brief Tell the subprocess to download the given item. + * + * \param Item the item to queue up. + * \return \b true if the item was successfully enqueued. + * + * Queues up a 600 URI Acquire message for the given item to be + * sent at the next possible moment. Does \e not flush the output + * queue. + */ bool QueueItem(pkgAcquire::Queue::QItem *Item); + + /** \brief Start up the worker and fill in #Config. + * + * Reads the first message from the worker, which is assumed to be + * a 100 Capabilities message. + * + * \return \b true if all operations completed successfully. + */ bool Start(); + + /** \brief Update the worker statistics (CurrentSize, TotalSize, + * etc). + */ void Pulse(); + + /** \return The fetch method configuration. */ inline const MethodConfig *GetConf() const {return Config;}; - + + /** \brief Create a new Worker to download files. + * + * \param OwnerQ The queue into which this worker should be + * placed. + * + * \param Config A location in which to store information about + * the fetch method. + * + * \param Log The download progress indicator that should be used + * to report the progress of this worker. + */ Worker(Queue *OwnerQ,MethodConfig *Config,pkgAcquireStatus *Log); + + /** \brief Create a new Worker that should just retrieve + * information about the fetch method. + * + * Nothing in particular forces you to refrain from actually + * downloading stuff, but the various status callbacks won't be + * invoked. + * + * \param Config A location in which to store information about + * the fetch method. + */ Worker(MethodConfig *Config); + + /** \brief Clean up this worker. + * + * Closes the file descriptors; if MethodConfig::NeedsCleanup is + * \b false, also rudely interrupts the worker with a SIGINT. + */ ~Worker(); }; +/** @} */ + #endif diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 62209e65b..fff1b2b6a 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -266,7 +266,11 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access) Worker Work(Conf); if (Work.Start() == false) return 0; - + + /* if a method uses DownloadLimit, we switch to SingleInstance mode */ + if(_config->FindI("Acquire::"+Access+"::DlLimit",0) > 0) + Conf->SingleInstance = true; + return Conf; } /*}}}*/ @@ -814,7 +818,13 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) unsigned long ETA = (unsigned long)((TotalBytes - CurrentBytes) / CurrentCPS); - snprintf(msg,sizeof(msg), _("Downloading file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str()); + // only show the ETA if it makes sense + if (ETA > 0 && ETA < 172800 /* two days */ ) + snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str()); + else + snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems); + + // build the status str status << "dlstatus:" << i diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 27bb3d363..64dafdc9d 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -29,6 +29,40 @@ ##################################################################### */ /*}}}*/ + +/** \defgroup acquire Acquire system + * + * \brief The Acquire system is responsible for retrieving files from + * local or remote URIs and postprocessing them (for instance, + * verifying their authenticity). The core class in this system is + * pkgAcquire, which is responsible for managing the download queues + * during the download. There is at least one download queue for + * each supported protocol; protocols such as http may provide one + * queue per host. + * + * Each file to download is represented by a subclass of + * pkgAcquire::Item. The files add themselves to the download + * queue(s) by providing their URI information to + * pkgAcquire::Item::QueueURI, which calls pkgAcquire::Enqueue. + * + * Once the system is set up, the Run method will spawn subprocesses + * to handle the enqueued URIs; the scheduler will then take items + * from the queues and feed them into the handlers until the queues + * are empty. + * + * \todo Acquire supports inserting an object into several queues at + * once, but it is not clear what its behavior in this case is, and + * no subclass of pkgAcquire::Item seems to actually use this + * capability. + */ + +/** \addtogroup acquire + * + * @{ + * + * \file acquire.h + */ + #ifndef PKGLIB_ACQUIRE_H #define PKGLIB_ACQUIRE_H @@ -46,6 +80,15 @@ using std::string; #include <unistd.h> class pkgAcquireStatus; + +/** \brief The core download scheduler. + * + * This class represents an ongoing download. It manages the lists + * of active and pending downloads and handles setting up and tearing + * down download-related structures. + * + * \todo Why all the protected data items and methods? + */ class pkgAcquire { public: @@ -60,97 +103,299 @@ class pkgAcquire typedef vector<Item *>::iterator ItemIterator; typedef vector<Item *>::const_iterator ItemCIterator; - + protected: - // List of items to fetch + /** \brief A list of items to download. + * + * This is built monotonically as items are created and only + * emptied when the download shuts down. + */ vector<Item *> Items; - // List of active queues and fetched method configuration parameters + /** \brief The head of the list of active queues. + * + * \todo why a hand-managed list of queues instead of std::list or + * std::set? + */ Queue *Queues; + + /** \brief The head of the list of active workers. + * + * \todo why a hand-managed list of workers instead of std::list + * or std::set? + */ Worker *Workers; + + /** \brief The head of the list of acquire method configurations. + * + * Each protocol (http, ftp, gzip, etc) via which files can be + * fetched can have a representation in this list. The + * configuration data is filled in by parsing the 100 Capabilities + * string output by a method on startup (see + * pkgAcqMethod::pkgAcqMethod and pkgAcquire::GetConfig). + * + * \todo why a hand-managed config dictionary instead of std::map? + */ MethodConfig *Configs; + + /** \brief The progress indicator for this download. */ pkgAcquireStatus *Log; + + /** \brief The total size of the files which are to be fetched. + * + * This is not necessarily the total number of bytes to download + * when, e.g., download resumption and list updates via patches + * are taken into account. + */ unsigned long ToFetch; - // Configurable parameters for the schedular - enum {QueueHost,QueueAccess} QueueMode; + // Configurable parameters for the scheduler + + /** \brief Represents the queuing strategy for remote URIs. */ + enum QueueStrategy { + /** \brief Generate one queue for each protocol/host combination; downloads from + * multiple hosts can proceed in parallel. + */ + QueueHost, + /** \brief Generate a single queue for each protocol; serialize + * downloads from multiple hosts. + */ + QueueAccess} QueueMode; + + /** \brief If \b true, debugging information will be dumped to std::clog. */ bool Debug; + /** \brief If \b true, a download is currently in progress. */ bool Running; - + + /** \brief Add the given item to the list of items. */ void Add(Item *Item); + + /** \brief Remove the given item from the list of items. */ void Remove(Item *Item); + + /** \brief Add the given worker to the list of workers. */ void Add(Worker *Work); + + /** \brief Remove the given worker from the list of workers. */ void Remove(Worker *Work); + /** \brief Insert the given fetch request into the appropriate queue. + * + * \param Item The URI to download and the item to download it + * for. Copied by value into the queue; no reference to Item is + * retained. + */ void Enqueue(ItemDesc &Item); + + /** \brief Remove all fetch requests for this item from all queues. */ void Dequeue(Item *Item); + + /** \brief Determine the fetch method and queue of a URI. + * + * \param URI The URI to fetch. + * + * \param[out] Config A location in which to place the method via + * which the URI is to be fetched. + * + * \return the string-name of the queue in which a fetch request + * for the given URI should be placed. + */ string QueueName(string URI,MethodConfig const *&Config); - // FDSET managers for derived classes + /** \brief Build up the set of file descriptors upon which select() should + * block. + * + * The default implementation inserts the file descriptors + * corresponding to active downloads. + * + * \param[out] Fd The largest file descriptor in the generated sets. + * + * \param[out] RSet The set of file descriptors that should be + * watched for input. + * + * \param[out] WSet The set of file descriptors that should be + * watched for output. + */ virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet); + + /** Handle input from and output to file descriptors which select() + * has determined are ready. The default implementation + * dispatches to all active downloads. + * + * \param RSet The set of file descriptors that are ready for + * input. + * + * \param WSet The set of file descriptors that are ready for + * output. + */ virtual void RunFds(fd_set *RSet,fd_set *WSet); - // A queue calls this when it dequeues an item + /** \brief Check for idle queues with ready-to-fetch items. + * + * Called by pkgAcquire::Queue::Done each time an item is dequeued + * but remains on some queues; i.e., another queue should start + * fetching it. + */ void Bump(); public: + /** \brief Retrieve information about a fetch method by name. + * + * \param Access The name of the method to look up. + * + * \return the method whose name is Access, or \b NULL if no such method exists. + */ MethodConfig *GetConfig(string Access); - enum RunResult {Continue,Failed,Cancelled}; + /** \brief Provides information on how a download terminated. */ + enum RunResult { + /** \brief All files were fetched successfully. */ + Continue, + + /** \brief Some files failed to download. */ + Failed, + + /** \brief The download was cancelled by the user (i.e., #Log's + * pkgAcquireStatus::Pulse() method returned \b false). + */ + Cancelled}; - RunResult Run(int PulseIntervall=500000); + /** \brief Download all the items that have been Add()ed to this + * download process. + * + * This method will block until the download completes, invoking + * methods on #Log to report on the progress of the download. + * + * \param PulseInterval The method pkgAcquireStatus::Pulse will be + * invoked on #Log at intervals of PulseInterval milliseconds. + * + * \return the result of the download. + */ + RunResult Run(int PulseInterval=500000); + + /** \brief Remove all items from this download process, terminate + * all download workers, and empty all queues. + */ void Shutdown(); - // Simple iteration mechanism + /** \brief Get the first #Worker object. + * + * \return the first active worker in this download process. + */ inline Worker *WorkersBegin() {return Workers;}; + + /** \brief Advance to the next #Worker object. + * + * \return the worker immediately following I, or \b NULL if none + * exists. + */ Worker *WorkerStep(Worker *I); + + /** \brief Get the head of the list of items. */ inline ItemIterator ItemsBegin() {return Items.begin();}; + + /** \brief Get the end iterator of the list of items. */ inline ItemIterator ItemsEnd() {return Items.end();}; // Iterate over queued Item URIs class UriIterator; + /** \brief Get the head of the list of enqueued item URIs. + * + * This iterator will step over every element of every active + * queue. + */ UriIterator UriBegin(); + /** \brief Get the end iterator of the list of enqueued item URIs. */ UriIterator UriEnd(); - // Cleans out the download dir + /** Deletes each entry in the given directory that is not being + * downloaded by this object. For instance, when downloading new + * list files, calling Clean() will delete the old ones. + * + * \param Dir The directory to be cleaned out. + * + * \return \b true if the directory exists and is readable. + */ bool Clean(string Dir); - // Returns the size of the total download set + /** \return the total size in bytes of all the items included in + * this download. + */ double TotalNeeded(); + + /** \return the size in bytes of all non-local items included in + * this download. + */ double FetchNeeded(); + + /** \return the amount of data to be fetched that is already + * present on the filesystem. + */ double PartialPresent(); + /** \brief Construct a new pkgAcquire. + * + * \param Log The progress indicator associated with this + * download, or \b NULL for none. This object is not owned by the + * download process and will not be deleted when the pkgAcquire + * object is destroyed. Naturally, it should live for at least as + * long as the pkgAcquire object does. + */ pkgAcquire(pkgAcquireStatus *Log = 0); + + /** \brief Destroy this pkgAcquire object. + * + * Destroys all queue, method, and item objects associated with + * this download. + */ virtual ~pkgAcquire(); }; -// Description of an Item+URI +/** \brief Represents a single download source from which an item + * should be downloaded. + * + * An item may have several assocated ItemDescs over its lifetime. + */ struct pkgAcquire::ItemDesc { + /** \brief The URI from which to download this item. */ string URI; + /** brief A description of this item. */ string Description; + /** brief A shorter description of this item. */ string ShortDesc; + /** brief The underlying item which is to be downloaded. */ Item *Owner; }; -// List of possible items queued for download. +/** \brief A single download queue in a pkgAcquire object. + * + * \todo Why so many protected values? + */ class pkgAcquire::Queue { friend class pkgAcquire; friend class pkgAcquire::UriIterator; friend class pkgAcquire::Worker; + + /** \brief The next queue in the pkgAcquire object's list of queues. */ Queue *Next; protected: - // Queued item + /** \brief A single item placed in this queue. */ struct QItem : pkgAcquire::ItemDesc { - QItem *Next; + /** \brief The next item in the queue. */ + QItem *Next; + /** \brief The worker associated with this item, if any. */ pkgAcquire::Worker *Worker; - + + /** \brief Assign the ItemDesc portion of this QItem from + * another ItemDesc + */ void operator =(pkgAcquire::ItemDesc const &I) { URI = I.URI; @@ -160,45 +405,141 @@ class pkgAcquire::Queue }; }; - // Name of the queue + /** \brief The name of this queue. */ string Name; - // Items queued into this queue + /** \brief The head of the list of items contained in this queue. + * + * \todo why a by-hand list instead of an STL structure? + */ QItem *Items; + + /** \brief The head of the list of workers associated with this queue. + * + * \todo This is plural because support exists in Queue for + * multiple workers. However, it does not appear that there is + * any way to actually associate more than one worker with a + * queue. + * + * \todo Why not just use a std::set? + */ pkgAcquire::Worker *Workers; + + /** \brief the download scheduler with which this queue is associated. */ pkgAcquire *Owner; + + /** \brief The number of entries in this queue that are currently + * being downloaded. + */ signed long PipeDepth; + + /** \brief The maximum number of entries that this queue will + * attempt to download at once. + */ unsigned long MaxPipeDepth; public: - // Put an item into this queue + /** \brief Insert the given fetch request into this queue. */ void Enqueue(ItemDesc &Item); + + /** \brief Remove all fetch requests for the given item from this queue. + * + * \return \b true if at least one request was removed from the queue. + */ bool Dequeue(Item *Owner); - // Find a Queued item + /** \brief Locate an item in this queue. + * + * \param URI A URI to match against. + * \param Owner A pkgAcquire::Worker to match against. + * + * \return the first item in the queue whose URI is #URI and that + * is being downloaded by #Owner. + */ QItem *FindItem(string URI,pkgAcquire::Worker *Owner); + + /** Presumably this should start downloading an item? + * + * \todo Unimplemented. Implement it or remove? + */ bool ItemStart(QItem *Itm,unsigned long Size); + + /** \brief Remove the given item from this queue and set its state + * to pkgAcquire::Item::StatDone. + * + * If this is the only queue containing the item, the item is also + * removed from the main queue by calling pkgAcquire::Dequeue. + * + * \param Itm The item to remove. + * + * \return \b true if no errors are encountered. + */ bool ItemDone(QItem *Itm); + /** \brief Start the worker process associated with this queue. + * + * If a worker process is already associated with this queue, + * this is equivalent to calling Cycle(). + * + * \return \b true if the startup was successful. + */ bool Startup(); + + /** \brief Shut down the worker process associated with this queue. + * + * \param Final If \b true, then the process is stopped unconditionally. + * Otherwise, it is only stopped if it does not need cleanup + * as indicated by the pkgAcqMethod::NeedsCleanup member of + * its configuration. + * + * \return \b true. + */ bool Shutdown(bool Final); + + /** \brief Send idle items to the worker process. + * + * Fills up the pipeline by inserting idle items into the worker's queue. + */ bool Cycle(); + + /** \brief Check for items that could be enqueued. + * + * Call this after an item placed in multiple queues has gone from + * the pkgAcquire::Item::StatFetching state to the + * pkgAcquire::Item::StatIdle state, to possibly refill an empty queue. + * This is an alias for Cycle(). + * + * \todo Why both this and Cycle()? Are they expected to be + * different someday? + */ void Bump(); + /** \brief Create a new Queue. + * + * \param Name The name of the new queue. + * \param Owner The download process that owns the new queue. + */ Queue(string Name,pkgAcquire *Owner); + + /** Shut down all the worker processes associated with this queue + * and empty the queue. + */ ~Queue(); }; +/** \brief Iterates over all the URIs being fetched by a pkgAcquire object. */ class pkgAcquire::UriIterator { + /** The next queue to iterate over. */ pkgAcquire::Queue *CurQ; + /** The item that we currently point at. */ pkgAcquire::Queue::QItem *CurItem; public: - // Advance to the next item inline void operator ++() {operator ++();}; + void operator ++(int) { CurItem = CurItem->Next; @@ -209,11 +550,14 @@ class pkgAcquire::UriIterator } }; - // Accessors inline pkgAcquire::ItemDesc const *operator ->() const {return CurItem;}; inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;}; inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;}; + /** \brief Create a new UriIterator. + * + * \param Q The queue over which this UriIterator should iterate. + */ UriIterator(pkgAcquire::Queue *Q) : CurQ(Q), CurItem(0) { while (CurItem == 0 && CurQ != 0) @@ -224,61 +568,200 @@ class pkgAcquire::UriIterator } }; -// Configuration information from each method +/** \brief Information about the properties of a single acquire method. */ struct pkgAcquire::MethodConfig { + /** \brief The next link on the acquire method list. + * + * \todo Why not an STL container? + */ MethodConfig *Next; + /** \brief The name of this acquire method (e.g., http). */ string Access; + /** \brief The implementation version of this acquire method. */ string Version; + + /** \brief If \b true, only one download queue should be created for this + * method. + */ bool SingleInstance; + + /** \brief If \b true, this method supports pipelined downloading. */ bool Pipeline; + + /** \brief If \b true, the worker process should send the entire + * APT configuration tree to the fetch subprocess when it starts + * up. + */ bool SendConfig; + + /** \brief If \b true, this fetch method does not require network access; + * all files are to be acquired from the local disk. + */ bool LocalOnly; + + /** \brief If \b true, the subprocess has to carry out some cleanup + * actions before shutting down. + * + * For instance, the cdrom method needs to unmount the CD after it + * finishes. + */ bool NeedsCleanup; + + /** \brief If \b true, this fetch method acquires files from removable media. */ bool Removable; + /** \brief Set up the default method parameters. + * + * All fields are initialized to NULL, "", or \b false as + * appropriate. + */ MethodConfig(); }; +/** \brief A monitor object for downloads controlled by the pkgAcquire class. + * + * \todo Why protected members? + * + * \todo Should the double members be uint64_t? + */ class pkgAcquireStatus { protected: + /** \brief The last time at which this monitor object was updated. */ struct timeval Time; + + /** \brief The time at which the download started. */ struct timeval StartTime; + + /** \brief The number of bytes fetched as of the previous call to + * pkgAcquireStatus::Pulse, including local items. + */ double LastBytes; + + /** \brief The current rate of download as of the most recent call + * to pkgAcquireStatus::Pulse, in bytes per second. + */ double CurrentCPS; + + /** \brief The number of bytes fetched as of the most recent call + * to pkgAcquireStatus::Pulse, including local items. + */ double CurrentBytes; + + /** \brief The total number of bytes that need to be fetched. + * + * \warning This member is inaccurate, as new items might be + * enqueued while the download is in progress! + */ double TotalBytes; + + /** \brief The total number of bytes accounted for by items that + * were successfully fetched. + */ double FetchedBytes; + + /** \brief The amount of time that has elapsed since the download + * started. + */ unsigned long ElapsedTime; + + /** \brief The total number of items that need to be fetched. + * + * \warning This member is inaccurate, as new items might be + * enqueued while the download is in progress! + */ unsigned long TotalItems; + + /** \brief The number of items that have been successfully downloaded. */ unsigned long CurrentItems; public: + /** \brief If \b true, the download scheduler should call Pulse() + * at the next available opportunity. + */ bool Update; + + /** \brief If \b true, extra Pulse() invocations will be performed. + * + * With this option set, Pulse() will be called every time that a + * download item starts downloading, finishes downloading, or + * terminates with an error. + */ bool MorePulses; - // Called by items when they have finished a real download + /** \brief Invoked when a local or remote file has been completely fetched. + * + * \param Size The size of the file fetched. + * + * \param ResumePoint How much of the file was already fetched. + */ virtual void Fetched(unsigned long Size,unsigned long ResumePoint); - // Called to change media + /** \brief Invoked when the user should be prompted to change the + * inserted removable media. + * + * This method should not return until the user has confirmed to + * the user interface that the media change is complete. + * + * \param Media The name of the media type that should be changed. + * + * \param Drive The identifying name of the drive whose media + * should be changed. + * + * \return \b true if the user confirms the media change, \b + * false if it is cancelled. + * + * \todo This is a horrible blocking monster; it should be CPSed + * with prejudice. + */ virtual bool MediaChange(string Media,string Drive) = 0; - // Each of these is called by the workers when an event occures + /** \brief Invoked when an item is confirmed to be up-to-date. + + * For instance, when an HTTP download is informed that the file on + * the server was not modified. + */ virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when some of an item's data is fetched. */ virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when an item is successfully and completely fetched. */ virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when the process of fetching an item encounters + * a fatal error. + */ virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {}; - virtual bool Pulse(pkgAcquire *Owner); // returns false on user cancel + + /** \brief Periodically invoked while the Acquire process is underway. + * + * Subclasses should first call pkgAcquireStatus::Pulse(), then + * update their status output. The download process is blocked + * while Pulse() is being called. + * + * \return \b false if the user asked to cancel the whole Acquire process. + * + * \see pkgAcquire::Run + */ + virtual bool Pulse(pkgAcquire *Owner); + + /** \brief Invoked when the Acquire process starts running. */ virtual void Start(); + + /** \brief Invoked when the Acquire process stops running. */ virtual void Stop(); + /** \brief Initialize all counters to 0 and the time to the current time. */ pkgAcquireStatus(); virtual ~pkgAcquireStatus() {}; }; +/** @} */ + #endif diff --git a/apt-pkg/algorithms.cc b/apt-pkg/algorithms.cc index 479927d65..ac9d3be0b 100644 --- a/apt-pkg/algorithms.cc +++ b/apt-pkg/algorithms.cc @@ -20,10 +20,12 @@ #include <apt-pkg/algorithms.h> #include <apt-pkg/error.h> #include <apt-pkg/configuration.h> +#include <apt-pkg/version.h> #include <apt-pkg/sptr.h> + #include <apti18n.h> - +#include <sys/types.h> #include <iostream> /*}}}*/ using namespace std; @@ -220,6 +222,8 @@ void pkgSimulate::ShortBreaks() the necessary calculations to deal with the problems. */ bool pkgApplyStatus(pkgDepCache &Cache) { + pkgDepCache::ActionGroup group(Cache); + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; I++) { if (I->VersionList == 0) @@ -230,13 +234,13 @@ bool pkgApplyStatus(pkgDepCache &Cache) I->InstState == pkgCache::State::HoldReInstReq) { if (I->CurrentVer != 0 && I.CurrentVer().Downloadable() == true) - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); else { // Is this right? Will dpkg choke on an upgrade? if (Cache[I].CandidateVer != 0 && Cache[I].CandidateVerIter(Cache).Downloadable() == true) - Cache.MarkInstall(I); + Cache.MarkInstall(I, false, 0, false); else return _error->Error(_("The package %s needs to be reinstalled, " "but I can't find an archive for it."),I.Name()); @@ -253,12 +257,12 @@ bool pkgApplyStatus(pkgDepCache &Cache) case pkgCache::State::HalfConfigured: if ((I->CurrentVer != 0 && I.CurrentVer().Downloadable() == true) || I.State() != pkgCache::PkgIterator::NeedsUnpack) - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); else { if (Cache[I].CandidateVer != 0 && Cache[I].CandidateVerIter(Cache).Downloadable() == true) - Cache.MarkInstall(I); + Cache.MarkInstall(I, true, 0, false); else Cache.MarkDelete(I); } @@ -284,10 +288,12 @@ bool pkgApplyStatus(pkgDepCache &Cache) on the result. */ bool pkgFixBroken(pkgDepCache &Cache) { + pkgDepCache::ActionGroup group(Cache); + // Auto upgrade all broken packages for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; I++) if (Cache[I].NowBroken() == true) - Cache.MarkInstall(I,true); + Cache.MarkInstall(I, true, 0, false); /* Fix packages that are in a NeedArchive state but don't have a downloadable install version */ @@ -300,7 +306,7 @@ bool pkgFixBroken(pkgDepCache &Cache) if (Cache[I].InstVerIter(Cache).Downloadable() == false) continue; - Cache.MarkInstall(I,true); + Cache.MarkInstall(I, true, 0, false); } pkgProblemResolver Fix(&Cache); @@ -317,23 +323,25 @@ bool pkgFixBroken(pkgDepCache &Cache) */ bool pkgDistUpgrade(pkgDepCache &Cache) { + pkgDepCache::ActionGroup group(Cache); + /* Auto upgrade all installed packages, this provides the basis for the installation */ for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; I++) if (I->CurrentVer != 0) - Cache.MarkInstall(I,true); + Cache.MarkInstall(I, true, 0, false); /* Now, auto upgrade all essential packages - this ensures that the essential packages are present and working */ for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; I++) if ((I->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) - Cache.MarkInstall(I,true); + Cache.MarkInstall(I, true, 0, false); /* We do it again over all previously installed packages to force conflict resolution on them all. */ for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; I++) if (I->CurrentVer != 0) - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); pkgProblemResolver Fix(&Cache); @@ -345,7 +353,7 @@ bool pkgDistUpgrade(pkgDepCache &Cache) if (I->SelectedState == pkgCache::State::Hold) { Fix.Protect(I); - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); } } } @@ -360,6 +368,8 @@ bool pkgDistUpgrade(pkgDepCache &Cache) to install packages not marked for install */ bool pkgAllUpgrade(pkgDepCache &Cache) { + pkgDepCache::ActionGroup group(Cache); + pkgProblemResolver Fix(&Cache); if (Cache.BrokenCount() != 0) @@ -376,7 +386,7 @@ bool pkgAllUpgrade(pkgDepCache &Cache) continue; if (I->CurrentVer != 0 && Cache[I].InstallVer != 0) - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); } return Fix.ResolveByKeep(); @@ -389,6 +399,8 @@ bool pkgAllUpgrade(pkgDepCache &Cache) the package is restored. */ bool pkgMinimizeUpgrade(pkgDepCache &Cache) { + pkgDepCache::ActionGroup group(Cache); + if (Cache.BrokenCount() != 0) return false; @@ -405,9 +417,9 @@ bool pkgMinimizeUpgrade(pkgDepCache &Cache) continue; // Keep it and see if that is OK - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); if (Cache.BrokenCount() != 0) - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); else { // If keep didnt actually do anything then there was no change.. @@ -565,6 +577,8 @@ void pkgProblemResolver::MakeScores() installable */ bool pkgProblemResolver::DoUpgrade(pkgCache::PkgIterator Pkg) { + pkgDepCache::ActionGroup group(Cache); + if ((Flags[Pkg->ID] & Upgradable) == 0 || Cache[Pkg].Upgradable() == false) return false; if ((Flags[Pkg->ID] & Protected) == Protected) @@ -573,7 +587,7 @@ bool pkgProblemResolver::DoUpgrade(pkgCache::PkgIterator Pkg) Flags[Pkg->ID] &= ~Upgradable; bool WasKept = Cache[Pkg].Keep(); - Cache.MarkInstall(Pkg,false); + Cache.MarkInstall(Pkg, false, 0, false); // This must be a virtual package or something like that. if (Cache[Pkg].InstVerIter(Cache).end() == true) @@ -658,7 +672,7 @@ bool pkgProblemResolver::DoUpgrade(pkgCache::PkgIterator Pkg) if (Fail == true) { if (WasKept == true) - Cache.MarkKeep(Pkg); + Cache.MarkKeep(Pkg, false, false); else Cache.MarkDelete(Pkg); return false; @@ -685,6 +699,8 @@ bool pkgProblemResolver::DoUpgrade(pkgCache::PkgIterator Pkg) upgrade packages to advoid problems. */ bool pkgProblemResolver::Resolve(bool BrokenFix) { + pkgDepCache::ActionGroup group(Cache); + unsigned long Size = Cache.Head().PackageCount; // Record which packages are marked for install @@ -700,7 +716,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) { if (Cache[I].InstBroken() == true && BrokenFix == true) { - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); if (Cache[I].Install() == true) Again = true; } @@ -766,14 +782,14 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) pkgCache::Version *OldVer = Cache[I].InstallVer; Flags[I->ID] &= ReInstateTried; - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); if (Cache[I].InstBroken() == true || OldBreaks < Cache.BrokenCount()) { if (OldVer == 0) Cache.MarkDelete(I); else - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); } else if (Debug == true) @@ -818,7 +834,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) { if (Debug == true) clog << " Or group keep for " << I.Name() << endl; - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); Change = true; } } @@ -868,7 +884,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) } Change = true; - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); break; } @@ -905,7 +921,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) /* See if a keep will do, unless the package is protected, then installing it will be necessary */ bool Installed = Cache[I].Install(); - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); if (Cache[I].InstBroken() == false) { // Unwind operation will be keep now @@ -914,7 +930,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) // Restore if (InOr == true && Installed == true) - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); if (Debug == true) clog << " Holding Back " << I.Name() << " rather than change " << Start.TargetPkg().Name() << endl; @@ -986,7 +1002,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) // Restore if (InOr == true && Installed == true) - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); if (Debug == true) clog << " Holding Back " << I.Name() << " because I can't find " << Start.TargetPkg().Name() << endl; @@ -1031,7 +1047,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) { if (Debug == true) clog << " Fixing " << I.Name() << " via keep of " << J->Pkg.Name() << endl; - Cache.MarkKeep(J->Pkg); + Cache.MarkKeep(J->Pkg, false, false); } if (Counter > 1) @@ -1061,6 +1077,20 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) return _error->Error(_("Unable to correct problems, you have held broken packages.")); } + // set the auto-flags (mvo: I'm not sure if we _really_ need this, but + // I didn't managed + pkgCache::PkgIterator I = Cache.PkgBegin(); + for (;I.end() != true; I++) { + if (Cache[I].NewInstall() && !(Flags[I->ID] & PreInstalled)) { + if(_config->FindI("Debug::pkgAutoRemove",false)) { + std::clog << "Resolve installed new pkg: " << I.Name() + << " (now marking it as auto)" << std::endl; + } + Cache[I].Flags |= pkgCache::Flag::Auto; + } + } + + return true; } /*}}}*/ @@ -1071,6 +1101,8 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) system was non-broken previously. */ bool pkgProblemResolver::ResolveByKeep() { + pkgDepCache::ActionGroup group(Cache); + unsigned long Size = Cache.Head().PackageCount; if (Debug == true) @@ -1104,7 +1136,7 @@ bool pkgProblemResolver::ResolveByKeep() { if (Debug == true) clog << "Keeping package " << I.Name() << endl; - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); if (Cache[I].InstBroken() == false) { K = PList - 1; @@ -1152,7 +1184,7 @@ bool pkgProblemResolver::ResolveByKeep() { if (Debug == true) clog << " Keeping Package " << Pkg.Name() << " due to dep" << endl; - Cache.MarkKeep(Pkg); + Cache.MarkKeep(Pkg, false, false); } if (Cache[I].InstBroken() == false) @@ -1189,6 +1221,8 @@ bool pkgProblemResolver::ResolveByKeep() /* This is used to make sure protected packages are installed */ void pkgProblemResolver::InstallProtect() { + pkgDepCache::ActionGroup group(Cache); + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; I++) { if ((Flags[I->ID] & Protected) == Protected) @@ -1196,7 +1230,7 @@ void pkgProblemResolver::InstallProtect() if ((Flags[I->ID] & ToRemove) == ToRemove) Cache.MarkDelete(I); else - Cache.MarkInstall(I,false); + Cache.MarkInstall(I, false, 0, false); } } } @@ -1232,3 +1266,4 @@ void pkgPrioSortList(pkgCache &Cache,pkgCache::Version **List) qsort(List,Count,sizeof(*List),PrioComp); } /*}}}*/ + diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 64fa4636e..d5a9c7b0d 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -107,7 +107,7 @@ class pkgCache::VerIterator // Iteration void operator ++(int) {if (Ver != Owner->VerP) Ver = Owner->VerP + Ver->NextVer;}; inline void operator ++() {operator ++(0);}; - inline bool end() const {return Ver == Owner->VerP?true:false;}; + inline bool end() const {return Owner == NULL || (Ver == Owner->VerP?true:false);}; inline void operator =(const VerIterator &B) {Ver = B.Ver; Owner = B.Owner;}; // Comparison diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index 09e454be9..14a000fa5 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -110,7 +110,7 @@ Configuration::Item *Configuration::Lookup(Item *Head,const char *S, return 0; I = new Item; - I->Tag = string(S,Len); + I->Tag.assign(S,Len); I->Next = *Last; I->Parent = Head; *Last = I; @@ -161,7 +161,7 @@ string Configuration::Find(const char *Name,const char *Default) const if (Itm == 0 || Itm->Value.empty() == true) { if (Default == 0) - return string(); + return ""; else return Default; } @@ -180,7 +180,7 @@ string Configuration::FindFile(const char *Name,const char *Default) const if (Itm == 0 || Itm->Value.empty() == true) { if (Default == 0) - return string(); + return ""; else return Default; } @@ -294,7 +294,7 @@ string Configuration::FindAny(const char *Name,const char *Default) const // Configuration::CndSet - Conditinal Set a value /*{{{*/ // --------------------------------------------------------------------- /* This will not overwrite */ -void Configuration::CndSet(const char *Name,string Value) +void Configuration::CndSet(const char *Name,const string &Value) { Item *Itm = Lookup(Name,true); if (Itm == 0) @@ -306,7 +306,7 @@ void Configuration::CndSet(const char *Name,string Value) // Configuration::Set - Set a value /*{{{*/ // --------------------------------------------------------------------- /* */ -void Configuration::Set(const char *Name,string Value) +void Configuration::Set(const char *Name,const string &Value) { Item *Itm = Lookup(Name,true); if (Itm == 0) @@ -330,7 +330,7 @@ void Configuration::Set(const char *Name,int Value) // Configuration::Clear - Clear an single value from a list /*{{{*/ // --------------------------------------------------------------------- /* */ -void Configuration::Clear(string Name, int Value) +void Configuration::Clear(const string Name, int Value) { char S[300]; snprintf(S,sizeof(S),"%i",Value); @@ -340,7 +340,7 @@ void Configuration::Clear(string Name, int Value) // Configuration::Clear - Clear an single value from a list /*{{{*/ // --------------------------------------------------------------------- /* */ -void Configuration::Clear(string Name, string Value) +void Configuration::Clear(const string Name, string Value) { Item *Top = Lookup(Name.c_str(),false); if (Top == 0 || Top->Child == 0) @@ -377,7 +377,7 @@ void Configuration::Clear(string Name) if (Top == 0) return; - Top->Value = string(); + Top->Value.clear(); Item *Stop = Top; Top = Top->Child; Stop->Child = 0; @@ -485,7 +485,7 @@ string Configuration::Item::FullTag(const Item *Stop) const sections like 'zone "foo.org" { .. };' This causes each section to be added in with a tag like "zone::foo.org" instead of being split tag/value. AsSectional enables Sectional parsing.*/ -bool ReadConfigFile(Configuration &Conf,string FName,bool AsSectional, +bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional, unsigned Depth) { // Open the stream for reading @@ -711,13 +711,13 @@ bool ReadConfigFile(Configuration &Conf,string FName,bool AsSectional, } // Empty the buffer - LineBuffer = string(); + LineBuffer.clear(); // Move up a tag, but only if there is no bit to parse if (TermChar == '}') { if (StackPos == 0) - ParentTag = string(); + ParentTag.clear(); else ParentTag = Stack[--StackPos]; } @@ -742,8 +742,8 @@ bool ReadConfigFile(Configuration &Conf,string FName,bool AsSectional, // ReadConfigDir - Read a directory of config files /*{{{*/ // --------------------------------------------------------------------- /* */ -bool ReadConfigDir(Configuration &Conf,string Dir,bool AsSectional, - unsigned Depth) +bool ReadConfigDir(Configuration &Conf,const string &Dir,bool AsSectional, + unsigned Depth) { DIR *D = opendir(Dir.c_str()); if (D == 0) diff --git a/apt-pkg/contrib/configuration.h b/apt-pkg/contrib/configuration.h index 789bc82cf..0d4078dab 100644 --- a/apt-pkg/contrib/configuration.h +++ b/apt-pkg/contrib/configuration.h @@ -69,30 +69,30 @@ class Configuration public: string Find(const char *Name,const char *Default = 0) const; - string Find(string Name,const char *Default = 0) const {return Find(Name.c_str(),Default);}; + string Find(const string Name,const char *Default = 0) const {return Find(Name.c_str(),Default);}; string FindFile(const char *Name,const char *Default = 0) const; string FindDir(const char *Name,const char *Default = 0) const; int FindI(const char *Name,int Default = 0) const; - int FindI(string Name,int Default = 0) const {return FindI(Name.c_str(),Default);}; + int FindI(const string Name,int Default = 0) const {return FindI(Name.c_str(),Default);}; bool FindB(const char *Name,bool Default = false) const; - bool FindB(string Name,bool Default = false) const {return FindB(Name.c_str(),Default);}; + bool FindB(const string Name,bool Default = false) const {return FindB(Name.c_str(),Default);}; string FindAny(const char *Name,const char *Default = 0) const; - inline void Set(string Name,string Value) {Set(Name.c_str(),Value);}; - void CndSet(const char *Name,string Value); - void Set(const char *Name,string Value); + inline void Set(const string Name,string Value) {Set(Name.c_str(),Value);}; + void CndSet(const char *Name,const string &Value); + void Set(const char *Name,const string &Value); void Set(const char *Name,int Value); - inline bool Exists(string Name) const {return Exists(Name.c_str());}; + inline bool Exists(const string Name) const {return Exists(Name.c_str());}; bool Exists(const char *Name) const; bool ExistsAny(const char *Name) const; // clear a whole tree - void Clear(string Name); + void Clear(const string Name); // remove a certain value from a list (e.g. the list of "APT::Keep-Fds") - void Clear(string List, string Value); - void Clear(string List, int Value); + void Clear(const string List, string Value); + void Clear(const string List, int Value); inline const Item *Tree(const char *Name) const {return Lookup(Name);}; @@ -106,10 +106,12 @@ class Configuration extern Configuration *_config; -bool ReadConfigFile(Configuration &Conf,string FName,bool AsSectional = false, +bool ReadConfigFile(Configuration &Conf,const string &FName, + bool AsSectional = false, unsigned Depth = 0); -bool ReadConfigDir(Configuration &Conf,string Dir,bool AsSectional = false, - unsigned Depth = 0); +bool ReadConfigDir(Configuration &Conf,const string &Dir, + bool AsSectional = false, + unsigned Depth = 0); #endif diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index b17b94319..9b22a90d3 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -36,6 +36,7 @@ bool Hashes::AddFD(int Fd,unsigned long Size) Size -= Res; MD5.Add(Buf,Res); SHA1.Add(Buf,Res); + SHA256.Add(Buf,Res); } return true; } diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h index 40bbe00a0..eefa7bf41 100644 --- a/apt-pkg/contrib/hashes.h +++ b/apt-pkg/contrib/hashes.h @@ -19,6 +19,7 @@ #include <apt-pkg/md5.h> #include <apt-pkg/sha1.h> +#include <apt-pkg/sha256.h> #include <algorithm> @@ -30,10 +31,11 @@ class Hashes MD5Summation MD5; SHA1Summation SHA1; + SHA256Summation SHA256; inline bool Add(const unsigned char *Data,unsigned long Size) { - return MD5.Add(Data,Size) && SHA1.Add(Data,Size); + return MD5.Add(Data,Size) && SHA1.Add(Data,Size) && SHA256.Add(Data,Size); }; inline bool Add(const char *Data) {return Add((unsigned char *)Data,strlen(Data));}; bool AddFD(int Fd,unsigned long Size); diff --git a/apt-pkg/contrib/md5.h b/apt-pkg/contrib/md5.h index 9447e9956..e280d714e 100644 --- a/apt-pkg/contrib/md5.h +++ b/apt-pkg/contrib/md5.h @@ -29,6 +29,7 @@ #include <string> #include <algorithm> +#include <stdint.h> using std::string; using std::min; diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h index caffa0f90..e329b167a 100644 --- a/apt-pkg/contrib/mmap.h +++ b/apt-pkg/contrib/mmap.h @@ -94,7 +94,7 @@ class DynamicMMap : public MMap unsigned long RawAllocate(unsigned long Size,unsigned long Aln = 0); unsigned long Allocate(unsigned long ItemSize); unsigned long WriteString(const char *String,unsigned long Len = (unsigned long)-1); - inline unsigned long WriteString(string S) {return WriteString(S.c_str(),S.length());}; + inline unsigned long WriteString(const string &S) {return WriteString(S.c_str(),S.length());}; void UsePools(Pool &P,unsigned int Count) {Pools = &P; PoolCount = Count;}; DynamicMMap(FileFd &F,unsigned long Flags,unsigned long WorkSpace = 2*1024*1024); diff --git a/apt-pkg/contrib/progress.cc b/apt-pkg/contrib/progress.cc index 8eb36fc20..cb272e389 100644 --- a/apt-pkg/contrib/progress.cc +++ b/apt-pkg/contrib/progress.cc @@ -50,7 +50,7 @@ void OpProgress::Progress(unsigned long Cur) // --------------------------------------------------------------------- /* */ void OpProgress::OverallProgress(unsigned long Current, unsigned long Total, - unsigned long Size,string Op) + unsigned long Size,const string &Op) { this->Current = Current; this->Total = Total; @@ -67,7 +67,7 @@ void OpProgress::OverallProgress(unsigned long Current, unsigned long Total, // OpProgress::SubProgress - Set the sub progress state /*{{{*/ // --------------------------------------------------------------------- /* */ -void OpProgress::SubProgress(unsigned long SubTotal,string Op) +void OpProgress::SubProgress(unsigned long SubTotal,const string &Op) { this->SubTotal = SubTotal; SubOp = Op; diff --git a/apt-pkg/contrib/progress.h b/apt-pkg/contrib/progress.h index d0b1f5f94..20caf4cdf 100644 --- a/apt-pkg/contrib/progress.h +++ b/apt-pkg/contrib/progress.h @@ -59,9 +59,9 @@ class OpProgress void Progress(unsigned long Current); void SubProgress(unsigned long SubTotal); - void SubProgress(unsigned long SubTotal,string Op); + void SubProgress(unsigned long SubTotal,const string &Op); void OverallProgress(unsigned long Current,unsigned long Total, - unsigned long Size,string Op); + unsigned long Size,const string &Op); virtual void Done() {}; OpProgress(); diff --git a/apt-pkg/contrib/sha256.cc b/apt-pkg/contrib/sha256.cc new file mode 100644 index 000000000..ad2ddb2d3 --- /dev/null +++ b/apt-pkg/contrib/sha256.cc @@ -0,0 +1,424 @@ +/* + * Cryptographic API. + * + * SHA-256, as specified in + * http://csrc.nist.gov/cryptval/shs/sha256-384-512.pdf + * + * SHA-256 code by Jean-Luc Cooke <jlcooke@certainkey.com>. + * + * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com> + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> + * + * Ported from the Linux kernel to Apt by Anthony Towns <ajt@debian.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#define SHA256_DIGEST_SIZE 32 +#define SHA256_HMAC_BLOCK_SIZE 64 + +#define ror32(value,bits) (((value) >> (bits)) | ((value) << (32 - (bits)))) + +#include <apt-pkg/sha256.h> +#include <apt-pkg/strutl.h> +#include <string.h> +#include <unistd.h> +#include <stdint.h> +#include <stdlib.h> +#include <stdio.h> +#include <arpa/inet.h> + +typedef uint32_t u32; +typedef uint8_t u8; + +static inline u32 Ch(u32 x, u32 y, u32 z) +{ + return z ^ (x & (y ^ z)); +} + +static inline u32 Maj(u32 x, u32 y, u32 z) +{ + return (x & y) | (z & (x | y)); +} + +#define e0(x) (ror32(x, 2) ^ ror32(x,13) ^ ror32(x,22)) +#define e1(x) (ror32(x, 6) ^ ror32(x,11) ^ ror32(x,25)) +#define s0(x) (ror32(x, 7) ^ ror32(x,18) ^ (x >> 3)) +#define s1(x) (ror32(x,17) ^ ror32(x,19) ^ (x >> 10)) + +#define H0 0x6a09e667 +#define H1 0xbb67ae85 +#define H2 0x3c6ef372 +#define H3 0xa54ff53a +#define H4 0x510e527f +#define H5 0x9b05688c +#define H6 0x1f83d9ab +#define H7 0x5be0cd19 + +static inline void LOAD_OP(int I, u32 *W, const u8 *input) +{ + W[I] = ( ((u32) input[I + 0] << 24) + | ((u32) input[I + 1] << 16) + | ((u32) input[I + 2] << 8) + | ((u32) input[I + 3])); +} + +static inline void BLEND_OP(int I, u32 *W) +{ + W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; +} + +static void sha256_transform(u32 *state, const u8 *input) +{ + u32 a, b, c, d, e, f, g, h, t1, t2; + u32 W[64]; + int i; + + /* load the input */ + for (i = 0; i < 16; i++) + LOAD_OP(i, W, input); + + /* now blend */ + for (i = 16; i < 64; i++) + BLEND_OP(i, W); + + /* load the state into our registers */ + a=state[0]; b=state[1]; c=state[2]; d=state[3]; + e=state[4]; f=state[5]; g=state[6]; h=state[7]; + + /* now iterate */ + t1 = h + e1(e) + Ch(e,f,g) + 0x428a2f98 + W[ 0]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0x71374491 + W[ 1]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0xb5c0fbcf + W[ 2]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0xe9b5dba5 + W[ 3]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0x3956c25b + W[ 4]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0x59f111f1 + W[ 5]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0x923f82a4 + W[ 6]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0xab1c5ed5 + W[ 7]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + t1 = h + e1(e) + Ch(e,f,g) + 0xd807aa98 + W[ 8]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0x12835b01 + W[ 9]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0x243185be + W[10]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0x550c7dc3 + W[11]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0x72be5d74 + W[12]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0x80deb1fe + W[13]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0x9bdc06a7 + W[14]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0xc19bf174 + W[15]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + t1 = h + e1(e) + Ch(e,f,g) + 0xe49b69c1 + W[16]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0xefbe4786 + W[17]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0x0fc19dc6 + W[18]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0x240ca1cc + W[19]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0x2de92c6f + W[20]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0x4a7484aa + W[21]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0x5cb0a9dc + W[22]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0x76f988da + W[23]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + t1 = h + e1(e) + Ch(e,f,g) + 0x983e5152 + W[24]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0xa831c66d + W[25]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0xb00327c8 + W[26]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0xbf597fc7 + W[27]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0xc6e00bf3 + W[28]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0xd5a79147 + W[29]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0x06ca6351 + W[30]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0x14292967 + W[31]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + t1 = h + e1(e) + Ch(e,f,g) + 0x27b70a85 + W[32]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0x2e1b2138 + W[33]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0x4d2c6dfc + W[34]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0x53380d13 + W[35]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0x650a7354 + W[36]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0x766a0abb + W[37]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0x81c2c92e + W[38]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0x92722c85 + W[39]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + t1 = h + e1(e) + Ch(e,f,g) + 0xa2bfe8a1 + W[40]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0xa81a664b + W[41]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0xc24b8b70 + W[42]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0xc76c51a3 + W[43]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0xd192e819 + W[44]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0xd6990624 + W[45]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0xf40e3585 + W[46]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0x106aa070 + W[47]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + t1 = h + e1(e) + Ch(e,f,g) + 0x19a4c116 + W[48]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0x1e376c08 + W[49]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0x2748774c + W[50]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0x34b0bcb5 + W[51]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0x391c0cb3 + W[52]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0x4ed8aa4a + W[53]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0x5b9cca4f + W[54]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0x682e6ff3 + W[55]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + t1 = h + e1(e) + Ch(e,f,g) + 0x748f82ee + W[56]; + t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; + t1 = g + e1(d) + Ch(d,e,f) + 0x78a5636f + W[57]; + t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; + t1 = f + e1(c) + Ch(c,d,e) + 0x84c87814 + W[58]; + t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; + t1 = e + e1(b) + Ch(b,c,d) + 0x8cc70208 + W[59]; + t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; + t1 = d + e1(a) + Ch(a,b,c) + 0x90befffa + W[60]; + t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; + t1 = c + e1(h) + Ch(h,a,b) + 0xa4506ceb + W[61]; + t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; + t1 = b + e1(g) + Ch(g,h,a) + 0xbef9a3f7 + W[62]; + t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; + t1 = a + e1(f) + Ch(f,g,h) + 0xc67178f2 + W[63]; + t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; + + state[0] += a; state[1] += b; state[2] += c; state[3] += d; + state[4] += e; state[5] += f; state[6] += g; state[7] += h; + + /* clear any sensitive info... */ + a = b = c = d = e = f = g = h = t1 = t2 = 0; + memset(W, 0, 64 * sizeof(u32)); +} + +SHA256Summation::SHA256Summation() +{ + Sum.state[0] = H0; + Sum.state[1] = H1; + Sum.state[2] = H2; + Sum.state[3] = H3; + Sum.state[4] = H4; + Sum.state[5] = H5; + Sum.state[6] = H6; + Sum.state[7] = H7; + Sum.count[0] = Sum.count[1] = 0; + memset(Sum.buf, 0, sizeof(Sum.buf)); + Done = false; +} + +bool SHA256Summation::Add(const u8 *data, unsigned long len) +{ + struct sha256_ctx *sctx = ∑ + unsigned int i, index, part_len; + + if (Done) return false; + + /* Compute number of bytes mod 128 */ + index = (unsigned int)((sctx->count[0] >> 3) & 0x3f); + + /* Update number of bits */ + if ((sctx->count[0] += (len << 3)) < (len << 3)) { + sctx->count[1]++; + sctx->count[1] += (len >> 29); + } + + part_len = 64 - index; + + /* Transform as many times as possible. */ + if (len >= part_len) { + memcpy(&sctx->buf[index], data, part_len); + sha256_transform(sctx->state, sctx->buf); + + for (i = part_len; i + 63 < len; i += 64) + sha256_transform(sctx->state, &data[i]); + index = 0; + } else { + i = 0; + } + + /* Buffer remaining input */ + memcpy(&sctx->buf[index], &data[i], len-i); + + return true; +} + +SHA256SumValue SHA256Summation::Result() +{ + struct sha256_ctx *sctx = ∑ + if (!Done) { + u8 bits[8]; + unsigned int index, pad_len, t; + static const u8 padding[64] = { 0x80, }; + + /* Save number of bits */ + t = sctx->count[0]; + bits[7] = t; t >>= 8; + bits[6] = t; t >>= 8; + bits[5] = t; t >>= 8; + bits[4] = t; + t = sctx->count[1]; + bits[3] = t; t >>= 8; + bits[2] = t; t >>= 8; + bits[1] = t; t >>= 8; + bits[0] = t; + + /* Pad out to 56 mod 64. */ + index = (sctx->count[0] >> 3) & 0x3f; + pad_len = (index < 56) ? (56 - index) : ((64+56) - index); + Add(padding, pad_len); + + /* Append length (before padding) */ + Add(bits, 8); + } + + Done = true; + + /* Store state in digest */ + + SHA256SumValue res; + u8 *out = res.Sum; + + int i, j; + unsigned int t; + for (i = j = 0; i < 8; i++, j += 4) { + t = sctx->state[i]; + out[j+3] = t; t >>= 8; + out[j+2] = t; t >>= 8; + out[j+1] = t; t >>= 8; + out[j ] = t; + } + + return res; +} + +// SHA256SumValue::SHA256SumValue - Constructs the sum from a string /*{{{*/ +// --------------------------------------------------------------------- +/* The string form of a SHA256 is a 64 character hex number */ +SHA256SumValue::SHA256SumValue(string Str) +{ + memset(Sum,0,sizeof(Sum)); + Set(Str); +} + + /*}}}*/ +// SHA256SumValue::SHA256SumValue - Default constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Sets the value to 0 */ +SHA256SumValue::SHA256SumValue() +{ + memset(Sum,0,sizeof(Sum)); +} + + /*}}}*/ +// SHA256SumValue::Set - Set the sum from a string /*{{{*/ +// --------------------------------------------------------------------- +/* Converts the hex string into a set of chars */ +bool SHA256SumValue::Set(string Str) +{ + return Hex2Num(Str,Sum,sizeof(Sum)); +} + /*}}}*/ +// SHA256SumValue::Value - Convert the number into a string /*{{{*/ +// --------------------------------------------------------------------- +/* Converts the set of chars into a hex string in lower case */ +string SHA256SumValue::Value() const +{ + char Conv[16] = + { '0','1','2','3','4','5','6','7','8','9','a','b', + 'c','d','e','f' + }; + char Result[65]; + Result[64] = 0; + + // Convert each char into two letters + int J = 0; + int I = 0; + for (; I != 64; J++,I += 2) + { + Result[I] = Conv[Sum[J] >> 4]; + Result[I + 1] = Conv[Sum[J] & 0xF]; + } + + return string(Result); +} + + + +// SHA256SumValue::operator == - Comparator /*{{{*/ +// --------------------------------------------------------------------- +/* Call memcmp on the buffer */ +bool SHA256SumValue::operator == (const SHA256SumValue & rhs) const +{ + return memcmp(Sum,rhs.Sum,sizeof(Sum)) == 0; +} + /*}}}*/ + + +// SHA256Summation::AddFD - Add content of file into the checksum /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool SHA256Summation::AddFD(int Fd,unsigned long Size) +{ + unsigned char Buf[64 * 64]; + int Res = 0; + int ToEOF = (Size == 0); + while (Size != 0 || ToEOF) + { + unsigned n = sizeof(Buf); + if (!ToEOF) n = min(Size,(unsigned long)n); + Res = read(Fd,Buf,n); + if (Res < 0 || (!ToEOF && (unsigned) Res != n)) // error, or short read + return false; + if (ToEOF && Res == 0) // EOF + break; + Size -= Res; + Add(Buf,Res); + } + return true; +} + /*}}}*/ + diff --git a/apt-pkg/contrib/sha256.h b/apt-pkg/contrib/sha256.h new file mode 100644 index 000000000..9e88f5ece --- /dev/null +++ b/apt-pkg/contrib/sha256.h @@ -0,0 +1,75 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: sha1.h,v 1.3 2001/05/07 05:05:47 jgg Exp $ +/* ###################################################################### + + SHA256SumValue - Storage for a SHA-256 hash. + SHA256Summation - SHA-256 Secure Hash Algorithm. + + This is a C++ interface to a set of SHA256Sum functions, that mirrors + the equivalent MD5 & SHA1 classes. + + ##################################################################### */ + /*}}}*/ +#ifndef APTPKG_SHA256_H +#define APTPKG_SHA256_H + +#ifdef __GNUG__ +#pragma interface "apt-pkg/sha256.h" +#endif + +#include <string> +#include <algorithm> +#include <stdint.h> + +using std::string; +using std::min; + +class SHA256Summation; + +class SHA256SumValue +{ + friend class SHA256Summation; + unsigned char Sum[32]; + + public: + + // Accessors + bool operator ==(const SHA256SumValue &rhs) const; + string Value() const; + inline void Value(unsigned char S[32]) + {for (int I = 0; I != sizeof(Sum); I++) S[I] = Sum[I];}; + inline operator string() const {return Value();}; + bool Set(string Str); + inline void Set(unsigned char S[32]) + {for (int I = 0; I != sizeof(Sum); I++) Sum[I] = S[I];}; + + SHA256SumValue(string Str); + SHA256SumValue(); +}; + +struct sha256_ctx { + uint32_t count[2]; + uint32_t state[8]; + uint8_t buf[128]; +}; + +class SHA256Summation +{ + struct sha256_ctx Sum; + + bool Done; + + public: + + bool Add(const unsigned char *inbuf,unsigned long inlen); + inline bool Add(const char *Data) {return Add((unsigned char *)Data,strlen(Data));}; + bool AddFD(int Fd,unsigned long Size); + inline bool Add(const unsigned char *Beg,const unsigned char *End) + {return Add(Beg,End-Beg);}; + SHA256SumValue Result(); + + SHA256Summation(); +}; + +#endif diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index 303cb27db..37d263794 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -242,10 +242,10 @@ bool ParseCWord(const char *&String,string &Res) // QuoteString - Convert a string into quoted from /*{{{*/ // --------------------------------------------------------------------- /* */ -string QuoteString(string Str,const char *Bad) +string QuoteString(const string &Str, const char *Bad) { string Res; - for (string::iterator I = Str.begin(); I != Str.end(); I++) + for (string::const_iterator I = Str.begin(); I != Str.end(); I++) { if (strchr(Bad,*I) != 0 || isprint(*I) == 0 || *I <= 0x20 || *I >= 0x7F) @@ -263,7 +263,7 @@ string QuoteString(string Str,const char *Bad) // DeQuoteString - Convert a string from quoted from /*{{{*/ // --------------------------------------------------------------------- /* This undoes QuoteString */ -string DeQuoteString(string Str) +string DeQuoteString(const string &Str) { string Res; for (string::const_iterator I = Str.begin(); I != Str.end(); I++) @@ -360,7 +360,7 @@ string TimeToStr(unsigned long Sec) // SubstVar - Substitute a string for another string /*{{{*/ // --------------------------------------------------------------------- /* This replaces all occurances of Subst with Contents in Str. */ -string SubstVar(string Str,string Subst,string Contents) +string SubstVar(const string &Str,const string &Subst,const string &Contents) { string::size_type Pos = 0; string::size_type OldPos = 0; @@ -391,21 +391,18 @@ string SubstVar(string Str,const struct SubstVar *Vars) /* This converts a URI into a safe filename. It quotes all unsafe characters and converts / to _ and removes the scheme identifier. The resulting file name should be unique and never occur again for a different file */ -string URItoFileName(string URI) +string URItoFileName(const string &URI) { // Nuke 'sensitive' items ::URI U(URI); - U.User = string(); - U.Password = string(); - U.Access = ""; + U.User.clear(); + U.Password.clear(); + U.Access.clear(); // "\x00-\x20{}|\\\\^\\[\\]<>\"\x7F-\xFF"; - URI = QuoteString(U,"\\|{}[]<>\"^~=!@#$%^&*"); - string::iterator J = URI.begin(); - for (; J != URI.end(); J++) - if (*J == '/') - *J = '_'; - return URI; + string NewURI = QuoteString(U,"\\|{}[]<>\"^~_=!@#$%^&*"); + replace(NewURI.begin(),NewURI.end(),'/','_'); + return NewURI; } /*}}}*/ // Base64Encode - Base64 Encoding routine for short strings /*{{{*/ @@ -414,7 +411,7 @@ string URItoFileName(string URI) from wget and then patched and bug fixed. This spec can be found in rfc2045 */ -string Base64Encode(string S) +string Base64Encode(const string &S) { // Conversion table. static char tbl[64] = {'A','B','C','D','E','F','G','H', @@ -583,17 +580,17 @@ int stringcasecmp(string::const_iterator A,string::const_iterator AEnd, // --------------------------------------------------------------------- /* The format is like those used in package files and the method communication system */ -string LookupTag(string Message,const char *Tag,const char *Default) +string LookupTag(const string &Message,const char *Tag,const char *Default) { // Look for a matching tag. int Length = strlen(Tag); - for (string::iterator I = Message.begin(); I + Length < Message.end(); I++) + for (string::const_iterator I = Message.begin(); I + Length < Message.end(); I++) { // Found the tag if (I[Length] == ':' && stringcasecmp(I,I+Length,Tag) == 0) { // Find the end of line and strip the leading/trailing spaces - string::iterator J; + string::const_iterator J; I += Length + 1; for (; isspace(*I) != 0 && I < Message.end(); I++); for (J = I; *J != '\n' && J < Message.end(); J++); @@ -615,7 +612,7 @@ string LookupTag(string Message,const char *Tag,const char *Default) // --------------------------------------------------------------------- /* This inspects the string to see if it is true or if it is false and then returns the result. Several varients on true/false are checked. */ -int StringToBool(string Text,int Default) +int StringToBool(const string &Text,int Default) { char *End; int Res = strtol(Text.c_str(),&End,0); @@ -781,7 +778,7 @@ static time_t timegm(struct tm *t) 'timegm' to convert a struct tm in UTC to a time_t. For some bizzar reason the C library does not provide any such function :< This also handles the weird, but unambiguous FTP time format*/ -bool StrToTime(string Val,time_t &Result) +bool StrToTime(const string &Val,time_t &Result) { struct tm Tm; char Month[10]; @@ -868,7 +865,7 @@ static int HexDigit(int c) // Hex2Num - Convert a long hex number into a buffer /*{{{*/ // --------------------------------------------------------------------- /* The length of the buffer must be exactly 1/2 the length of the string. */ -bool Hex2Num(string Str,unsigned char *Num,unsigned int Length) +bool Hex2Num(const string &Str,unsigned char *Num,unsigned int Length) { if (Str.length() != Length*2) return false; @@ -1029,7 +1026,7 @@ char *safe_snprintf(char *Buffer,char *End,const char *Format,...) // --------------------------------------------------------------------- /* The domain list is a comma seperate list of domains that are suffix matched against the argument */ -bool CheckDomainList(string Host,string List) +bool CheckDomainList(const string &Host,const string &List) { string::const_iterator Start = List.begin(); for (string::const_iterator Cur = List.begin(); Cur <= List.end(); Cur++) @@ -1052,7 +1049,7 @@ bool CheckDomainList(string Host,string List) // URI::CopyFrom - Copy from an object /*{{{*/ // --------------------------------------------------------------------- /* This parses the URI into all of its components */ -void URI::CopyFrom(string U) +void URI::CopyFrom(const string &U) { string::const_iterator I = U.begin(); @@ -1081,9 +1078,9 @@ void URI::CopyFrom(string U) SingleSlash = U.end(); // We can now write the access and path specifiers - Access = string(U,0,FirstColon - U.begin()); + Access.assign(U.begin(),FirstColon); if (SingleSlash != U.end()) - Path = string(U,SingleSlash - U.begin()); + Path.assign(SingleSlash,U.end()); if (Path.empty() == true) Path = "/"; @@ -1113,14 +1110,14 @@ void URI::CopyFrom(string U) if (At == SingleSlash) { if (FirstColon < SingleSlash) - Host = string(U,FirstColon - U.begin(),SingleSlash - FirstColon); + Host.assign(FirstColon,SingleSlash); } else { - Host = string(U,At - U.begin() + 1,SingleSlash - At - 1); - User = string(U,FirstColon - U.begin(),SecondColon - FirstColon); + Host.assign(At+1,SingleSlash); + User.assign(FirstColon,SecondColon); if (SecondColon < At) - Password = string(U,SecondColon - U.begin() + 1,At - SecondColon - 1); + Password.assign(SecondColon+1,At); } // Now we parse the RFC 2732 [] hostnames. @@ -1148,7 +1145,7 @@ void URI::CopyFrom(string U) // Tsk, weird. if (InBracket == true) { - Host = string(); + Host.clear(); return; } @@ -1159,7 +1156,7 @@ void URI::CopyFrom(string U) return; Port = atoi(string(Host,Pos+1).c_str()); - Host = string(Host,0,Pos); + Host.assign(Host,0,Pos); } /*}}}*/ // URI::operator string - Convert the URI to a string /*{{{*/ @@ -1214,12 +1211,12 @@ URI::operator string() // URI::SiteOnly - Return the schema and site for the URI /*{{{*/ // --------------------------------------------------------------------- /* */ -string URI::SiteOnly(string URI) +string URI::SiteOnly(const string &URI) { ::URI U(URI); - U.User = string(); - U.Password = string(); - U.Path = string(); + U.User.clear(); + U.Password.clear(); + U.Path.clear(); U.Port = 0; return U; } diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index 72fc34d6d..254087267 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -44,24 +44,24 @@ char *_strstrip(char *String); char *_strtabexpand(char *String,size_t Len); bool ParseQuoteWord(const char *&String,string &Res); bool ParseCWord(const char *&String,string &Res); -string QuoteString(string Str,const char *Bad); -string DeQuoteString(string Str); +string QuoteString(const string &Str,const char *Bad); +string DeQuoteString(const string &Str); string SizeToStr(double Bytes); string TimeToStr(unsigned long Sec); -string Base64Encode(string Str); -string URItoFileName(string URI); +string Base64Encode(const string &Str); +string URItoFileName(const string &URI); string TimeRFC1123(time_t Date); -bool StrToTime(string Val,time_t &Result); -string LookupTag(string Message,const char *Tag,const char *Default = 0); -int StringToBool(string Text,int Default = -1); +bool StrToTime(const string &Val,time_t &Result); +string LookupTag(const string &Message,const char *Tag,const char *Default = 0); +int StringToBool(const string &Text,int Default = -1); bool ReadMessages(int Fd, vector<string> &List); bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base = 0); -bool Hex2Num(string Str,unsigned char *Num,unsigned int Length); +bool Hex2Num(const string &Str,unsigned char *Num,unsigned int Length); bool TokSplitString(char Tok,char *Input,char **List, unsigned long ListMax); void ioprintf(ostream &out,const char *format,...) APT_FORMAT2; char *safe_snprintf(char *Buffer,char *End,const char *Format,...) APT_FORMAT3; -bool CheckDomainList(string Host,string List); +bool CheckDomainList(const string &Host, const string &List); #define APT_MKSTRCMP(name,func) \ inline int name(const char *A,const char *AEnd,const char *B) {return func(A,AEnd,B,B+strlen(B));}; \ @@ -102,7 +102,7 @@ inline const char *DeNull(const char *s) {return (s == 0?"(null)":s);}; class URI { - void CopyFrom(string From); + void CopyFrom(const string &From); public: @@ -114,9 +114,9 @@ class URI unsigned int Port; operator string(); - inline void operator =(string From) {CopyFrom(From);}; + inline void operator =(const string &From) {CopyFrom(From);}; inline bool empty() {return Access.empty();}; - static string SiteOnly(string URI); + static string SiteOnly(const string &URI); URI(string Path) {CopyFrom(Path);}; URI() : Port(0) {}; @@ -128,7 +128,7 @@ struct SubstVar const string *Contents; }; string SubstVar(string Str,const struct SubstVar *Vars); -string SubstVar(string Str,string Subst,string Contents); +string SubstVar(const string &Str,const string &Subst,const string &Contents); struct RxChoiceList { diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 97553ab82..c2b26b5eb 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -420,12 +420,12 @@ const char *debListParser::ParseDepends(const char *Start,const char *Stop, const char *End = I; for (; End > Start && isspace(End[-1]); End--); - Ver = string(Start,End-Start); + Ver.assign(Start,End-Start); I++; } else { - Ver = string(); + Ver.clear(); Op = pkgCache::Dep::NoOp; } @@ -547,11 +547,12 @@ bool debListParser::ParseProvides(pkgCache::VerIterator Ver) Start = ParseDepends(Start,Stop,Package,Version,Op); if (Start == 0) return _error->Error("Problem parsing Provides line"); - if (Op != pkgCache::Dep::NoOp) - return _error->Error("Malformed provides line"); - - if (NewProvides(Ver,Package,Version) == false) - return false; + if (Op != pkgCache::Dep::NoOp) { + _error->Warning("Ignoring Provides line with DepCompareOp for package %s", Package.c_str()); + } else { + if (NewProvides(Ver,Package,Version) == false) + return false; + } if (Start == Stop) break; diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index a3fc39302..8cf31b326 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -175,7 +175,7 @@ bool debReleaseIndex::IsTrusted() const string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(MetaIndexURI("Release")) + ".gpg"; - if(_config->FindB("APT::Authentication::Trust-CDROM", false)) + if(_config->FindB("APT::Authentication::TrustCDROM", false)) if(URI.substr(0,strlen("cdrom:")) == "cdrom:") return true; diff --git a/apt-pkg/deb/debversion.cc b/apt-pkg/deb/debversion.cc index aeee61929..064d8fa5b 100644 --- a/apt-pkg/deb/debversion.cc +++ b/apt-pkg/deb/debversion.cc @@ -59,7 +59,7 @@ int debVersioningSystem::CmpFragment(const char *A,const char *AEnd, } /* Iterate over the whole string - What this does is to spilt the whole string into groups of + What this does is to split the whole string into groups of numeric and non numeric portions. For instance: a67bhgs89 Has 4 portions 'a', '67', 'bhgs', '89'. A more normal: @@ -140,6 +140,27 @@ int debVersioningSystem::DoCmpVersion(const char *A,const char *AEnd, if (rhs == BEnd) rhs = B; + // Special case: a zero epoch is the same as no epoch, + // so remove it. + if (lhs != A) + { + for (; *A == '0'; ++A); + if (A == lhs) + { + ++A; + ++lhs; + } + } + if (rhs != B) + { + for (; *B == '0'; ++B); + if (B == rhs) + { + ++B; + ++rhs; + } + } + // Compare the epoch int Res = CmpFragment(A,lhs,B,rhs); if (Res != 0) diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index fe8fbca74..667db8ff2 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -375,8 +375,8 @@ bool pkgDPkgPM::Go(int OutStatusFd) }, // Purge operation { - {"config-files", _("Preparing for remove with config %s")}, - {"not-installed", _("Removed with config %s")}, + {"config-files", _("Preparing to completely remove %s")}, + {"not-installed", _("Completely removed %s")}, {NULL, NULL} }, }; @@ -623,8 +623,8 @@ bool pkgDPkgPM::Go(int OutStatusFd) 'status: conffile-prompt: conffile : 'current-conffile' 'new-conffile' useredited distedited */ - char* list[4]; - TokSplitString(':', line, list, 5); + char* list[5]; + TokSplitString(':', line, list, sizeof(list)/sizeof(list[0])); char *pkg = list[1]; char *action = _strstrip(list[2]); diff --git a/apt-pkg/deb/dpkgpm.h b/apt-pkg/deb/dpkgpm.h index 2ff8a9ac7..0b181dc43 100644 --- a/apt-pkg/deb/dpkgpm.h +++ b/apt-pkg/deb/dpkgpm.h @@ -47,7 +47,7 @@ class pkgDPkgPM : public pkgPackageManager bool RunScripts(const char *Cnf); bool RunScriptsWithPkgs(const char *Cnf); bool SendV2Pkgs(FILE *F); - + // The Actuall installation implementation virtual bool Install(PkgIterator Pkg,string File); virtual bool Configure(PkgIterator Pkg); diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc index dd1c794c9..369eae70b 100644 --- a/apt-pkg/depcache.cc +++ b/apt-pkg/depcache.cc @@ -16,15 +16,52 @@ #include <apt-pkg/error.h> #include <apt-pkg/sptr.h> #include <apt-pkg/algorithms.h> - + +#include <apt-pkg/fileutl.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/tagfile.h> + +#include <iostream> +#include <sstream> +#include <set> + #include <apti18n.h> - /*}}}*/ + +pkgDepCache::ActionGroup::ActionGroup(pkgDepCache &cache) : + cache(cache), released(false) +{ + ++cache.group_level; +} + +void pkgDepCache::ActionGroup::release() +{ + if(!released) + { + if(cache.group_level == 0) + std::cerr << "W: Unbalanced action groups, expect badness" << std::endl; + else + { + --cache.group_level; + + if(cache.group_level == 0) + cache.MarkAndSweep(); + } + + released = false; + } +} + +pkgDepCache::ActionGroup::~ActionGroup() +{ + release(); +} // DepCache::pkgDepCache - Constructors /*{{{*/ // --------------------------------------------------------------------- /* */ pkgDepCache::pkgDepCache(pkgCache *pCache,Policy *Plcy) : - Cache(pCache), PkgState(0), DepState(0) + group_level(0), Cache(pCache), PkgState(0), DepState(0) { delLocalPolicy = 0; LocalPolicy = Plcy; @@ -47,6 +84,10 @@ pkgDepCache::~pkgDepCache() /* This allocats the extension buffers and initializes them. */ bool pkgDepCache::Init(OpProgress *Prog) { + // Suppress mark updates during this operation (just in case) and + // run a mark operation when Init terminates. + ActionGroup actions(*this); + delete [] PkgState; delete [] DepState; PkgState = new StateCache[Head().PackageCount]; @@ -72,7 +113,7 @@ bool pkgDepCache::Init(OpProgress *Prog) // Find the proper cache slot StateCache &State = PkgState[I->ID]; State.iFlags = 0; - + // Figure out the install version State.CandidateVer = GetCandidateVer(I); State.InstallVer = I.CurrentVer(); @@ -94,11 +135,130 @@ bool pkgDepCache::Init(OpProgress *Prog) if(Prog != 0) Prog->Done(); - + return true; } /*}}}*/ +bool pkgDepCache::readStateFile(OpProgress *Prog) +{ + FileFd state_file; + string state = _config->FindDir("Dir::State") + "extended_states"; + if(FileExists(state)) { + state_file.Open(state, FileFd::ReadOnly); + int file_size = state_file.Size(); + if(Prog != NULL) + Prog->OverallProgress(0, file_size, 1, + _("Reading state information")); + + pkgTagFile tagfile(&state_file); + pkgTagSection section; + int amt=0; + while(tagfile.Step(section)) { + string pkgname = section.FindS("Package"); + pkgCache::PkgIterator pkg=Cache->FindPkg(pkgname); + // Silently ignore unknown packages and packages with no actual + // version. + if(!pkg.end() && !pkg.VersionList().end()) { + short reason = section.FindI("Auto-Installed", 0); + if(reason > 0) + PkgState[pkg->ID].Flags |= Flag::Auto; + if(_config->FindB("Debug::pkgAutoRemove",false)) + std::cout << "Auto-Installed : " << pkgname << std::endl; + amt+=section.size(); + if(Prog != NULL) + Prog->OverallProgress(amt, file_size, 1, + _("Reading state information")); + } + if(Prog != NULL) + Prog->OverallProgress(file_size, file_size, 1, + _("Reading state information")); + } + } + + return true; +} + +bool pkgDepCache::writeStateFile(OpProgress *prog) +{ + if(_config->FindB("Debug::pkgAutoRemove",false)) + std::clog << "pkgDepCache::writeStateFile()" << std::endl; + + FileFd StateFile; + string state = _config->FindDir("Dir::State") + "extended_states"; + + // if it does not exist, create a empty one + if(!FileExists(state)) + { + StateFile.Open(state, FileFd::WriteEmpty); + StateFile.Close(); + } + + // open it + if(!StateFile.Open(state, FileFd::ReadOnly)) + return _error->Error(_("Failed to open StateFile %s"), + state.c_str()); + + FILE *OutFile; + string outfile = state + ".tmp"; + if((OutFile = fopen(outfile.c_str(),"w")) == NULL) + return _error->Error(_("Failed to write temporary StateFile %s"), + outfile.c_str()); + + // first merge with the existing sections + pkgTagFile tagfile(&StateFile); + pkgTagSection section; + std::set<string> pkgs_seen; + const char *nullreorderlist[] = {0}; + while(tagfile.Step(section)) { + string pkgname = section.FindS("Package"); + // Silently ignore unknown packages and packages with no actual + // version. + pkgCache::PkgIterator pkg=Cache->FindPkg(pkgname); + if(pkg.end() || pkg.VersionList().end()) + continue; + bool oldAuto = section.FindI("Auto-Installed"); + bool newAuto = (PkgState[pkg->ID].Flags & Flag::Auto); + if(_config->FindB("Debug::pkgAutoRemove",false)) + std::clog << "Update exisiting AutoInstall info: " + << pkg.Name() << std::endl; + TFRewriteData rewrite[2]; + rewrite[0].Tag = "Auto-Installed"; + rewrite[0].Rewrite = newAuto ? "1" : "0"; + rewrite[0].NewTag = 0; + rewrite[1].Tag = 0; + TFRewrite(OutFile, section, nullreorderlist, rewrite); + fprintf(OutFile,"\n"); + pkgs_seen.insert(pkgname); + } + + // then write the ones we have not seen yet + std::ostringstream ostr; + for(pkgCache::PkgIterator pkg=Cache->PkgBegin(); !pkg.end(); pkg++) { + if(PkgState[pkg->ID].Flags & Flag::Auto) { + if (pkgs_seen.find(pkg.Name()) != pkgs_seen.end()) { + if(_config->FindB("Debug::pkgAutoRemove",false)) + std::clog << "Skipping already written " << pkg.Name() << std::endl; + continue; + } + if(_config->FindB("Debug::pkgAutoRemove",false)) + std::clog << "Writing new AutoInstall: " + << pkg.Name() << std::endl; + ostr.str(string("")); + ostr << "Package: " << pkg.Name() + << "\nAuto-Installed: 1\n\n"; + fprintf(OutFile,ostr.str().c_str()); + fprintf(OutFile,"\n"); + } + } + fclose(OutFile); + + // move the outfile over the real file + rename(outfile.c_str(), state.c_str()); + + return true; +} + // DepCache::CheckDep - Checks a single dependency /*{{{*/ // --------------------------------------------------------------------- /* This first checks the dependency against the main target package and @@ -452,12 +612,14 @@ void pkgDepCache::Update(OpProgress *Prog) if (Prog != 0) Prog->Progress(Done); + + readStateFile(Prog); } /*}}}*/ // DepCache::Update - Update the deps list of a package /*{{{*/ // --------------------------------------------------------------------- /* This is a helper for update that only does the dep portion of the scan. - It is mainly ment to scan reverse dependencies. */ + It is mainly meant to scan reverse dependencies. */ void pkgDepCache::Update(DepIterator D) { // Update the reverse deps @@ -509,7 +671,7 @@ void pkgDepCache::Update(PkgIterator const &Pkg) // DepCache::MarkKeep - Put the package in the keep state /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgDepCache::MarkKeep(PkgIterator const &Pkg,bool Soft) +void pkgDepCache::MarkKeep(PkgIterator const &Pkg, bool Soft, bool FromUser) { // Simplifies other routines. if (Pkg.end() == true) @@ -521,6 +683,9 @@ void pkgDepCache::MarkKeep(PkgIterator const &Pkg,bool Soft) Pkg.CurrentVer().Downloadable() == false) return; + /** \todo Can this be moved later in the method? */ + ActionGroup group(*this); + /* We changed the soft state all the time so the UI is a bit nicer to use */ StateCache &P = PkgState[Pkg->ID]; @@ -537,7 +702,8 @@ void pkgDepCache::MarkKeep(PkgIterator const &Pkg,bool Soft) if (Pkg->VersionList == 0) return; - P.Flags &= ~Flag::Auto; + if(FromUser && !P.Marked) + P.Flags &= ~Flag::Auto; RemoveSizes(Pkg); RemoveStates(Pkg); @@ -563,6 +729,8 @@ void pkgDepCache::MarkDelete(PkgIterator const &Pkg, bool rPurge) if (Pkg.end() == true) return; + ActionGroup group(*this); + // Check that it is not already marked for delete StateCache &P = PkgState[Pkg->ID]; P.iFlags &= ~(AutoKept | Purge); @@ -585,7 +753,6 @@ void pkgDepCache::MarkDelete(PkgIterator const &Pkg, bool rPurge) else P.Mode = ModeDelete; P.InstallVer = 0; - P.Flags &= Flag::Auto; AddStates(Pkg); Update(Pkg); @@ -596,7 +763,7 @@ void pkgDepCache::MarkDelete(PkgIterator const &Pkg, bool rPurge) // --------------------------------------------------------------------- /* */ void pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, - unsigned long Depth) + unsigned long Depth, bool FromUser) { if (Depth > 100) return; @@ -605,6 +772,8 @@ void pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, if (Pkg.end() == true) return; + ActionGroup group(*this); + /* Check that it is not already marked for install and that it can be installed */ StateCache &P = PkgState[Pkg->ID]; @@ -613,7 +782,7 @@ void pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, P.CandidateVer == (Version *)Pkg.CurrentVer())) { if (P.CandidateVer == (Version *)Pkg.CurrentVer() && P.InstallVer == 0) - MarkKeep(Pkg); + MarkKeep(Pkg, false, FromUser); return; } @@ -633,7 +802,20 @@ void pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, P.Mode = ModeInstall; P.InstallVer = P.CandidateVer; - P.Flags &= ~Flag::Auto; + + if(FromUser) + { + // Set it to manual if it's a new install or cancelling the + // removal of a garbage package. + if(P.Status == 2 || (!Pkg.CurrentVer().end() && !P.Marked)) + P.Flags &= ~Flag::Auto; + } + else + { + // Set it to auto if this is a new install. + if(P.Status == 2) + P.Flags |= Flag::Auto; + } if (P.CandidateVer == (Version *)Pkg.CurrentVer()) P.Mode = ModeKeep; @@ -710,15 +892,14 @@ void pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, } } - if (InstPkg.end() == false) + if (InstPkg.end() == false) { - MarkInstall(InstPkg,true,Depth + 1); - - // Set the autoflag, after MarkInstall because MarkInstall unsets it - if (P->CurrentVer == 0) - PkgState[InstPkg->ID].Flags |= Flag::Auto; + if(_config->FindB("Debug::pkgDepCache::AutoInstall",false) == true) + std::clog << "Installing " << InstPkg.Name() + << " as dep of " << Pkg.Name() + << std::endl; + MarkInstall(InstPkg, true, Depth + 1, false); } - continue; } @@ -732,7 +913,6 @@ void pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, PkgIterator Pkg = Ver.ParentPkg(); MarkDelete(Pkg); - PkgState[Pkg->ID].Flags |= Flag::Auto; } continue; } @@ -744,6 +924,8 @@ void pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, /* */ void pkgDepCache::SetReInstall(PkgIterator const &Pkg,bool To) { + ActionGroup group(*this); + RemoveSizes(Pkg); RemoveStates(Pkg); @@ -762,9 +944,11 @@ void pkgDepCache::SetReInstall(PkgIterator const &Pkg,bool To) /* */ void pkgDepCache::SetCandidateVersion(VerIterator TargetVer) { + ActionGroup group(*this); + pkgCache::PkgIterator Pkg = TargetVer.ParentPkg(); StateCache &P = PkgState[Pkg->ID]; - + RemoveSizes(Pkg); RemoveStates(Pkg); @@ -777,6 +961,18 @@ void pkgDepCache::SetCandidateVersion(VerIterator TargetVer) Update(Pkg); AddSizes(Pkg); } + +void pkgDepCache::MarkAuto(const PkgIterator &Pkg, bool Auto) +{ + StateCache &state = PkgState[Pkg->ID]; + + ActionGroup group(*this); + + if(Auto) + state.Flags |= Flag::Auto; + else + state.Flags &= ~Flag::Auto; +} /*}}}*/ // StateCache::Update - Compute the various static display things /*{{{*/ // --------------------------------------------------------------------- @@ -867,3 +1063,216 @@ bool pkgDepCache::Policy::IsImportantDep(DepIterator Dep) return Dep.IsCritical(); } /*}}}*/ + +pkgDepCache::DefaultRootSetFunc::DefaultRootSetFunc() + : constructedSuccessfully(false) +{ + Configuration::Item const *Opts; + Opts = _config->Tree("APT::NeverAutoRemove"); + if (Opts != 0 && Opts->Child != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + + regex_t *p = new regex_t; + if(regcomp(p,Opts->Value.c_str(), + REG_EXTENDED | REG_ICASE | REG_NOSUB) != 0) + { + regfree(p); + delete p; + _error->Error("Regex compilation error for APT::NeverAutoRemove"); + return; + } + + rootSetRegexp.push_back(p); + } + } + + constructedSuccessfully = true; +} + +pkgDepCache::DefaultRootSetFunc::~DefaultRootSetFunc() +{ + for(unsigned int i = 0; i < rootSetRegexp.size(); i++) + { + regfree(rootSetRegexp[i]); + delete rootSetRegexp[i]; + } +} + + +bool pkgDepCache::DefaultRootSetFunc::InRootSet(const pkgCache::PkgIterator &pkg) +{ + for(unsigned int i = 0; i < rootSetRegexp.size(); i++) + if (regexec(rootSetRegexp[i], pkg.Name(), 0, 0, 0) == 0) + return true; + + return false; +} + +pkgDepCache::InRootSetFunc *pkgDepCache::GetRootSetFunc() +{ + DefaultRootSetFunc *f = new DefaultRootSetFunc; + if(f->wasConstructedSuccessfully()) + return f; + else + { + delete f; + return NULL; + } +} + +bool pkgDepCache::MarkFollowsRecommends() +{ + return _config->FindB("APT::AutoRemove::RecommendsImportant", true); +} + +bool pkgDepCache::MarkFollowsSuggests() +{ + return _config->FindB("APT::AutoRemove::SuggestsImportant", false); +} + +// the main mark algorithm +bool pkgDepCache::MarkRequired(InRootSetFunc &userFunc) +{ + bool follow_recommends; + bool follow_suggests; + + // init the states + for(PkgIterator p = PkgBegin(); !p.end(); ++p) + { + PkgState[p->ID].Marked = false; + PkgState[p->ID].Garbage = false; + + // debug output + if(_config->FindB("Debug::pkgAutoRemove",false) + && PkgState[p->ID].Flags & Flag::Auto) + std::clog << "AutoDep: " << p.Name() << std::endl; + } + + // init vars + follow_recommends = MarkFollowsRecommends(); + follow_suggests = MarkFollowsSuggests(); + + + + // do the mark part, this is the core bit of the algorithm + for(PkgIterator p = PkgBegin(); !p.end(); ++p) + { + if(!(PkgState[p->ID].Flags & Flag::Auto) || + (p->Flags & Flag::Essential) || + userFunc.InRootSet(p)) + + { + // the package is installed (and set to keep) + if(PkgState[p->ID].Keep() && !p.CurrentVer().end()) + MarkPackage(p, p.CurrentVer(), + follow_recommends, follow_suggests); + // the package is to be installed + else if(PkgState[p->ID].Install()) + MarkPackage(p, PkgState[p->ID].InstVerIter(*this), + follow_recommends, follow_suggests); + } + } + + return true; +} + +// mark a single package in Mark-and-Sweep +void pkgDepCache::MarkPackage(const pkgCache::PkgIterator &pkg, + const pkgCache::VerIterator &ver, + bool follow_recommends, + bool follow_suggests) +{ + pkgDepCache::StateCache &state = PkgState[pkg->ID]; + VerIterator candver = state.CandidateVerIter(*this); + VerIterator instver = state.InstVerIter(*this); + +#if 0 + // If a package was garbage-collected but is now being marked, we + // should re-select it + // For cases when a pkg is set to upgrade and this trigger the + // removal of a no-longer used dependency. if the pkg is set to + // keep again later it will result in broken deps + if(state.Delete() && state.RemoveReason = Unused) + { + if(ver==candver) + mark_install(pkg, false, false, NULL); + else if(ver==pkg.CurrentVer()) + MarkKeep(pkg, false, false); + + instver=state.InstVerIter(*this); + } +#endif + + // Ignore versions other than the InstVer, and ignore packages + // that are already going to be removed or just left uninstalled. + if(!(ver == instver && !instver.end())) + return; + + // if we are marked already we are done + if(state.Marked) + return; + + //std::cout << "Setting Marked for: " << pkg.Name() << std::endl; + state.Marked=true; + + if(!ver.end()) + { + for(DepIterator d = ver.DependsList(); !d.end(); ++d) + { + if(d->Type == Dep::Depends || + d->Type == Dep::PreDepends || + (follow_recommends && + d->Type == Dep::Recommends) || + (follow_suggests && + d->Type == Dep::Suggests)) + { + // Try all versions of this package. + for(VerIterator V = d.TargetPkg().VersionList(); + !V.end(); ++V) + { + if(_system->VS->CheckDep(V.VerStr(), d->CompareOp, d.TargetVer())) + { + MarkPackage(V.ParentPkg(), V, + follow_recommends, follow_suggests); + } + } + // Now try virtual packages + for(PrvIterator prv=d.TargetPkg().ProvidesList(); + !prv.end(); ++prv) + { + if(_system->VS->CheckDep(prv.ProvideVersion(), d->CompareOp, + d.TargetVer())) + { + MarkPackage(prv.OwnerPkg(), prv.OwnerVer(), + follow_recommends, follow_suggests); + } + } + } + } + } +} + +bool pkgDepCache::Sweep() +{ + // do the sweep + for(PkgIterator p=PkgBegin(); !p.end(); ++p) + { + StateCache &state=PkgState[p->ID]; + + // if it is not marked and it is installed, it's garbage + if(!state.Marked && (!p.CurrentVer().end() || state.Install()) && + !state.Delete()) + { + state.Garbage=true; + if(_config->FindB("Debug::pkgAutoRemove",false)) + std::cout << "Garbage: " << p.Name() << std::endl; + } + } + + return true; +} diff --git a/apt-pkg/depcache.h b/apt-pkg/depcache.h index 6d51920e9..fd935c268 100644 --- a/apt-pkg/depcache.h +++ b/apt-pkg/depcache.h @@ -1,4 +1,4 @@ -// -*- mode: cpp; mode: fold -*- +// -*- mode: c++; mode: fold -*- // Description /*{{{*/ // $Id: depcache.h,v 1.14 2001/02/20 07:03:17 jgg Exp $ /* ###################################################################### @@ -45,9 +45,71 @@ #include <apt-pkg/pkgcache.h> #include <apt-pkg/progress.h> +#include <regex.h> + +#include <vector> + class pkgDepCache : protected pkgCache::Namespace { public: + + /** \brief An arbitrary predicate on packages. */ + class InRootSetFunc + { + public: + virtual bool InRootSet(const pkgCache::PkgIterator &pkg) {return false;}; + virtual ~InRootSetFunc() {}; + }; + + private: + /** \brief Mark a single package and all its unmarked important + * dependencies during mark-and-sweep. + * + * Recursively invokes itself to mark all dependencies of the + * package. + * + * \param pkg The package to mark. + * + * \param ver The version of the package that is to be marked. + * + * \param follow_recommends If \b true, recommendations of the + * package will be recursively marked. + * + * \param follow_suggests If \b true, suggestions of the package + * will be recursively marked. + */ + void MarkPackage(const pkgCache::PkgIterator &pkg, + const pkgCache::VerIterator &ver, + bool follow_recommends, + bool follow_suggests); + + /** \brief Update the Marked field of all packages. + * + * Each package's StateCache::Marked field will be set to \b true + * if and only if it can be reached from the root set. By + * default, the root set consists of the set of manually installed + * or essential packages, but it can be extended using the + * parameter #rootFunc. + * + * \param rootFunc A callback that can be used to add extra + * packages to the root set. + * + * \return \b false if an error occured. + */ + bool MarkRequired(InRootSetFunc &rootFunc); + + /** \brief Set the StateCache::Garbage flag on all packages that + * should be removed. + * + * Packages that were not marked by the last call to #MarkRequired + * are tested to see whether they are actually garbage. If so, + * they are marked as such. + * + * \return \b false if an error occured. + */ + bool Sweep(); + + public: // These flags are used in DepState enum DepFlags {DepNow = (1 << 0),DepInstall = (1 << 1),DepCVer = (1 << 2), @@ -63,6 +125,84 @@ class pkgDepCache : protected pkgCache::Namespace enum VersionTypes {NowVersion, InstallVersion, CandidateVersion}; enum ModeList {ModeDelete = 0, ModeKeep = 1, ModeInstall = 2}; + + /** \brief Represents an active action group. + * + * An action group is a group of actions that are currently being + * performed. While an active group is active, certain routine + * clean-up actions that would normally be performed after every + * cache operation are delayed until the action group is + * completed. This is necessary primarily to avoid inefficiencies + * when modifying a large number of packages at once. + * + * This class represents an active action group. Creating an + * instance will create an action group; destroying one will + * destroy the corresponding action group. + * + * The following operations are suppressed by this class: + * + * - Keeping the Marked and Garbage flags up to date. + * + * \note This can be used in the future to easily accumulate + * atomic actions for undo or to display "what apt did anyway"; + * e.g., change the counter of how many action groups are active + * to a std::set of pointers to them and use those to store + * information about what happened in a group in the group. + */ + class ActionGroup + { + pkgDepCache &cache; + + bool released; + + /** Action groups are noncopyable. */ + ActionGroup(const ActionGroup &other); + public: + /** \brief Create a new ActionGroup. + * + * \param cache The cache that this ActionGroup should + * manipulate. + * + * As long as this object exists, no automatic cleanup + * operations will be undertaken. + */ + ActionGroup(pkgDepCache &cache); + + /** \brief Clean up the action group before it is destroyed. + * + * If it is destroyed later, no second cleanup wil be run. + */ + void release(); + + /** \brief Destroy the action group. + * + * If this is the last action group, the automatic cache + * cleanup operations will be undertaken. + */ + ~ActionGroup(); + }; + + /** \brief Returns \b true for packages matching a regular + * expression in APT::NeverAutoRemove. + */ + class DefaultRootSetFunc : public InRootSetFunc + { + std::vector<regex_t *> rootSetRegexp; + bool constructedSuccessfully; + + public: + DefaultRootSetFunc(); + ~DefaultRootSetFunc(); + + /** \return \b true if the class initialized successfully, \b + * false otherwise. Used to avoid throwing an exception, since + * APT classes generally don't. + */ + bool wasConstructedSuccessfully() const { return constructedSuccessfully; } + + bool InRootSet(const pkgCache::PkgIterator &pkg); + }; + struct StateCache { // Epoch stripped text versions of the two version fields @@ -79,6 +219,17 @@ class pkgDepCache : protected pkgCache::Namespace unsigned short Flags; unsigned short iFlags; // Internal flags + /** \brief \b true if this package can be reached from the root set. */ + bool Marked; + + /** \brief \b true if this package is unused and should be removed. + * + * This differs from !#Marked, because it is possible that some + * unreachable packages will be protected from becoming + * garbage. + */ + bool Garbage; + // Various tree indicators signed char Status; // -1,0,1,2 unsigned char Mode; // ModeList @@ -119,6 +270,14 @@ class pkgDepCache : protected pkgCache::Namespace virtual ~Policy() {}; }; + + private: + /** The number of open "action groups"; certain post-action + * operations are suppressed if this number is > 0. + */ + int group_level; + + friend class ActionGroup; protected: @@ -182,16 +341,68 @@ class pkgDepCache : protected pkgCache::Namespace inline StateCache &operator [](PkgIterator const &I) {return PkgState[I->ID];}; inline unsigned char &operator [](DepIterator const &I) {return DepState[I->ID];}; - // Manipulators - void MarkKeep(PkgIterator const &Pkg,bool Soft = false); + /** \return A function identifying packages in the root set other + * than manually installed packages and essential packages, or \b + * NULL if an error occurs. + * + * \todo Is this the best place for this function? Perhaps the + * settings for mark-and-sweep should be stored in a single + * external class? + */ + virtual InRootSetFunc *GetRootSetFunc(); + + /** \return \b true if the garbage collector should follow recommendations. + */ + virtual bool MarkFollowsRecommends(); + + /** \return \b true if the garbage collector should follow suggestions. + */ + virtual bool MarkFollowsSuggests(); + + /** \brief Update the Marked and Garbage fields of all packages. + * + * This routine is implicitly invoked after all state manipulators + * and when an ActionGroup is destroyed. It invokes #MarkRequired + * and #Sweep to do its dirty work. + * + * \param rootFunc A predicate that returns \b true for packages + * that should be added to the root set. + */ + bool MarkAndSweep(InRootSetFunc &rootFunc) + { + return MarkRequired(rootFunc) && Sweep(); + } + + bool MarkAndSweep() + { + std::auto_ptr<InRootSetFunc> f(GetRootSetFunc()); + if(f.get() != NULL) + return MarkAndSweep(*f.get()); + else + return false; + } + + /** \name State Manipulators + */ + // @{ + void MarkKeep(PkgIterator const &Pkg, bool Soft = false, + bool FromUser = true); void MarkDelete(PkgIterator const &Pkg,bool Purge = false); void MarkInstall(PkgIterator const &Pkg,bool AutoInst = true, - unsigned long Depth = 0); + unsigned long Depth = 0, bool FromUser = true); void SetReInstall(PkgIterator const &Pkg,bool To); void SetCandidateVersion(VerIterator TargetVer); + + /** Set the "is automatically installed" flag of Pkg. */ + void MarkAuto(const PkgIterator &Pkg, bool Auto); + // @} // This is for debuging void Update(OpProgress *Prog = 0); + + // read persistent states + bool readStateFile(OpProgress *prog); + bool writeStateFile(OpProgress *prog); // Size queries inline double UsrSize() {return iUsrSize;}; diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index 6118845e8..6aa486a7f 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: init.cc,v 1.21 2004/02/27 00:46:44 mdz Exp $ +// $Id: init.cc,v 1.20 2003/02/09 20:31:05 doogie Exp $ /* ###################################################################### Init - Initialize the package library @@ -64,13 +64,14 @@ bool pkgInitConfig(Configuration &Cnf) // Configuration Cnf.Set("Dir::Etc","etc/apt/"); Cnf.Set("Dir::Etc::sourcelist","sources.list"); + Cnf.Set("Dir::Etc::sourceparts","sources.list.d"); Cnf.Set("Dir::Etc::vendorlist","vendors.list"); Cnf.Set("Dir::Etc::vendorparts","vendors.list.d"); Cnf.Set("Dir::Etc::main","apt.conf"); Cnf.Set("Dir::Etc::parts","apt.conf.d"); Cnf.Set("Dir::Etc::preferences","preferences"); Cnf.Set("Dir::Bin::methods","/usr/lib/apt/methods"); - + bool Res = true; // Read an alternate config file diff --git a/apt-pkg/init.h b/apt-pkg/init.h index e21351797..51a7ba2eb 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -17,8 +17,8 @@ #include <apt-pkg/pkgsystem.h> // See the makefile -#define APT_PKG_MAJOR 3 -#define APT_PKG_MINOR 10 +#define APT_PKG_MAJOR 4 +#define APT_PKG_MINOR 1 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 0e6aecc65..29c8ee135 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=3.11 +MAJOR=4.1 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) APT_DOMAIN:=libapt-pkg$(MAJOR) @@ -21,11 +21,11 @@ APT_DOMAIN:=libapt-pkg$(MAJOR) # Source code for the contributed non-core things SOURCE = contrib/mmap.cc contrib/error.cc contrib/strutl.cc \ contrib/configuration.cc contrib/progress.cc contrib/cmndline.cc \ - contrib/md5.cc contrib/sha1.cc contrib/hashes.cc \ + contrib/md5.cc contrib/sha1.cc contrib/sha256.cc contrib/hashes.cc \ contrib/cdromutl.cc contrib/crc-16.cc \ contrib/fileutl.cc HEADERS = mmap.h error.h configuration.h fileutl.h cmndline.h \ - md5.h crc-16.h cdromutl.h strutl.h sptr.h sha1.h hashes.h + md5.h crc-16.h cdromutl.h strutl.h sptr.h sha1.h sha256.h hashes.h # Source code for the core main library SOURCE+= pkgcache.cc version.cc depcache.cc \ diff --git a/apt-pkg/packagemanager.cc b/apt-pkg/packagemanager.cc index 155408bb4..b0dd43629 100644 --- a/apt-pkg/packagemanager.cc +++ b/apt-pkg/packagemanager.cc @@ -106,7 +106,7 @@ bool pkgPackageManager::FixMissing() // Okay, this file is missing and we need it. Mark it for keep Bad = true; - Cache.MarkKeep(I); + Cache.MarkKeep(I, false, false); } // We have to empty the list otherwise it will not have the new changes @@ -593,7 +593,7 @@ pkgPackageManager::OrderResult pkgPackageManager::OrderInstall() Pkg.State() == pkgCache::PkgIterator::NeedsNothing && (Cache[Pkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) { - _error->Error("Internal Error, trying to manipulate a kept package"); + _error->Error("Internal Error, trying to manipulate a kept package (%s)",Pkg.Name()); return Failed; } @@ -631,12 +631,11 @@ pkgPackageManager::OrderResult pkgPackageManager::OrderInstall() // --------------------------------------------------------------------- /* This uses the filenames in FileNames and the information in the DepCache to perform the installation of packages.*/ -pkgPackageManager::OrderResult pkgPackageManager::DoInstall(int status_fd) +pkgPackageManager::OrderResult pkgPackageManager::DoInstall(int statusFd) { - OrderResult Res = OrderInstall(); - if (Res != Failed) - if (Go(status_fd) == false) - return Failed; - return Res; + if(DoInstallPreFork() == Failed) + return Failed; + + return DoInstallPostFork(statusFd); } /*}}}*/ diff --git a/apt-pkg/packagemanager.h b/apt-pkg/packagemanager.h index f64637d03..48f53576c 100644 --- a/apt-pkg/packagemanager.h +++ b/apt-pkg/packagemanager.h @@ -28,7 +28,9 @@ #endif #include <string> +#include <iostream> #include <apt-pkg/pkgcache.h> +#include <apt-pkg/depcache.h> using std::string; @@ -70,13 +72,39 @@ class pkgPackageManager : protected pkgCache::Namespace virtual bool Remove(PkgIterator /*Pkg*/,bool /*Purge*/=false) {return false;}; virtual bool Go(int statusFd=-1) {return true;}; virtual void Reset() {}; - + + // the result of the operation + OrderResult Res; + public: // Main action members bool GetArchives(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs); - OrderResult DoInstall(int statusFd=-1); + + // Do the installation + OrderResult DoInstall(int statusFd=-1); + + // stuff that needs to be done before the fork() of a library that + // uses apt + OrderResult DoInstallPreFork() { + Res = OrderInstall(); + return Res; + }; + + // stuff that needs to be done after the fork + OrderResult DoInstallPostFork(int statusFd=-1) { + bool goResult = Go(statusFd); + if(goResult == false) + return Failed; + + // if all was fine update the state file + if(Res == Completed) { + Cache.writeStateFile(NULL); + } + return Res; + }; + bool FixMissing(); pkgPackageManager(pkgDepCache *Cache); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 4452079a2..162ab4f27 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -164,7 +164,7 @@ bool pkgCache::ReMap() /* This is used to generate the hash entries for the HashTable. With my package list from bo this function gets 94% table usage on a 512 item table (480 used items) */ -unsigned long pkgCache::sHash(string Str) const +unsigned long pkgCache::sHash(const string &Str) const { unsigned long Hash = 0; for (string::const_iterator I = Str.begin(); I != Str.end(); I++) @@ -184,7 +184,7 @@ unsigned long pkgCache::sHash(const char *Str) const // Cache::FindPkg - Locate a package by name /*{{{*/ // --------------------------------------------------------------------- /* Returns 0 on error, pointer to the package otherwise */ -pkgCache::PkgIterator pkgCache::FindPkg(string Name) +pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) { // Look at the hash bucket Package *Pkg = PkgP + HeaderP->HashTable[Hash(Name)]; diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 6a54ad5ba..c7a3172cc 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -95,7 +95,7 @@ class pkgCache string CacheFile; MMap ⤅ - unsigned long sHash(string S) const; + unsigned long sHash(const string &S) const; unsigned long sHash(const char *S) const; public: @@ -119,14 +119,14 @@ class pkgCache inline void *DataEnd() {return ((unsigned char *)Map.Data()) + Map.Size();}; // String hashing function (512 range) - inline unsigned long Hash(string S) const {return sHash(S);}; + inline unsigned long Hash(const string &S) const {return sHash(S);}; inline unsigned long Hash(const char *S) const {return sHash(S);}; // Usefull transformation things const char *Priority(unsigned char Priority); // Accessors - PkgIterator FindPkg(string Name); + PkgIterator FindPkg(const string &Name); Header &Head() {return *HeaderP;}; inline PkgIterator PkgBegin(); inline PkgIterator PkgEnd(); diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 1ba791b45..3f02725c1 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -26,6 +26,8 @@ #include <apt-pkg/sptr.h> #include <apt-pkg/pkgsystem.h> +#include <apt-pkg/tagfile.h> + #include <apti18n.h> #include <vector> @@ -315,7 +317,7 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) // CacheGenerator::NewPackage - Add a new package /*{{{*/ // --------------------------------------------------------------------- /* This creates a new package structure and adds it to the hash table */ -bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name) +bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name) { Pkg = Cache.FindPkg(Name); if (Pkg.end() == false) @@ -379,7 +381,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, // --------------------------------------------------------------------- /* This puts a version structure in the linked list */ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, - string VerStr, + const string &VerStr, unsigned long Next) { // Get a structure @@ -459,8 +461,8 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, /* This creates a dependency element in the tree. It is linked to the version and to the package that it is pointing to. */ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, - string PackageName, - string Version, + const string &PackageName, + const string &Version, unsigned int Op, unsigned int Type) { @@ -524,8 +526,8 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, // --------------------------------------------------------------------- /* */ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, - string PackageName, - string Version) + const string &PackageName, + const string &Version) { pkgCache &Cache = Owner->Cache; @@ -564,7 +566,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, // --------------------------------------------------------------------- /* This is used to select which file is to be associated with all newly added versions. The caller is responsible for setting the IMS fields. */ -bool pkgCacheGenerator::SelectFile(string File,string Site, +bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, const pkgIndexFile &Index, unsigned long Flags) { @@ -648,7 +650,7 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, /* This just verifies that each file in the list of index files exists, has matching attributes with the cache and the cache does not have any extra files. */ -static bool CheckValidity(string CacheFile, FileIterator Start, +static bool CheckValidity(const string &CacheFile, FileIterator Start, FileIterator End,MMap **OutMap = 0) { // No file, certainly invalid diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 6ab8594d9..fae1a60a6 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -54,19 +54,19 @@ class pkgCacheGenerator // Flag file dependencies bool FoundFileDeps; - bool NewPackage(pkgCache::PkgIterator &Pkg,string Pkg); + bool NewPackage(pkgCache::PkgIterator &Pkg,const string &Pkg); bool NewFileVer(pkgCache::VerIterator &Ver,ListParser &List); bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); - unsigned long NewVersion(pkgCache::VerIterator &Ver,string VerStr,unsigned long Next); + unsigned long NewVersion(pkgCache::VerIterator &Ver,const string &VerStr,unsigned long Next); map_ptrloc NewDescription(pkgCache::DescIterator &Desc,const string &Lang,const MD5SumValue &md5sum,map_ptrloc Next); public: unsigned long WriteUniqString(const char *S,unsigned int Size); - inline unsigned long WriteUniqString(string S) {return WriteUniqString(S.c_str(),S.length());}; + inline unsigned long WriteUniqString(const string &S) {return WriteUniqString(S.c_str(),S.length());}; void DropProgress() {Progress = 0;}; - bool SelectFile(string File,string Site,pkgIndexFile const &Index, + bool SelectFile(const string &File,const string &Site,pkgIndexFile const &Index, unsigned long Flags = 0); bool MergeList(ListParser &List,pkgCache::VerIterator *Ver = 0); inline pkgCache &GetCache() {return Cache;}; @@ -97,12 +97,13 @@ class pkgCacheGenerator::ListParser inline unsigned long WriteUniqString(string S) {return Owner->WriteUniqString(S);}; inline unsigned long WriteUniqString(const char *S,unsigned int Size) {return Owner->WriteUniqString(S,Size);}; - inline unsigned long WriteString(string S) {return Owner->Map.WriteString(S);}; + inline unsigned long WriteString(const string &S) {return Owner->Map.WriteString(S);}; inline unsigned long WriteString(const char *S,unsigned int Size) {return Owner->Map.WriteString(S,Size);}; - bool NewDepends(pkgCache::VerIterator Ver,string Package, - string Version,unsigned int Op, + bool NewDepends(pkgCache::VerIterator Ver,const string &Package, + const string &Version,unsigned int Op, unsigned int Type); - bool NewProvides(pkgCache::VerIterator Ver,string Package,string Version); + bool NewProvides(pkgCache::VerIterator Ver,const string &Package, + const string &Version); public: diff --git a/apt-pkg/pkgrecords.cc b/apt-pkg/pkgrecords.cc index f62f945b5..b22f3e73f 100644 --- a/apt-pkg/pkgrecords.cc +++ b/apt-pkg/pkgrecords.cc @@ -42,9 +42,6 @@ pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), Files(0) if (Files[I->ID] == 0) return; } - // We store that to make sure that the destructor won't segfault, - // even if the Cache object was destructed before this instance. - PackageFileCount = Cache.HeaderP->PackageFileCount; } /*}}}*/ // Records::~pkgRecords - Destructor /*{{{*/ @@ -52,7 +49,7 @@ pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), Files(0) /* */ pkgRecords::~pkgRecords() { - for (unsigned I = 0; I != PackageFileCount; I++) + for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++) delete Files[I]; delete [] Files; } diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h index ece91680e..31c444dbf 100644 --- a/apt-pkg/pkgrecords.h +++ b/apt-pkg/pkgrecords.h @@ -33,7 +33,6 @@ class pkgRecords pkgCache &Cache; Parser **Files; - int PackageFileCount; public: diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 95aba0cb5..e3b4d94f8 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: sourcelist.cc,v 1.25 2004/06/07 23:08:00 mdz Exp $ +// $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $ /* ###################################################################### List of Sources @@ -21,6 +21,13 @@ #include <apti18n.h> #include <fstream> + +// CNC:2003-03-03 - This is needed for ReadDir stuff. +#include <algorithm> +#include <stdio.h> +#include <dirent.h> +#include <sys/stat.h> +#include <unistd.h> /*}}}*/ using namespace std; @@ -142,23 +149,66 @@ pkgSourceList::~pkgSourceList() /* */ bool pkgSourceList::ReadMainList() { - return Read(_config->FindFile("Dir::Etc::sourcelist")); + // CNC:2003-03-03 - Multiple sources list support. + bool Res = true; +#if 0 + Res = ReadVendors(); + if (Res == false) + return false; +#endif + + Reset(); + // CNC:2003-11-28 - Entries in sources.list have priority over + // entries in sources.list.d. + string Main = _config->FindFile("Dir::Etc::sourcelist"); + if (FileExists(Main) == true) + Res &= ReadAppend(Main); + + string Parts = _config->FindDir("Dir::Etc::sourceparts"); + if (FileExists(Parts) == true) + Res &= ReadSourceDir(Parts); + + return Res; } /*}}}*/ +// CNC:2003-03-03 - Needed to preserve backwards compatibility. +// SourceList::Reset - Clear the sourcelist contents /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgSourceList::Reset() +{ + for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++) + delete *I; + SrcList.erase(SrcList.begin(),SrcList.end()); +} + /*}}}*/ +// CNC:2003-03-03 - Function moved to ReadAppend() and Reset(). // SourceList::Read - Parse the sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ bool pkgSourceList::Read(string File) { + Reset(); + return ReadAppend(File); +} + /*}}}*/ +// SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ReadAppend(string File) +{ // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str()); +#if 0 // Now Reset() does this. for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++) delete *I; SrcList.erase(SrcList.begin(),SrcList.end()); - char Buffer[300]; +#endif + // CNC:2003-12-10 - 300 is too short. + char Buffer[1024]; int CurLine = 0; while (F.eof() == false) @@ -172,7 +222,10 @@ bool pkgSourceList::Read(string File) char *I; - for (I = Buffer; *I != 0 && *I != '#'; I++); + // CNC:2003-02-20 - Do not break if '#' is inside []. + for (I = Buffer; *I != 0 && *I != '#'; I++) + if (*I == '[') + for (I++; *I != 0 && *I != ']'; I++); *I = 0; const char *C = _strstrip(Buffer); @@ -259,3 +312,55 @@ bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const return true; } /*}}}*/ +// CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>. +// SourceList::ReadSourceDir - Read a directory with sources files +// Based on ReadConfigDir() /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ReadSourceDir(string Dir) +{ + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + vector<string> List; + + for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) + { + if (Ent->d_name[0] == '.') + continue; + + // CNC:2003-12-02 Only accept .list files as valid sourceparts + if (flExtension(Ent->d_name) != "list") + continue; + + // Skip bad file names ala run-parts + const char *C = Ent->d_name; + for (; *C != 0; C++) + if (isalpha(*C) == 0 && isdigit(*C) == 0 + && *C != '_' && *C != '-' && *C != '.') + break; + if (*C != 0) + continue; + + // Make sure it is a file and not something else + string File = flCombine(Dir,Ent->d_name); + struct stat St; + if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0) + continue; + + List.push_back(File); + } + closedir(D); + + sort(List.begin(),List.end()); + + // Read the files + for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++) + if (ReadAppend(*I) == false) + return false; + return true; + +} + /*}}}*/ + diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 5d8427017..123ae6984 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -77,6 +77,11 @@ class pkgSourceList bool ReadMainList(); bool Read(string File); + + // CNC:2003-03-03 + void Reset(); + bool ReadAppend(string File); + bool ReadSourceDir(string Dir); // List accessors inline const_iterator begin() const {return SrcList.begin();}; diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index cae0fa819..25e2930fa 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -35,20 +35,20 @@ pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long Size) : Fd(*pFd), Size(Size) { - if (Fd.IsOpen() == false) + if (Fd.IsOpen() == false || Fd.Size() == 0) { Buffer = 0; Start = End = Buffer = 0; - Done = true; iOffset = 0; + Map = NULL; return; } - Buffer = new char[Size]; - Start = End = Buffer; - Done = false; + Map = new MMap (Fd, MMap::Public | MMap::ReadOnly); + Buffer = (char *) Map->Data (); + Start = Buffer; + End = Buffer + Map->Size (); iOffset = 0; - Fill(); } /*}}}*/ // TagFile::~pkgTagFile - Destructor /*{{{*/ @@ -56,7 +56,7 @@ pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long Size) : /* */ pkgTagFile::~pkgTagFile() { - delete [] Buffer; + delete Map; } /*}}}*/ // TagFile::Step - Advance to the next section /*{{{*/ @@ -64,14 +64,13 @@ pkgTagFile::~pkgTagFile() /* If the Section Scanner fails we refill the buffer and try again. */ bool pkgTagFile::Step(pkgTagSection &Tag) { + if (Start == End) + return false; + if (Tag.Scan(Start,End - Start) == false) { - if (Fill() == false) - return false; - - if (Tag.Scan(Start,End - Start) == false) - return _error->Error(_("Unable to parse package file %s (1)"), - Fd.Name().c_str()); + return _error->Error(_("Unable to parse package file %s (1)"), + Fd.Name().c_str()); } Start += Tag.size(); iOffset += Tag.size(); @@ -80,50 +79,6 @@ bool pkgTagFile::Step(pkgTagSection &Tag) return true; } /*}}}*/ -// TagFile::Fill - Top up the buffer /*{{{*/ -// --------------------------------------------------------------------- -/* This takes the bit at the end of the buffer and puts it at the start - then fills the rest from the file */ -bool pkgTagFile::Fill() -{ - unsigned long EndSize = End - Start; - unsigned long Actual = 0; - - memmove(Buffer,Start,EndSize); - Start = Buffer; - End = Buffer + EndSize; - - if (Done == false) - { - // See if only a bit of the file is left - if (Fd.Read(End,Size - (End - Buffer),&Actual) == false) - return false; - if (Actual != Size - (End - Buffer)) - Done = true; - End += Actual; - } - - if (Done == true) - { - if (EndSize <= 3 && Actual == 0) - return false; - if (Size - (End - Buffer) < 4) - return true; - - // Append a double new line if one does not exist - unsigned int LineCount = 0; - for (const char *E = End - 1; E - End < 6 && (*E == '\n' || *E == '\r'); E--) - if (*E == '\n') - LineCount++; - for (; LineCount < 2; LineCount++) - *End++ = '\n'; - - return true; - } - - return true; -} - /*}}}*/ // TagFile::Jump - Jump to a pre-recorded location in the file /*{{{*/ // --------------------------------------------------------------------- /* This jumps to a pre-recorded file location and reads the record @@ -141,22 +96,10 @@ bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long Offset) // Reposition and reload.. iOffset = Offset; - Done = false; - if (Fd.Seek(Offset) == false) - return false; - End = Start = Buffer; - - if (Fill() == false) - return false; - - if (Tag.Scan(Start,End - Start) == true) - return true; - - // This appends a double new line (for the real eof handling) - if (Fill() == false) - return false; + Start = Buffer + iOffset; - if (Tag.Scan(Start,End - Start) == false) + // Start != End is a special case to not fail on empty TagFiles + if (Start != End && Tag.Scan(Start,End - Start) == false) return _error->Error(_("Unable to parse package file %s (2)"),Fd.Name().c_str()); return true; @@ -181,7 +124,7 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength) Stop = Section = Start; memset(AlphaIndexes,0,sizeof(AlphaIndexes)); - if (Stop == 0) + if (Stop == 0 || MaxLength == 0) return false; TagCount = 0; @@ -212,6 +155,12 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength) Stop++; } + if ((Stop+1 >= End) && (End[-1] == '\n' || End[-1] == '\r')) + { + Indexes[TagCount] = (End - 1) - Section; + return true; + } + return false; } /*}}}*/ @@ -394,7 +343,8 @@ static const char *iTFRewritePackageOrder[] = { "Filename", "Size", "MD5Sum", - "SHA1Sum", + "SHA1", + "SHA256", "MSDOS-Filename", // Obsolete "Description", 0}; diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index 8c948754d..5cff2681c 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -25,6 +25,7 @@ #endif #include <apt-pkg/fileutl.h> +#include <apt-pkg/mmap.h> #include <stdio.h> class pkgTagSection @@ -69,15 +70,13 @@ class pkgTagSection class pkgTagFile { FileFd &Fd; + MMap *Map; char *Buffer; char *Start; char *End; - bool Done; unsigned long iOffset; unsigned long Size; - bool Fill(); - public: bool Step(pkgTagSection &Section); |