From 9995c4a0bc2d8567ff9f11d338ad610754682372 Mon Sep 17 00:00:00 2001 From: Ishan Jayawardena Date: Mon, 13 Jun 2011 15:16:14 +0530 Subject: added support for downloading debdelta files and queuing successfully downloaded debdeltas for processing with the debdelta apt method, in apt-pkg/acquire-item.{cc,h} in pkgAcqArchive class. Currently, no index for deltas are used. Proper progress reporting and methods to verify the debdeltas are yet to be added. --- apt-pkg/acquire-item.cc | 323 +++++++++++++++++++++++++++++++++++++-------- apt-pkg/acquire-item.h | 104 +++++++++++++++ apt-pkg/acquire-method.cc | 22 ++- apt-pkg/acquire-worker.cc | 37 +++++- apt-pkg/acquire.cc | 19 ++- apt-pkg/algorithms.cc | 12 +- apt-pkg/cachefile.cc | 86 ++++++------ apt-pkg/contrib/fileutl.cc | 23 +++- apt-pkg/contrib/fileutl.h | 1 + apt-pkg/contrib/strutl.cc | 6 +- apt-pkg/init.cc | 4 +- apt-pkg/packagemanager.cc | 6 +- apt-pkg/sourcelist.cc | 9 +- 13 files changed, 521 insertions(+), 131 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c3817f6ee..42e517b8e 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -271,14 +271,6 @@ void pkgAcqSubIndex::Done(string Message,unsigned long Size,string Md5Hash, /*{{ string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); - /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */ - indexRecords SubIndexParser; - if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) { - Status = StatError; - ErrorText = SubIndexParser.ErrorText; - return; - } - // sucess in downloading the index // rename the index if(Debug) @@ -462,7 +454,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ found=true; else if (found == false) continue; - if(Debug) std::clog << "Need to get diff: " << d.file << std::endl; available_patches.push_back(d); @@ -587,7 +578,6 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), available_patches(diffs), ServerSha1(ServerSha1) { - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -704,7 +694,6 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ if(Debug) std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; - QueueURI(Desc); return true; @@ -720,14 +709,13 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, /* string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); - + // sucess in downloading a diff, enter ApplyDiff state if(State == StateFetchDiff) { - // rred excepts the patch as $FinalFile.ed Rename(DestFile,FinalFile+".ed"); - + std::clog << " Sending to rred method FinalFile: " << FinalFile << std::endl; if(Debug) std::clog << "Sending to rred method: " << FinalFile << std::endl; @@ -739,7 +727,6 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, /* return; } - // success in download/apply a diff, queue next (if needed) if(State == StateApplyDiff) { @@ -902,30 +889,6 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, ReportMirrorFailure("HashChecksumFailure"); return; } - - /* Verify the index file for correctness (all indexes must - * have a Package field) (LP: #346386) (Closes: #627642) */ - { - FileFd fd(DestFile, FileFd::ReadOnly); - pkgTagSection sec; - pkgTagFile tag(&fd); - - // Only test for correctness if the file is not empty (empty is ok) - if (fd.Size() > 0) { - if (_error->PendingError() || !tag.Step(sec)) { - Status = StatError; - _error->DumpErrors(); - Rename(DestFile,DestFile + ".FAILED"); - return; - } else if (!sec.Exists("Package")) { - Status = StatError; - ErrorText = ("Encountered a section with no Package: header"); - Rename(DestFile,DestFile + ".FAILED"); - return; - } - } - } - // Done, move it into position string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); @@ -1362,16 +1325,6 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ /*}}}*/ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { -#if 0 - /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642) - * FIXME: Disabled; it breaks unsigned repositories without hashes */ - if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile)) - { - Status = StatError; - ErrorText = MetaIndexParser->ErrorText; - return; - } -#endif for (vector ::const_iterator Target = IndexTargets->begin(); Target != IndexTargets->end(); Target++) @@ -1535,12 +1488,6 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) LookupTag(Message,"Message").c_str()); RunScripts("APT::Update::Auth-Failure"); return; - } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { - /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ - _error->Error(_("GPG error: %s: %s"), - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - return; } else { _error->Warning(_("GPG error: %s: %s"), Desc.Description.c_str(), @@ -2065,6 +2012,272 @@ string pkgAcqFile::Custom600Headers() return ""; } /*}}}*/ + +pkgAcqDebdelta::pkgAcqDebdelta(pkgAcquire *Owner,pkgSourceList *Sources, + pkgRecords *Recs,pkgCache::VerIterator const &Version, + string &StoreFilename) : + Item(Owner), Version(Version), Sources(Sources), Recs(Recs), + StoreFilename(StoreFilename), Vf(Version.FileList()), + Trusted(false) +{ + DebdeltaName = string(Version.ParentPkg().Name()) + "_" + + string(Version.ParentPkg().CurVersion()) + "_" + + string(Version.ParentPkg().CandVersion()) + "_" + + string(Version.Arch()) + ".debdelta"; + Retries = _config->FindI("Acquire::Retries",0); + + if (Version.Arch() == 0) + { + _error->Error(_("I wasn't able to locate a file for the %s package. " + "This might mean you need to manually fix this package. " + "(due to missing arch)"), + Version.ParentPkg().Name()); + return; + } + + /* We need to find a filename to determine the extension. We make the + assumption here that all the available sources for this version share + the same extension.. */ + // Skip not source sources, they do not have file fields. + for (; Vf.end() == false; Vf++) + { + if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0) + continue; + break; + } + + // Does not really matter here.. we are going to fail out below + if (Vf.end() != true) + { + // If this fails to get a file name we will bomb out below. + pkgRecords::Parser &Parse = Recs->Lookup(Vf); + if (_error->PendingError() == true) + return; + + DebdeltaName = StoreFilename = QuoteString(DebdeltaName, ":"); + DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename); + } + + // check if we have one trusted source for the package. if so, switch + // to "TrustedOnly" mode + for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++) + { + pkgIndexFile *Index; + if (Sources->FindIndex(i.File(),Index) == false) + continue; + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + { + std::cerr << "Checking index: " << Index->Describe() + << "(Trusted=" << Index->IsTrusted() << ")\n"; + } + if (Index->IsTrusted()) { + Trusted = true; + break; + } + } + + // "allow-unauthenticated" restores apts old fetching behaviour + // that means that e.g. unauthenticated file:// uris are higher + // priority than authenticated http:// uris + if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true) + Trusted = false; + std::cerr << "\n[Debdelta] pkgAcqDebdelta::pkgAcqDebdelta()" << std::endl; + std::cerr << " DebdeltaName : " << DebdeltaName << std::endl; + std::cerr << " StoreFilename: " << StoreFilename << std::endl; + std::cerr << " DestFile : " << DestFile << std::endl; + DebdeltaStatus = Fetching; + // Select a source + if (QueueNext() == false && _error->PendingError() == false) + _error->Error(_("I wasn't able to locate a file for the %s package. " + "This might mean you need to manually fix this package."), + Version.ParentPkg().Name()); +} + +bool pkgAcqDebdelta::QueueNext() +{ + string const ForceHash = _config->Find("Acquire::ForceHash"); + + for (; Vf.end() == false; Vf++) + { + // Ignore not source sources + if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0) + continue; + + // Try to cross match against the source list + pkgIndexFile *Index; + if (Sources->FindIndex(Vf.File(),Index) == false) + continue; + + // only try to get a trusted package from another source if that source + // is also trusted + if(Trusted && !Index->IsTrusted()) + continue; + + // Grab the text package record + pkgRecords::Parser &Parse = Recs->Lookup(Vf); + if (_error->PendingError() == true) + return false; + + string PkgFile = Parse.FileName(); + //std::cerr << " PkgFile:" << PkgFile << std::endl; + if (ForceHash.empty() == false) + { + if(stringcasecmp(ForceHash, "sha256") == 0) + ExpectedHash = HashString("SHA256", Parse.SHA256Hash()); + else if (stringcasecmp(ForceHash, "sha1") == 0) + ExpectedHash = HashString("SHA1", Parse.SHA1Hash()); + else + ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); + } + else + { + string Hash; + if ((Hash = Parse.SHA256Hash()).empty() == false) + ExpectedHash = HashString("SHA256", Hash); + else if ((Hash = Parse.SHA1Hash()).empty() == false) + ExpectedHash = HashString("SHA1", Hash); + else + ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); + } + if (PkgFile.empty() == true) + return _error->Error(_("The package index files are corrupted. No Filename: " + "field for package %s."), + Version.ParentPkg().Name()); + // See if we already have the deb + FileSize = Version->Size; + string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile); + struct stat Buf; + if (stat(FinalFile.c_str(),&Buf) == 0) + { + std::cerr << "[Debdelta] File already exists. " << FinalFile << std::endl; + // TODO: check if this ok. Make sure the size matches + if ((unsigned)Buf.st_size == Version->Size) + { + Complete = true; + Local = true; + Status = StatDone; + DebdeltaStatus = Completed; + StoreFilename = DestFile = FinalFile; + return true; + } + /* Hmm, we have a file and its size does not match, this means it is + an old style mismatched arch */ + unlink(FinalFile.c_str()); + } + + // Create the item + Local = false; + // See if we already have the debdelta in the partial dir. + if (FileExists(DestFile)) + { + // TODO: verify the sum/size of it. + std::cerr << "[Debdelta] File already exists. " << DestFile << std::endl; + Desc.URI = "debdelta:" + DestFile; + Mode = "debdelta"; + DebdeltaStatus = Patching; + Local = true; + } + else + { + Desc.URI = flNotFile(Index->ArchiveURI(PkgFile)) + DebdeltaName; + } + Desc.Description = "[Debdelta] " + Index->ArchiveInfo(Version); + Desc.Owner = this; + Desc.ShortDesc = "[Debdelta] " + string(Version.ParentPkg().Name()); + std::cerr << "[Debdelta] pkgAcqDebdelta::QueueNext()" << std::endl; + std::cerr << " DestFile : " << DestFile << std::endl; + std::cerr << " Desc.URI : " << Desc.URI << std::endl; + std::cerr << " StoreFileName: " << StoreFilename << std::endl; + Vf++; + QueueURI(Desc); + return true; + } + return false; +} + +void pkgAcqDebdelta::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + std::cerr << "\n[Debdelta] Failed to download " << Desc.URI << std::endl; + std::cerr << "[Debdelta] Message:\n" << Message << std::endl; + // TODO: find out what went wrong and display to the user + new pkgAcqArchive(Owner, Sources, Recs, Version, StoreFilename); + Complete = false; + Status = StatError; + DebdeltaStatus = FetchingFailure; + Item::Failed(Message, Cnf); + Dequeue(); +} + +void pkgAcqDebdelta::Done(string Message,unsigned long Size,string Hash, + pkgAcquire::MethodConfig *Cnf) +{ + // TODO: there must be two stages within this method. + // one for downloading debdelta. another for verifying the resulting .deb + std::cerr << "[Debdelta] pkgAcqDebdelta::Done() state: " << DebdeltaStatus << std::endl; + // Grab the output filename + string FileName = LookupTag(Message,"Filename"); + std::cerr << " StoreFileName: " << StoreFilename + << "\n DestFile : " << DestFile + << "\n Desc.URI : " << Desc.URI + << "\n FileName : " << FileName << std::endl; + if (DebdeltaStatus == Patching) + { + std::cerr << "[Debdelta] Patching Done. Verifying "<< FileName << "..." << std::endl; + if (ExpectedHash.toStr() != Hash) + { + Status = StatError; + ErrorText = _("[Debdelta] Hash Sum mismatch"); + if(FileExists(DestFile)) + if (!Local) + Rename(DestFile, DestFile + ".FAILED"); + //return; // TODO: UNCOMMENT + } + // Check the size + if (Size != Version->Size) + { + Status = StatError; + ErrorText = _("[Debdelta] Size mismatch"); + //return; // TODO: UNCOMMEnT + } + DebdeltaStatus = Completed; + Complete = true; + Status = StatDone; + StoreFilename = FileName; + Item::Done(Message,Size,Hash,Cnf); + return; + } + // TODO: Check the sum/size + //std::cerr << "[Debdelta] Verifying " << DestFile << std::endl; + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "[Debdelta] Method gave a blank filename"; + return; + } + if (FileName != DestFile) // this is set in the file/http methods + { + // file => FileName != DestFile (i.e. FileName is the local file) + // http => FileName == DestFile (i.e. FileName is some local place. may be within "partial" dir) + Local = true; + } + DebdeltaStatus = Patching; + Rename(DestFile, FileName); + DestFile = StoreFilename = FileName; + chmod(DestFile.c_str(), 0644); + Desc.URI = "debdelta:" + StoreFilename; + Mode = "debdelta"; + QueueURI(Desc); +} + +void pkgAcqDebdelta::Finished() +{ + std::cerr << "[Debdelta] pkgAcqDebdelta::Finished() state: " << DebdeltaStatus << std::endl; + if (Status == pkgAcquire::Item::StatDone && + Complete == true) + return; + StoreFilename = string(); +} + bool IndexTarget::IsOptional() const { if (strncmp(ShortDesc.c_str(), "Translation", 11) != 0) return false; diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index f763577ee..ebe503cda 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -1018,6 +1018,110 @@ class pkgAcqFile : public pkgAcquire::Item bool IsIndexFile=false); }; /*}}}*/ + +/** \brief An item that is responsible for fetching a debdelta file. {{{ + * + * If the package file already exists in the cache, nothing will be + * done + */ +class pkgAcqDebdelta : public pkgAcquire::Item +{ + protected: + string DebdeltaName; + /** \brief The package version being fetched. */ + pkgCache::VerIterator Version; + + /** \brief The fetch command that is currently being processed. */ + pkgAcquire::ItemDesc Desc; + + /** \brief The list of sources from which to pick archives to + * download this package from. + */ + pkgSourceList *Sources; + + /** \brief A package records object, used to look up the file + * corresponding to each version of the package. + */ + pkgRecords *Recs; + + /** \brief The hashsum of this package. */ + HashString ExpectedHash; + + /** \brief A location in which the actual filename of the package + * should be stored. + */ + string &StoreFilename; + + /** \brief The next file for this version to try to download. */ + pkgCache::VerFileIterator Vf; + + /** \brief How many (more) times to try to find a new source from + * which to download this package version if it fails. + * + * Set from Acquire::Retries. + */ + unsigned int Retries; + + /** \brief \b true if this version file is being downloaded from a + * trusted source. + */ + bool Trusted; + + /** \brief Queue up the next available file for this version. */ + bool QueueNext(); + + public: + + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(string Message,unsigned long Size,string Hash, + pkgAcquire::MethodConfig *Cnf); + virtual string DescURI() {return Desc.URI;}; + virtual string ShortDesc() {return Desc.ShortDesc;}; + virtual void Finished(); + virtual string HashSum() {return ExpectedHash.toStr(); }; + virtual bool IsTrusted() {return Trusted;}; + + /** \brief Create a new pkgAcqArchive. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param Sources The sources from which to download version + * files. + * + * \param Recs A package records object, used to look up the file + * corresponding to each version of the package. + * + * \param Version The package version to download. + * + * \param StoreFilename A location in which the actual filename of + * the package should be stored. It will be set to a guessed + * basename in the constructor, and filled in with a fully + * qualified filename once the download finishes. + */ + pkgAcqDebdelta(pkgAcquire *Owner,pkgSourceList *Sources, + pkgRecords *Recs,pkgCache::VerIterator const &Version, + string &StoreFilename); + enum DebdeltaState + { + Fetching, + Patching, + Completed, + FetchingFailure + } DebdeltaStatus; +}; + /*}}}*/ + +/** \brief Retrieve an arbitrary file to the current directory. {{{ + * + * The file is retrieved even if it is accessed via a URL type that + * normally is a NOP, such as "file". If the download fails, the + * partial file is renamed to get a ".FAILED" extension. + */ +class pkgAcqDebdeltaIndex : public pkgAcquire::Item +{}; + /*}}}*/ + /** @} */ #endif diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index e9e102488..eb1e003c2 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -323,13 +323,12 @@ int pkgAcqMethod::Run(bool Single) if (Single == true && Messages.empty() == true) return -1; - string Message = Messages.front(); + string m = Messages.front(); Messages.erase(Messages.begin()); - // Fetch the message number char *End; - int Number = strtol(Message.c_str(),&End,10); - if (End == Message.c_str()) + int Number = strtol(m.c_str(),&End,10); + if (End == m.c_str()) { cerr << "Malformed message!" << endl; return 100; @@ -338,20 +337,19 @@ int pkgAcqMethod::Run(bool Single) switch (Number) { case 601: - if (Configuration(Message) == false) - return 100; + if (Configuration(m) == false) + return 100; break; case 600: { FetchItem *Tmp = new FetchItem; - - Tmp->Uri = LookupTag(Message,"URI"); - Tmp->DestFile = LookupTag(Message,"FileName"); - if (RFC1123StrToTime(LookupTag(Message,"Last-Modified").c_str(),Tmp->LastModified) == false) + Tmp->Uri = LookupTag(m,"URI"); + Tmp->DestFile = LookupTag(m,"FileName"); + if (RFC1123StrToTime(LookupTag(m,"Last-Modified").c_str(),Tmp->LastModified) == false) Tmp->LastModified = 0; - Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); - Tmp->FailIgnore = StringToBool(LookupTag(Message,"Fail-Ignore"),false); + Tmp->IndexFile = StringToBool(LookupTag(m,"Index-File"),false); + Tmp->FailIgnore = StringToBool(LookupTag(m,"Fail-Ignore"),false); Tmp->Next = 0; // Append it to the list diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 75e03232a..5fa53db3d 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -105,13 +105,18 @@ pkgAcquire::Worker::~Worker() bool pkgAcquire::Worker::Start() { // Get the method path + std::cerr << "\n ---------------------------------------------------->>>" << std::endl; + std::cerr << " pkgAcquire::Worker::Start() starting the worker process" << std::endl; string Method = _config->FindDir("Dir::Bin::Methods") + Access; + std::cerr << " Method: " << Method << std::endl; + std::cerr << " Access: " << Access << std::endl; if (FileExists(Method) == false) return _error->Error(_("The method driver %s could not be found."),Method.c_str()); if (Debug == true) clog << "Starting method '" << Method << '\'' << endl; - + std::cerr << " Starting method '" << Method << '\'' << std::endl; + std::cerr << " creating the pipes." << std::endl; // Create the pipes int Pipes[4] = {-1,-1,-1,-1}; if (pipe(Pipes) != 0 || pipe(Pipes+2) != 0) @@ -123,7 +128,7 @@ bool pkgAcquire::Worker::Start() } for (int I = 0; I != 4; I++) SetCloseExec(Pipes[I],true); - + std::cerr << " Forking off the sub process. Pipes[0]: " << Pipes[0] << std::endl; // Fork off the process Process = ExecFork(); if (Process == 0) @@ -138,6 +143,7 @@ bool pkgAcquire::Worker::Start() const char *Args[2]; Args[0] = Method.c_str(); Args[1] = 0; + std::cerr << Args[0] << std::endl; execv(Args[0],(char **)Args); cerr << "Failed to exec method " << Args[0] << endl; _exit(100); @@ -161,7 +167,7 @@ bool pkgAcquire::Worker::Start() RunMessages(); if (OwnerQ != 0) SendConfiguration(); - + std::cerr << "<<<----------------------------------------------------\n" << std::endl; return true; } /*}}}*/ @@ -170,8 +176,10 @@ bool pkgAcquire::Worker::Start() /* */ bool pkgAcquire::Worker::ReadMessages() { + std::cerr << "pkgAcquire::Worker::ReadMessages() <- [Worker::MessageQueue]" << std::endl; if (::ReadMessages(InFd,MessageQueue) == false) return MethodFailure(); + //std::cerr << " MessageQueue {" << MessageQueue[0] << "}" << std::endl; return true; } /*}}}*/ @@ -181,6 +189,7 @@ bool pkgAcquire::Worker::ReadMessages() the parsers in order. */ bool pkgAcquire::Worker::RunMessages() { + std::cerr << "pkgAcquire::Worker::RunMessages() of [Worker::MessageQueue]" << std::endl; while (MessageQueue.empty() == false) { string Message = MessageQueue.front(); @@ -212,28 +221,33 @@ bool pkgAcquire::Worker::RunMessages() } // Determine the message number and dispatch + std::cerr << " Message Number: " << Number << std::endl; switch (Number) { // 100 Capabilities case 100: + std::cerr <<" 100 Capabilities" << std::endl; if (Capabilities(Message) == false) return _error->Error("Unable to process Capabilities message from %s",Access.c_str()); break; // 101 Log case 101: + std::cerr <<" 101 Log" << std::endl; if (Debug == true) clog << " <- (log) " << LookupTag(Message,"Message") << endl; break; // 102 Status case 102: + std::cerr <<" 102 Status" << std::endl; Status = LookupTag(Message,"Message"); break; // 103 Redirect case 103: { + std::cerr <<" 103 Redirect" << std::endl; if (Itm == 0) { _error->Error("Method gave invalid 103 Redirect message"); @@ -248,6 +262,7 @@ bool pkgAcquire::Worker::RunMessages() // 200 URI Start case 200: { + std::cerr <<" 201 URI Start" << std::endl; if (Itm == 0) { _error->Error("Method gave invalid 200 URI Start message"); @@ -273,6 +288,7 @@ bool pkgAcquire::Worker::RunMessages() // 201 URI Done case 201: { + std::cerr <<" 201 URI Done" << std::endl; if (Itm == 0) { _error->Error("Method gave invalid 201 URI Done message"); @@ -332,6 +348,7 @@ bool pkgAcquire::Worker::RunMessages() // 400 URI Failure case 400: { + std::cerr <<" 400 URI Failure" << std::endl; if (Itm == 0) { _error->Error("Method gave invalid 400 URI Failure message"); @@ -364,11 +381,13 @@ bool pkgAcquire::Worker::RunMessages() // 401 General Failure case 401: + std::cerr <<" 401 General Failure" << std::endl; _error->Error("Method %s General failure: %s",Access.c_str(),LookupTag(Message,"Message").c_str()); break; // 403 Media Change case 403: + std::cerr <<" 403 Media Change" << std::endl; MediaChange(Message); break; } @@ -392,7 +411,7 @@ bool pkgAcquire::Worker::Capabilities(string Message) Config->LocalOnly = StringToBool(LookupTag(Message,"Local-Only"),false); Config->NeedsCleanup = StringToBool(LookupTag(Message,"Needs-Cleanup"),false); Config->Removable = StringToBool(LookupTag(Message,"Removable"),false); - + //std::cerr << "----Capabilities()" << std::endl; // Some debug text if (Debug == true) { @@ -405,7 +424,7 @@ bool pkgAcquire::Worker::Capabilities(string Message) " NeedsCleanup: " << Config->NeedsCleanup << " Removable: " << Config->Removable << endl; } - + //std::cerr << "----Capabilities()" << std::endl; return true; } /*}}}*/ @@ -458,6 +477,8 @@ bool pkgAcquire::Worker::MediaChange(string Message) /* */ bool pkgAcquire::Worker::SendConfiguration() { + std::cerr << "pkgAcquire::Worker::SendConfiguration() -> [Worker::OutQueue]" << std::endl; + std::cerr << " Sending 601 Configuration message to the method." << std::endl; if (Config->SendConfig == false) return true; @@ -496,7 +517,7 @@ bool pkgAcquire::Worker::SendConfiguration() clog << " -> " << Access << ':' << QuoteString(Message,"\n") << endl; OutQueue += Message; OutReady = true; - + //std::cerr << " OutQueue {" << Message.length() <<"}"<< std::endl; return true; } /*}}}*/ @@ -519,7 +540,7 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) clog << " -> " << Access << ':' << QuoteString(Message,"\n") << endl; OutQueue += Message; OutReady = true; - + return true; } /*}}}*/ @@ -528,10 +549,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) /* */ bool pkgAcquire::Worker::OutFdReady() { + //std::cerr << "pkgAcquire::Worker::OutFdReady()." << std::endl; int Res; do { Res = write(OutFd,OutQueue.c_str(),OutQueue.length()); + //std::cerr << " writing the OutQueue to the OutFd " << OutFd << std::endl; } while (Res < 0 && errno == EINTR); diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 9478cdfb4..43d5b6b52 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -208,6 +208,7 @@ void pkgAcquire::Enqueue(ItemDesc &Item) for (; I != 0 && I->Name != Name; I = I->Next); if (I == 0) { + //std::cerr << " Creating a new queue for the name: " << Name << std::endl; I = new Queue(Name,this); I->Next = Queues; Queues = I; @@ -228,9 +229,10 @@ void pkgAcquire::Enqueue(ItemDesc &Item) // Some trace stuff if (Debug == true) { - clog << "Fetching " << Item.URI << endl; - clog << " to " << Item.Owner->DestFile << endl; - clog << " Queue is: " << Name << endl; + clog << "pkgAcquire::Enqueue()" << endl; + clog << " Fetching " << Item.URI << endl; + clog << " to " << Item.Owner->DestFile << endl; + clog << " Queue is: " << Name << endl; } } /*}}}*/ @@ -280,6 +282,7 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config) pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access) { // Search for an existing config + //std::cerr << "pkgAcquire::GetConfig()" << std::endl; MethodConfig *Conf; for (Conf = Configs; Conf != 0; Conf = Conf->Next) if (Conf->Access == Access) @@ -292,6 +295,7 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access) Configs = Conf; // Create the worker to fetch the configuration + //std::cerr <<" creating a new worker to fetch the configuration" << std::endl; Worker Work(Conf); if (Work.Start() == false) return 0; @@ -349,9 +353,11 @@ void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet) pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall) { Running = true; - + //std::cerr << "pkgAcquire::Run(): Running." << std::endl; + //std::cerr << " starting up the queues." << std::endl; for (Queue *I = Queues; I != 0; I = I->Next) I->Startup(); + //std::cerr << "\npkgAcquire::Run(): started queues.\n" << std::endl; if (Log != 0) Log->Start(); @@ -640,7 +646,6 @@ bool pkgAcquire::Queue::Startup() pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access); if (Cnf == 0) return false; - Workers = new Worker(this,Cnf,Owner->Log); Owner->Add(Workers); if (Workers->Start() == false) @@ -721,6 +726,7 @@ bool pkgAcquire::Queue::ItemDone(QItem *Itm) is enabled then it keeps the pipe full. */ bool pkgAcquire::Queue::Cycle() { + //std::cerr << "pkgAcquire::Queue::Cycle():" << std::endl; if (Items == 0 || Workers == 0) return true; @@ -738,7 +744,7 @@ bool pkgAcquire::Queue::Cycle() // Nothing to do, queue is idle. if (I == 0) return true; - + //std::cerr << " Found an Item to queue. " << I->ShortDesc << " " << I->URI << std::endl; I->Worker = Workers; I->Owner->Status = pkgAcquire::Item::StatFetching; PipeDepth++; @@ -876,6 +882,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) /* We just reset the counters */ void pkgAcquireStatus::Start() { + //std::cerr << "pkgAcquireStatus::Start():" << std::endl; gettimeofday(&Time,0); gettimeofday(&StartTime,0); LastBytes = 0; diff --git a/apt-pkg/algorithms.cc b/apt-pkg/algorithms.cc index 2dae4258a..871d377a2 100644 --- a/apt-pkg/algorithms.cc +++ b/apt-pkg/algorithms.cc @@ -391,7 +391,7 @@ bool pkgAllUpgrade(pkgDepCache &Cache) { if (Cache[I].Install() == true) Fix.Protect(I); - + if (_config->FindB("APT::Ignore-Hold",false) == false) if (I->SelectedState == pkgCache::State::Hold) continue; @@ -1372,19 +1372,25 @@ bool ListUpdate(pkgAcquireStatus &Stat, pkgSourceList &List, int PulseInterval) { + //std::cerr << "ListUpdate(): Starting." << std::endl; pkgAcquire::RunResult res; pkgAcquire Fetcher; + //std::cerr << "ListUpdate(): Setting up the fetcher..." << std::endl; if (Fetcher.Setup(&Stat, _config->FindDir("Dir::State::Lists")) == false) return false; // Populate it with the source selection + //std::cerr << "ListUpdate(): Populating the fetcher with the indexes." << std::endl; if (List.GetIndexes(&Fetcher) == false) return false; // Run scripts + //std::cerr << "\nListUpdate(): Running the Pre-Invoke scripts..." << std::endl; RunScripts("APT::Update::Pre-Invoke"); // check arguments + //std::cerr << "ListUpdate(): PulseInterval: " << PulseInterval << std::endl; + //std::cerr << "ListUpdate(): Running the Fetcher.\n" << std::endl; if(PulseInterval>0) res = Fetcher.Run(PulseInterval); else @@ -1392,12 +1398,14 @@ bool ListUpdate(pkgAcquireStatus &Stat, if (res == pkgAcquire::Failed) return false; - + //std::cerr << "ListUpdate(): Res of the Fetcher is ok." << std::endl; bool Failed = false; bool TransientNetworkFailure = false; + //std::cerr << "ListUpdate(): Iterating through the Items that Fetcher fetched. Is this a failed set of items?" << std::endl; for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); I != Fetcher.ItemsEnd(); I++) { + //std::cerr << " (*I)->DescURI() " << (*I)->DescURI() << std::endl; if ((*I)->Status == pkgAcquire::Item::StatDone) continue; diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc index 964c5bd8b..932f5be03 100644 --- a/apt-pkg/cachefile.cc +++ b/apt-pkg/cachefile.cc @@ -23,7 +23,7 @@ #include #include - /*}}}*/ +using namespace std; /*}}}*/ // CacheFile::CacheFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -50,47 +50,47 @@ pkgCacheFile::~pkgCacheFile() /* */ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) { - if (Cache != NULL) - return true; - - if (_config->FindB("pkgCacheFile::Generate", true) == false) - { - Map = new MMap(*new FileFd(_config->FindFile("Dir::Cache::pkgcache"), - FileFd::ReadOnly),MMap::Public|MMap::ReadOnly); - Cache = new pkgCache(Map); - if (_error->PendingError() == true) - return false; - return true; - } - - const bool ErrorWasEmpty = _error->empty(); - if (WithLock == true) - if (_system->Lock() == false) - return false; + if (Cache != NULL) + return true; + + if (_config->FindB("pkgCacheFile::Generate", true) == false) + { + Map = new MMap(*new FileFd(_config->FindFile("Dir::Cache::pkgcache"), + FileFd::ReadOnly),MMap::Public|MMap::ReadOnly); + Cache = new pkgCache(Map); + if (_error->PendingError() == true) + return false; + return true; + } + + const bool ErrorWasEmpty = _error->empty(); + if (WithLock == true) + if (_system->Lock() == false) + return false; - if (_config->FindB("Debug::NoLocking",false) == true) - WithLock = false; + if (_config->FindB("Debug::NoLocking",false) == true) + WithLock = false; - if (_error->PendingError() == true) - return false; - - BuildSourceList(Progress); - - // Read the caches - bool Res = pkgCacheGenerator::MakeStatusCache(*SrcList,Progress,&Map,!WithLock); - if (Progress != NULL) - Progress->Done(); - if (Res == false) - return _error->Error(_("The package lists or status file could not be parsed or opened.")); - - /* This sux, remove it someday */ - if (ErrorWasEmpty == true && _error->empty() == false) - _error->Warning(_("You may want to run apt-get update to correct these problems")); - - Cache = new pkgCache(Map); - if (_error->PendingError() == true) - return false; - return true; + if (_error->PendingError() == true) + return false; + + BuildSourceList(Progress); + + // Read the caches + bool Res = pkgCacheGenerator::MakeStatusCache(*SrcList,Progress,&Map,!WithLock); + if (Progress != NULL) + Progress->Done(); + if (Res == false) + return _error->Error(_("The package lists or status file could not be parsed or opened.")); + + /* This sux, remove it someday */ + if (ErrorWasEmpty == true && _error->empty() == false) + _error->Warning(_("You may want to run apt-get update to correct these problems")); + + Cache = new pkgCache(Map); + if (_error->PendingError() == true) + return false; + return true; } /*}}}*/ // CacheFile::BuildSourceList - Open and build all relevant sources.list/*{{{*/ @@ -98,9 +98,13 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) /* */ bool pkgCacheFile::BuildSourceList(OpProgress *Progress) { - if (SrcList != NULL) + //std::cerr << "BuildSourceList(): Starting..." << std::endl; + if (SrcList != NULL) { + //std::cerr << " SrcList is not null. Returning." << std::endl; return true; + } + //std::cerr << " making a new SrcList." << std::endl; SrcList = new pkgSourceList(); if (SrcList->ReadMainList() == false) return _error->Error(_("The list of sources could not be read.")); diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 50019872e..b1724971f 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -148,7 +148,8 @@ bool CopyFile(FileFd &From,FileFd &To) return true; } - /*}}}*/ + +/*}}}*/ // GetLock - Gets a lock file /*{{{*/ // --------------------------------------------------------------------- /* This will create an empty file of the given name and lock it. Once this @@ -275,7 +276,22 @@ bool CreateDirectory(string const &Parent, string const &Path) } return true; } - /*}}}*/ + +// Rename - Rename a file /*{{{*/ +// --------------------------------------------------------------------- +/* This helper function is used by alot of item methods as thier final + step */ +bool Rename(string From, string To) +{ + if (rename(From.c_str(),To.c_str()) != 0) + { + return _error->Error("Rename failed. %s (%s -> %s).",strerror(errno), + From.c_str(),To.c_str()); + } + return true; +} + +/*}}}*/ // CreateAPTDirectoryIfNeeded - ensure that the given directory exists /*{{{*/ // --------------------------------------------------------------------- /* a small wrapper around CreateDirectory to check if it exists and to @@ -571,6 +587,7 @@ void SetNonBlock(int Fd,bool Block) in seconds. */ bool WaitFd(int Fd,bool write,unsigned long timeout) { + //std::cerr << "WaitFd()" << std::endl; fd_set Set; struct timeval tv; FD_ZERO(&Set); @@ -601,7 +618,7 @@ bool WaitFd(int Fd,bool write,unsigned long timeout) if (Res <= 0) return false; } - + //std::cerr << " waited for FD " << Fd << std::endl; return true; } /*}}}*/ diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index cde288ad2..7c1127798 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -97,6 +97,7 @@ bool FileExists(string File); bool RealFileExists(string File); bool DirectoryExists(string const &Path) __attrib_const; bool CreateDirectory(string const &Parent, string const &Path); +bool Rename(string From, string To); /** \brief Ensure the existence of the given Path * diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index 072dda3ac..a452f65f3 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -442,6 +442,7 @@ std::string OutputInDepth(const unsigned long Depth, const char* Separator) string URItoFileName(const string &URI) { // Nuke 'sensitive' items + //std::cerr << "\n>>>>URItoFileName():\n URI: " << URI << std::endl; ::URI U(URI); U.User.clear(); U.Password.clear(); @@ -450,6 +451,7 @@ string URItoFileName(const string &URI) // "\x00-\x20{}|\\\\^\\[\\]<>\"\x7F-\xFF"; string NewURI = QuoteString(U,"\\|{}[]<>\"^~_=!@#$%^&*"); replace(NewURI.begin(),NewURI.end(),'/','_'); + //std::cerr << " NewUri: " << NewURI << std::endl; return NewURI; } /*}}}*/ @@ -723,6 +725,7 @@ string TimeRFC1123(time_t Date) */ bool ReadMessages(int Fd, vector &List) { + //std::cerr << "ReadMessages() Reading from Fd " << Fd << std::endl; char Buffer[64000]; char *End = Buffer; // Represents any left-over from the previous iteration of the @@ -765,6 +768,7 @@ bool ReadMessages(int Fd, vector &List) I = Buffer; List.push_back(PartialMessage); + //std::cerr << " pushed back message:" << PartialMessage << std::endl; PartialMessage.clear(); } if (End != Buffer) @@ -790,7 +794,7 @@ bool ReadMessages(int Fd, vector &List) // -- dburrows 2008-04-02 return true; } - + //std::cerr << " calling WaidFd again." << std::endl; if (WaitFd(Fd) == false) return false; } diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index a30f27844..24bd73bda 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -32,6 +32,7 @@ const char *pkgLibVersion = Stringfy(APT_PKG_MAJOR) "." is prepended, this allows a fair degree of flexability. */ bool pkgInitConfig(Configuration &Cnf) { + std::cerr << "pkgInitConfig()" << std::endl; // General APT things Cnf.Set("APT::Architecture", COMMON_ARCH); Cnf.Set("APT::Build-Essential::", "build-essential"); @@ -72,7 +73,7 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.Set("Dir::Etc::preferencesparts","preferences.d"); Cnf.Set("Dir::Etc::trusted", "trusted.gpg"); Cnf.Set("Dir::Etc::trustedparts","trusted.gpg.d"); - Cnf.Set("Dir::Bin::methods","/usr/lib/apt/methods"); + Cnf.Set("Dir::Bin::methods","/usr/lib/apt/methods"); // /home/ishan/devel/apt/my/experiments/debian-sid/bin/methods Cnf.Set("Dir::Media::MountPath","/media/apt"); // State @@ -139,6 +140,7 @@ bool pkgInitSystem(Configuration &Cnf,pkgSystem *&Sys) { Sys = 0; string Label = Cnf.Find("Apt::System",""); + std::cerr << "pkgInitSystem() " << Label.c_str() << std::endl; if (Label.empty() == false) { Sys = pkgSystem::GetSystem(Label.c_str()); diff --git a/apt-pkg/packagemanager.cc b/apt-pkg/packagemanager.cc index fe9f6eb68..9982dc737 100644 --- a/apt-pkg/packagemanager.cc +++ b/apt-pkg/packagemanager.cc @@ -80,8 +80,10 @@ bool pkgPackageManager::GetArchives(pkgAcquire *Owner,pkgSourceList *Sources, // Skip already processed packages if (List->IsNow(Pkg) == false) continue; - - new pkgAcqArchive(Owner,Sources,Recs,Cache[Pkg].InstVerIter(Cache), + + //new pkgAcqArchive(Owner,Sources,Recs,Cache[Pkg].InstVerIter(Cache), + // FileNames[Pkg->ID]); + new pkgAcqDebdelta(Owner,Sources,Recs,Cache[Pkg].InstVerIter(Cache), FileNames[Pkg->ID]); } diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 851eefdfe..3559c1160 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -183,10 +183,12 @@ pkgSourceList::~pkgSourceList() /* */ bool pkgSourceList::ReadMainList() { + //std::cerr << "ReadMainList(): Starting." << std::endl; // CNC:2003-03-03 - Multiple sources list support. bool Res = true; #if 0 Res = ReadVendors(); + std::cerr << " ReadVendors " << Res << std::endl; if (Res == false) return false; #endif @@ -196,7 +198,8 @@ bool pkgSourceList::ReadMainList() // entries in sources.list.d. string Main = _config->FindFile("Dir::Etc::sourcelist"); string Parts = _config->FindDir("Dir::Etc::sourceparts"); - + //std::cerr << " Main: " << Main << std::endl; + //std::cerr << " Parts: " << Parts << std::endl; if (RealFileExists(Main) == true) Res &= ReadAppend(Main); else if (DirectoryExists(Parts) == false) @@ -218,6 +221,7 @@ bool pkgSourceList::ReadMainList() /* */ void pkgSourceList::Reset() { + std::cerr << "Reset(): Clear the sourcelist contents." << std::endl; for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++) delete *I; SrcList.erase(SrcList.begin(),SrcList.end()); @@ -238,6 +242,7 @@ bool pkgSourceList::Read(string File) /* */ bool pkgSourceList::ReadAppend(string File) { + std::cerr << "ReadAppend(): File " << File << std::endl; // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (!F != 0) @@ -255,6 +260,7 @@ bool pkgSourceList::ReadAppend(string File) while (F.eof() == false) { F.getline(Buffer,sizeof(Buffer)); + //std::cerr << " Buffer " << Buffer << std::endl; CurLine++; _strtabexpand(Buffer,sizeof(Buffer)); if (F.fail() && !F.eof()) @@ -318,6 +324,7 @@ bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File, /* */ bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const { + //std::cerr << "GetIndexes(): Loading the index files into the downloader." << std::endl; for (const_iterator I = SrcList.begin(); I != SrcList.end(); I++) if ((*I)->GetIndexes(Owner,GetAll) == false) return false; -- cgit v1.2.3