diff options
38 files changed, 1105 insertions, 100 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c035b9163..bb8a01db8 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -64,6 +64,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { Status = StatIdle; ErrorText = LookupTag(Message,"Message"); + UsedMirror = LookupTag(Message,"UsedMirror"); if (QueueCounter <= 1) { /* This indicates that the file is not available right now but might @@ -76,10 +77,17 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Dequeue(); return; } - + Status = StatError; Dequeue(); } + + // report mirror failure back to LP if we actually use a mirror + string FailReason = LookupTag(Message, "FailReason"); + if(FailReason.size() != 0) + ReportMirrorFailure(FailReason); + else + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -101,7 +109,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash, { // We just downloaded something.. string FileName = LookupTag(Message,"Filename"); - // we only inform the Log class if it was actually not a local thing + UsedMirror = LookupTag(Message,"UsedMirror"); if (Complete == false && !Local && FileName == DestFile) { if (Owner->Log != 0) @@ -110,7 +118,6 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash, if (FileSize == 0) FileSize= Size; - Status = StatDone; ErrorText = string(); Owner->Dequeue(this); @@ -132,6 +139,49 @@ void pkgAcquire::Item::Rename(string From,string To) } } /*}}}*/ + +void pkgAcquire::Item::ReportMirrorFailure(string FailCode) +{ + // we only act if a mirror was used at all + if(UsedMirror.empty()) + return; +#if 0 + std::cerr << "\nReportMirrorFailure: " + << UsedMirror + << " Uri: " << DescURI() + << " FailCode: " + << FailCode << std::endl; +#endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/lib/apt/apt-report-mirror-failure"); + if(!FileExists(report)) + return; + Args[i++] = report.c_str(); + Args[i++] = UsedMirror.c_str(); + Args[i++] = DescURI().c_str(); + Args[i++] = FailCode.c_str(); + Args[i++] = NULL; + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(Args[0], (char**)Args); + std::cerr << "Could not exec " << Args[0] << std::endl; + _exit(100); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Methods::Mirror::ProblemReporting").c_str()); + } +} + // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Get the DiffIndex file first and see if there are patches availabe @@ -624,7 +674,6 @@ string pkgAcqIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true"; - return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ @@ -692,6 +741,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, Status = StatAuthError; ErrorText = _("Hash Sum mismatch"); Rename(DestFile,DestFile + ".FAILED"); + ReportMirrorFailure("HashChecksumFailure"); return; } // Done, move it into position @@ -882,8 +932,9 @@ void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5, Rename(LastGoodSig, DestFile); // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved - new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, - DestFile, IndexTargets, MetaIndexParser); + new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, + MetaIndexShortDesc, DestFile, IndexTargets, + MetaIndexParser); } /*}}}*/ @@ -896,7 +947,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { Item::Failed(Message,Cnf); // move the sigfile back on transient network failures - if(FileExists(DestFile)) + if(FileExists(LastGoodSig)) Rename(LastGoodSig,Final); // set the status back to , Item::Failed likes to reset it @@ -971,6 +1022,15 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, /*{{{* if (AuthPass == true) { AuthDone(Message); + + // all cool, move Release file into place + Complete = true; + + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; } else { @@ -1022,22 +1082,15 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ return; } - // see if the download was a IMSHit + // make sure to verify against the right file on I-M-S hit IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false); + if(IMSHit) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + DestFile = FinalFile; + } Complete = true; - - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - - // If we get a IMS hit we can remove the empty file in partial - // othersie we move the file in place - if (IMSHit) - unlink(DestFile.c_str()); - else - Rename(DestFile,FinalFile); - - chmod(FinalFile.c_str(),0644); - DestFile = FinalFile; } /*}}}*/ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ @@ -1067,7 +1120,6 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ QueueIndexes(true); // Done, move signature file into position - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI) + ".gpg"; Rename(SigFile,VerifiedSigFile); @@ -1108,7 +1160,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ // Queue Packages file (either diff or full packages files, depending // on the users option) - if(_config->FindB("Acquire::PDiffs",true) == true) + if(_config->FindB("Acquire::PDiffs", true) == true) new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, ExpectedIndexHash); else @@ -1211,30 +1263,30 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { if (AuthPass == true) { - // if we fail the authentication but got the file via a IMS-Hit - // this means that the file wasn't downloaded and that it might be - // just stale (server problem, proxy etc). we delete what we have - // queue it again without i-m-s - // alternatively we could just unlink the file and let the user try again - if (IMSHit) + // gpgv method failed, if we have a good signature + string LastGoodSigFile = _config->FindDir("Dir::State::lists") + + "partial/" + URItoFileName(RealURI) + ".gpg.reverify"; + if(FileExists(LastGoodSigFile)) { - Complete = false; - Local = false; - AuthPass = false; - unlink(DestFile.c_str()); - - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI); - Desc.URI = RealURI; - QueueURI(Desc); + string VerifiedSigFile = _config->FindDir("Dir::State::lists") + + URItoFileName(RealURI) + ".gpg"; + Rename(LastGoodSigFile,VerifiedSigFile); + Status = StatTransientNetworkError; + _error->Warning(_("A error occurred during the signature " + "verification. The repository is not updated " + "and the previous index files will be used." + "GPG error: %s: %s\n"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + RunScripts("APT::Update::Auth-Failure"); return; + } else { + _error->Warning(_("GPG error: %s: %s"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); } - // gpgv method failed - _error->Warning("GPG error: %s: %s", - Desc.Description.c_str(), - LookupTag(Message,"Message").c_str()); - + ReportMirrorFailure("GPGFailure"); } // No Release file was present, or verification failed, so fall diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index b338b2a41..332df5a6b 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -143,6 +143,7 @@ class pkgAcquire::Item : public WeakPointable * download progress indicator's overall statistics. */ bool Local; + string UsedMirror; /** \brief The number of fetch queues into which this item has been * inserted. @@ -243,6 +244,17 @@ class pkgAcquire::Item : public WeakPointable /** \return \b true if this object is being fetched from a trusted source. */ virtual bool IsTrusted() {return false;}; + + // report mirror problems + /** \brief Report mirror problem + * + * This allows reporting mirror failures back to a centralized + * server. The apt-report-mirror-failure script is called for this + * + * \param FailCode A short failure string that is send + */ + void ReportMirrorFailure(string FailCode); + /** \brief Initialize an item. * @@ -551,7 +563,8 @@ class pkgAcqIndex : public pkgAcquire::Item * fallback is ".gz" or none. */ pkgAcqIndex(pkgAcquire *Owner,string URI,string URIDesc, - string ShortDesc, HashString ExpectedHash, string compressExt=""); + string ShortDesc, HashString ExpectedHash, + string compressExt=""); }; /*}}}*/ /** \brief An acquire item that is responsible for fetching a {{{ @@ -614,7 +627,6 @@ class pkgAcqMetaSig : public pkgAcquire::Item /** \brief The last good signature file */ string LastGoodSig; - /** \brief The fetch request that is currently being processed. */ pkgAcquire::ItemDesc Desc; diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index fe066741c..3008c8d1a 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -96,12 +96,11 @@ void pkgAcqMethod::Fail(string Err,bool Transient) } char S[1024]; + char *End = S; if (Queue != 0) { - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" - "Message: %s %s\n",Queue->Uri.c_str(),Err.c_str(), - FailExtra.c_str()); - + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" + "Message: %s %s\n",Queue->Uri.c_str(), Err.c_str(), IP.c_str()); // Dequeue FetchItem *Tmp = Queue; Queue = Queue->Next; @@ -110,10 +109,14 @@ void pkgAcqMethod::Fail(string Err,bool Transient) QueueBack = Queue; } else - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: <UNKNOWN>\n" - "Message: %s %s\n",Err.c_str(), - FailExtra.c_str()); - + { + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: <UNKNOWN>\n" + "Message: %s\n",Err.c_str()); + } + if(FailReason.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"FailReason: %s\n",FailReason.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); // Set the transient flag if (Transient == true) strcat(S,"Transient-Failure: true\n\n"); @@ -184,6 +187,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) End += snprintf(End,sizeof(S)-50 - (End - S),"SHA1-Hash: %s\n",Res.SHA1Sum.c_str()); if (Res.SHA256Sum.empty() == false) End += snprintf(End,sizeof(S)-50 - (End - S),"SHA256-Hash: %s\n",Res.SHA256Sum.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); if (Res.GPGVOutput.size() > 0) End += snprintf(End,sizeof(S)-50 - (End - S),"GPGVOutput:\n"); for (vector<string>::iterator I = Res.GPGVOutput.begin(); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index fab77e664..99a4605b1 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -59,7 +59,9 @@ class pkgAcqMethod vector<string> Messages; FetchItem *Queue; FetchItem *QueueBack; - string FailExtra; + string FailReason; + string UsedMirror; + string IP; // Handlers for messages virtual bool Configuration(string Message); @@ -68,14 +70,14 @@ class pkgAcqMethod // Outgoing messages void Fail(bool Transient = false); inline void Fail(const char *Why, bool Transient = false) {Fail(string(Why),Transient);}; - void Fail(string Why, bool Transient = false); - void URIStart(FetchResult &Res); - void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + bool MediaFail(string Required,string Drive); virtual void Exit() {}; public: - enum CnfFlags {SingleInstance = (1<<0), Pipeline = (1<<1), SendConfig = (1<<2), LocalOnly = (1<<3), NeedsCleanup = (1<<4), @@ -87,7 +89,8 @@ class pkgAcqMethod void Redirect(const string &NewURI); int Run(bool Single = false); - inline void SetFailExtraMsg(string Msg) {FailExtra = Msg;}; + inline void SetFailReason(string Msg) {FailReason = Msg;}; + inline void SetIP(string aIP) {IP = aIP;}; pkgAcqMethod(const char *Ver,unsigned long Flags = 0); virtual ~pkgAcqMethod() {}; diff --git a/apt-pkg/algorithms.cc b/apt-pkg/algorithms.cc index f1e51131a..f17c76d6c 100644 --- a/apt-pkg/algorithms.cc +++ b/apt-pkg/algorithms.cc @@ -1184,8 +1184,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix) return _error->Error(_("Unable to correct problems, you have held broken packages.")); } - // set the auto-flags (mvo: I'm not sure if we _really_ need this, but - // I didn't managed + // set the auto-flags (mvo: I'm not sure if we _really_ need this) pkgCache::PkgIterator I = Cache.PkgBegin(); for (;I.end() != true; I++) { if (Cache[I].NewInstall() && !(Flags[I->ID] & PreInstalled)) { diff --git a/apt-pkg/deb/debsystem.cc b/apt-pkg/deb/debsystem.cc index 59f826d96..ef5ec9703 100644 --- a/apt-pkg/deb/debsystem.cc +++ b/apt-pkg/deb/debsystem.cc @@ -18,7 +18,6 @@ #include <apt-pkg/error.h> #include <apt-pkg/fileutl.h> #include <apti18n.h> - #include <sys/types.h> #include <unistd.h> #include <dirent.h> @@ -79,8 +78,15 @@ bool debSystem::Lock() { close(LockFD); LockFD = -1; + const char *cmd; + if (getenv("SUDO_USER") != NULL) + cmd = "sudo dpkg --configure -a"; + else + cmd = "dpkg --configure -a"; + // TRANSLATORS: the %s contains the recovery command, usually + // dpkg --configure -a return _error->Error(_("dpkg was interrupted, you must manually " - "run 'dpkg --configure -a' to correct the problem. ")); + "run '%s' to correct the problem. "), cmd); } LockCount++; diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 8318fe37f..c724c69a9 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -12,6 +12,7 @@ #include <apt-pkg/error.h> #include <apt-pkg/configuration.h> #include <apt-pkg/depcache.h> +#include <apt-pkg/pkgrecords.h> #include <apt-pkg/strutl.h> #include <apti18n.h> #include <apt-pkg/fileutl.h> @@ -24,6 +25,7 @@ #include <sys/wait.h> #include <signal.h> #include <errno.h> +#include <string.h> #include <stdio.h> #include <string.h> #include <algorithm> @@ -354,7 +356,6 @@ bool pkgDPkgPM::RunScriptsWithPkgs(const char *Cnf) return true; } - /*}}}*/ // DPkgPM::DoStdin - Read stdin and pass to slave pty /*{{{*/ // --------------------------------------------------------------------- @@ -425,7 +426,7 @@ void pkgDPkgPM::ProcessDpkgStatusLine(int OutStatusFd, char *line) 'processing: trigproc: trigger' */ - char* list[5]; + char* list[6]; // dpkg sends multiline error messages sometimes (see // #374195 for a example. we should support this by // either patching dpkg to not send multiline over the @@ -476,6 +477,14 @@ void pkgDPkgPM::ProcessDpkgStatusLine(int OutStatusFd, char *line) if(strncmp(action,"error",strlen("error")) == 0) { + // urgs, sometime has ":" in its error string so that we + // end up with the error message split between list[3] + // and list[4], e.g. the message: + // "failed in buffer_write(fd) (10, ret=-1): backend dpkg-deb ..." + // concat them again + if( list[4] != NULL ) + list[3][strlen(list[3])] = ':'; + status << "pmerror:" << list[1] << ":" << (PackagesDone/float(PackagesTotal)*100.0) << ":" << list[3] @@ -484,6 +493,8 @@ void pkgDPkgPM::ProcessDpkgStatusLine(int OutStatusFd, char *line) write(OutStatusFd, status.str().c_str(), status.str().size()); if (Debug == true) std::clog << "send: '" << status.str() << "'" << endl; + pkgFailures++; + WriteApportReport(list[1], list[3]); return; } else if(strncmp(action,"conffile",strlen("conffile")) == 0) @@ -1175,3 +1186,186 @@ void pkgDPkgPM::Reset() List.erase(List.begin(),List.end()); } /*}}}*/ +// pkgDpkgPM::WriteApportReport - write out error report pkg failure /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) +{ + string pkgname, reportfile, srcpkgname, pkgver, arch; + string::size_type pos; + FILE *report; + + if (_config->FindB("Dpkg::ApportFailureReport", false) == false) + { + std::clog << "configured to not write apport reports" << std::endl; + return; + } + + // only report the first errors + if(pkgFailures > _config->FindI("APT::Apport::MaxReports", 3)) + { + std::clog << _("No apport report written because MaxReports is reached already") << std::endl; + return; + } + + // check if its not a follow up error + const char *needle = dgettext("dpkg", "dependency problems - leaving unconfigured"); + if(strstr(errormsg, needle) != NULL) { + std::clog << _("No apport report written because the error message indicates its a followup error from a previous failure.") << std::endl; + return; + } + + // do not report disk-full failures + if(strstr(errormsg, strerror(ENOSPC)) != NULL) { + std::clog << _("No apport report written because the error message indicates a disk full error") << std::endl; + return; + } + + // do not report out-of-memory failures + if(strstr(errormsg, strerror(ENOMEM)) != NULL) { + std::clog << _("No apport report written because the error message indicates a out of memory error") << std::endl; + return; + } + + // do not report dpkg I/O errors + // XXX - this message is localized, but this only matches the English version. This is better than nothing. + if(strstr(errormsg, "short read in buffer_copy (")) { + std::clog << _("No apport report written because the error message indicates a dpkg I/O error") << std::endl; + return; + } + + // get the pkgname and reportfile + pkgname = flNotDir(pkgpath); + pos = pkgname.find('_'); + if(pos != string::npos) + pkgname = pkgname.substr(0, pos); + + // find the package versin and source package name + pkgCache::PkgIterator Pkg = Cache.FindPkg(pkgname); + if (Pkg.end() == true) + return; + pkgCache::VerIterator Ver = Cache.GetCandidateVer(Pkg); + if (Ver.end() == true) + return; + pkgver = Ver.VerStr() == NULL ? "unknown" : Ver.VerStr(); + pkgRecords Recs(Cache); + pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); + srcpkgname = Parse.SourcePkg(); + if(srcpkgname.empty()) + srcpkgname = pkgname; + + // if the file exists already, we check: + // - if it was reported already (touched by apport). + // If not, we do nothing, otherwise + // we overwrite it. This is the same behaviour as apport + // - if we have a report with the same pkgversion already + // then we skip it + reportfile = flCombine("/var/crash",pkgname+".0.crash"); + if(FileExists(reportfile)) + { + struct stat buf; + char strbuf[255]; + + // check atime/mtime + stat(reportfile.c_str(), &buf); + if(buf.st_mtime > buf.st_atime) + return; + + // check if the existing report is the same version + report = fopen(reportfile.c_str(),"r"); + while(fgets(strbuf, sizeof(strbuf), report) != NULL) + { + if(strstr(strbuf,"Package:") == strbuf) + { + char pkgname[255], version[255]; + if(sscanf(strbuf, "Package: %s %s", pkgname, version) == 2) + if(strcmp(pkgver.c_str(), version) == 0) + { + fclose(report); + return; + } + } + } + fclose(report); + } + + // now write the report + arch = _config->Find("APT::Architecture"); + report = fopen(reportfile.c_str(),"w"); + if(report == NULL) + return; + if(_config->FindB("DPkgPM::InitialReportOnly",false) == true) + chmod(reportfile.c_str(), 0); + else + chmod(reportfile.c_str(), 0600); + fprintf(report, "ProblemType: Package\n"); + fprintf(report, "Architecture: %s\n", arch.c_str()); + time_t now = time(NULL); + fprintf(report, "Date: %s" , ctime(&now)); + fprintf(report, "Package: %s %s\n", pkgname.c_str(), pkgver.c_str()); + fprintf(report, "SourcePackage: %s\n", srcpkgname.c_str()); + fprintf(report, "ErrorMessage:\n %s\n", errormsg); + + // ensure that the log is flushed + if(term_out) + fflush(term_out); + + // attach terminal log it if we have it + string logfile_name = _config->FindFile("Dir::Log::Terminal"); + if (!logfile_name.empty()) + { + FILE *log = NULL; + char buf[1024]; + + fprintf(report, "DpkgTerminalLog:\n"); + log = fopen(logfile_name.c_str(),"r"); + if(log != NULL) + { + while( fgets(buf, sizeof(buf), log) != NULL) + fprintf(report, " %s", buf); + fclose(log); + } + } + + // log the ordering + const char *ops_str[] = {"Install", "Configure","Remove","Purge"}; + fprintf(report, "AptOrdering:\n"); + for (vector<Item>::iterator I = List.begin(); I != List.end(); I++) + fprintf(report, " %s: %s\n", (*I).Pkg.Name(), ops_str[(*I).Op]); + + // attach dmesg log (to learn about segfaults) + if (FileExists("/bin/dmesg")) + { + FILE *log = NULL; + char buf[1024]; + + fprintf(report, "Dmesg:\n"); + log = popen("/bin/dmesg","r"); + if(log != NULL) + { + while( fgets(buf, sizeof(buf), log) != NULL) + fprintf(report, " %s", buf); + fclose(log); + } + } + + // attach df -l log (to learn about filesystem status) + if (FileExists("/bin/df")) + { + FILE *log = NULL; + char buf[1024]; + + fprintf(report, "Df:\n"); + log = popen("/bin/df -l","r"); + if(log != NULL) + { + while( fgets(buf, sizeof(buf), log) != NULL) + fprintf(report, " %s", buf); + fclose(log); + } + } + + fclose(report); + +} + /*}}}*/ diff --git a/apt-pkg/deb/dpkgpm.h b/apt-pkg/deb/dpkgpm.h index 330c788a2..ce3e20f2e 100644 --- a/apt-pkg/deb/dpkgpm.h +++ b/apt-pkg/deb/dpkgpm.h @@ -33,6 +33,7 @@ class pkgDPkgPM : public pkgPackageManager string dpkg_error; protected: + int pkgFailures; // progress reporting struct DpkgState @@ -49,6 +50,7 @@ class pkgDPkgPM : public pkgPackageManager // the int is the state that is already done (e.g. a package that is // going to be install is already in state "half-installed") map<string,unsigned int> PackageOpsDone; + // progress reporting unsigned int PackagesDone; unsigned int PackagesTotal; @@ -70,6 +72,9 @@ class pkgDPkgPM : public pkgPackageManager bool SendV2Pkgs(FILE *F); void WriteHistoryTag(string tag, string value); + // apport integration + void WriteApportReport(const char *pkgpath, const char *errormsg); + // dpkg log bool OpenLog(); bool CloseLog(); diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index 60281deb3..21472eb3b 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -51,6 +51,7 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.Set("Dir::State::lists","lists/"); Cnf.Set("Dir::State::cdroms","cdroms.list"); + Cnf.Set("Dir::State::mirrors","mirrors/"); // Cache Cnf.Set("Dir::Cache","var/cache/apt/"); diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 0d4999ee7..e1f576875 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -409,6 +409,7 @@ static const char *iTFRewritePackageOrder[] = { "Section", "Installed-Size", "Maintainer", + "Original-Maintainer", "Architecture", "Source", "Version", @@ -438,6 +439,7 @@ static const char *iTFRewriteSourceOrder[] = {"Package", "Priority", "Section", "Maintainer", + "Original-Maintainer", "Build-Depends", "Build-Depends-Indep", "Build-Conflicts", diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc index 0c9aab28c..8b9eacae6 100644 --- a/cmdline/apt-cdrom.cc +++ b/cmdline/apt-cdrom.cc @@ -147,7 +147,7 @@ bool DoAdd(CommandLine &) pkgCdrom cdrom; bool res = true; - bool AutoDetect = _config->FindB("Acquire::cdrom::AutoDetect"); + bool AutoDetect = _config->FindB("Acquire::cdrom::AutoDetect", true); unsigned int count = 0; if (AutoDetect && UdevCdroms.Dlopen()) diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 44235e358..eaf04eee7 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -2306,6 +2306,33 @@ bool DoSource(CommandLine &CmdL) if (Last == 0) return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); + string srec = Last->AsStr(); + string::size_type pos = srec.find("\nVcs-"); + while (pos != string::npos) + { + pos += strlen("\nVcs-"); + string vcs = srec.substr(pos,srec.find(":",pos)-pos); + if(vcs == "Browser") + { + pos = srec.find("\nVcs-", pos); + continue; + } + pos += vcs.length()+2; + string::size_type epos = srec.find("\n", pos); + string uri = srec.substr(pos,epos-pos).c_str(); + ioprintf(c1out, _("NOTICE: '%s' packaging is maintained in " + "the '%s' version control system at:\n" + "%s\n"), + Src.c_str(), vcs.c_str(), uri.c_str()); + if(vcs == "Bzr") + ioprintf(c1out,_("Please use:\n" + "bzr get %s\n" + "to retrieve the latest (possibly unreleased) " + "updates to the package.\n"), + uri.c_str()); + break; + } + // Back track vector<pkgSrcRecords::File> Lst; if (Last->Files(Lst) == false) @@ -2968,7 +2995,6 @@ int main(int argc,const char *argv[]) /*{{{*/ {"remove",&DoInstall}, {"purge",&DoInstall}, {"autoremove",&DoInstall}, - {"purge",&DoInstall}, {"markauto",&DoMarkAuto}, {"unmarkauto",&DoMarkAuto}, {"dist-upgrade",&DoDistUpgrade}, diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure new file mode 100755 index 000000000..7c390642a --- /dev/null +++ b/cmdline/apt-report-mirror-failure @@ -0,0 +1,29 @@ +#!/usr/bin/python +# +# This is a stub that is meant to support failure reporting of +# mirrors to a central database +# +# its currently not used + +import sys +import urllib +import apt_pkg + +apt_pkg.init() +url = apt_pkg.Config.find("Acquire::Mirror::ReportFailures", "") + #"http://people.ubuntu.com:9000/mirror-failure") + #"http://localhost:9000/mirror-failure") +if not url: + sys.exit(0) + +print "Reporting mirror failure to '%s'" % url + +data = {} +data['mirror'] = sys.argv[1] +data['failurl'] = sys.argv[2] +data['error'] = sys.argv[3] +f = urllib.urlopen(url, urllib.urlencode(data)) +f.read() +f.close() + + diff --git a/cmdline/makefile b/cmdline/makefile index 3260e375b..917ccc96a 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -58,3 +58,9 @@ SOURCE=apt-mark TO=$(BIN) TARGET=program include $(COPY_H) + +# The apt-report-mirror-failure program +#SOURCE=apt-report-mirror-failure +#TO=$(BIN) +#TARGET=program +#include $(COPY_H) diff --git a/configure.in b/configure.in index 5e7122f33..8e5c636f7 100644 --- a/configure.in +++ b/configure.in @@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib) AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in) dnl -- SET THIS TO THE RELEASE VERSION -- -AC_DEFINE_UNQUOTED(VERSION,"0.7.26~exp4") +AC_DEFINE_UNQUOTED(VERSION,"0.7.26~exp6") PACKAGE="apt" AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE") AC_SUBST(PACKAGE) diff --git a/debian/apt.conf.autoremove b/debian/apt.conf.autoremove index b41be8397..2f00b9f8b 100644 --- a/debian/apt.conf.autoremove +++ b/debian/apt.conf.autoremove @@ -1,9 +1,23 @@ APT { NeverAutoRemove - { + { + "^linux-firmware$"; "^linux-image.*"; "^linux-restricted-modules.*"; - "^kfreebsd-image.*"; + "^linux-ubuntu-modules-.*"; + }; + + Never-MarkAuto-Sections + { + "metapackages"; + "restricted/metapackages"; + "universe/metapackages"; + "multiverse/metapackages"; + "oldlibs"; + "restricted/oldlibs"; + "universe/oldlibs"; + "multiverse/oldlibs"; + }; }; diff --git a/debian/apt.dirs b/debian/apt.dirs index 66556e453..f25e4600b 100644 --- a/debian/apt.dirs +++ b/debian/apt.dirs @@ -9,6 +9,7 @@ etc/apt/trusted.gpg.d etc/logrotate.d var/cache/apt/archives/partial var/lib/apt/lists/partial +var/lib/apt/mirrors/partial var/lib/apt/periodic var/log/apt usr/share/bug/apt diff --git a/debian/changelog b/debian/changelog index 3771ca415..1740d196c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,28 @@ +apt (0.7.26~exp6) experimental; urgency=low + + [ Michael Vogt ] + * merge the remaining Ubuntu change: + - on gpg verification failure warn and restore the last known + good state + - on failure display the IP of the server (useful for servers + that use round robin DNS) + - support Original-Maintainer in RewritePackageOrder + - enable cdrom autodetection via libudev by default + - show messsage about Vcs in use when apt-get source is run for + packages maintained in a Vcs + - better support transitional packages with mark auto-installed. + when the transitional package is in "oldlibs" the new package + is not marked auto installed (same is true for section + metapackages) + - provide new "deb mirror://archive.foo/mirrors.list sid main" + method expects a list of mirrors (generated on the server e.g. + via geoip) and will use that, including cycle on failure + - write apport crash file on package failure (disabled by default + on debian until apport is available) + - support mirror failure reporting (disabled by default on debian) + + -- Michael Vogt <mvo@debian.org> Wed, 09 Jun 2010 10:50:17 +0200 + apt (0.7.26~exp5) experimental; urgency=low [ David Kalnischkies ] diff --git a/debian/control b/debian/control index 4f41c130b..9ac0d582e 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Uploaders: Michael Vogt <mvo@debian.org>, Otavio Salvador <otavio@debian.org>, Christian Perrier <bubulle@debian.org>, Daniel Burrows <dburrows@debian.org>, Luca Bruno <lethalman88@gmail.com>, Julian Andres Klode <jak@debian.org> Standards-Version: 3.8.4 -Build-Depends: debhelper (>= 5.0), libdb-dev, gettext (>= 0.12), libcurl4-gnutls-dev | libcurl3-gnutls-dev (>= 7.15.5), debiandoc-sgml, xsltproc, docbook-xsl, po4a (>= 0.34-2), autotools-dev, autoconf, automake, doxygen +Build-Depends: debhelper (>= 5.0), libdb-dev, gettext (>= 0.12), libcurl4-gnutls-dev | libcurl3-gnutls-dev (>= 7.15.5), debiandoc-sgml, xsltproc, docbook-xsl, po4a (>= 0.34-2), autotools-dev, autoconf, automake, doxygen, intltool Build-Conflicts: autoconf2.13, automake1.4 Package: apt diff --git a/debian/rules b/debian/rules index 827fc2034..4c6795910 100755 --- a/debian/rules +++ b/debian/rules @@ -215,6 +215,8 @@ apt: build build-doc debian/shlibs.local cp debian/bugscript debian/$@/usr/share/bug/apt/script cp debian/apt.logrotate debian/$@/etc/logrotate.d/apt + cp share/ubuntu-archive.gpg debian/$@/usr/share/$@ + sed 's/^_//' share/apt-auth-failure.note > debian/$@/usr/share/$@/apt-auth-failure.note cp debian/apt.conf.autoremove debian/$@/etc/apt/apt.conf.d/01autoremove # copy lintian override @@ -225,6 +227,10 @@ apt: build build-doc debian/shlibs.local rm -f build/po/*.pot rm -f po/*.pot + # move the mirror failure script in place + #mv debian/$@/usr/bin/apt-report-mirror-failure \ + # debian/$@/usr/lib/apt/apt-report-mirror-failure \ + dh_installexamples -p$@ $(BLD)/docs/examples/* dh_installman -p$@ $(wildcard $(patsubst %,doc/%.[158],$(apt_MANPAGES)) $(patsubst %,doc/*/%.*.[158],$(apt_MANPAGES))) dh_installcron -p$@ diff --git a/methods/connect.cc b/methods/connect.cc index 2f6b4833e..a5af1f1a6 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -18,6 +18,7 @@ #include <stdio.h> #include <errno.h> #include <unistd.h> +#include <sstream> #include<set> #include<string> @@ -70,19 +71,17 @@ static bool DoConnect(struct addrinfo *Addr,string Host, Owner->Status(_("Connecting to %s (%s)"),Host.c_str(),Name); // if that addr did timeout before, we do not try it again - if(bad_addr.find(string(Name)) != bad_addr.end()) + if(bad_addr.find(string(Name)) != bad_addr.end()) return false; /* If this is an IP rotation store the IP we are using.. If something goes wrong this will get tacked onto the end of the error message */ if (LastHostAddr->ai_next != 0) { - char Name2[NI_MAXHOST + NI_MAXSERV + 10]; - snprintf(Name2,sizeof(Name2),_("[IP: %s %s]"),Name,Service); - Owner->SetFailExtraMsg(string(Name2)); - } - else - Owner->SetFailExtraMsg(""); + std::stringstream ss; + ioprintf(ss, _("[IP: %s %s]"),Name,Service); + Owner->SetIP(ss.str()); + } // Get a socket if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, @@ -100,7 +99,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, nonblocking */ if (WaitFd(Fd,true,TimeOut) == false) { bad_addr.insert(bad_addr.begin(), string(Name)); - Owner->SetFailExtraMsg("\nFailReason: Timeout"); + Owner->SetFailReason("Timeout"); return _error->Error(_("Could not connect to %s:%s (%s), " "connection timed out"),Host.c_str(),Service,Name); } @@ -115,9 +114,9 @@ static bool DoConnect(struct addrinfo *Addr,string Host, { errno = Err; if(errno == ECONNREFUSED) - Owner->SetFailExtraMsg("\nFailReason: ConnectionRefused"); + Owner->SetFailReason("ConnectionRefused"); else if (errno == ETIMEDOUT) - Owner->SetFailExtraMsg("\nFailReason: ConnectionTimedOut"); + Owner->SetFailReason("ConnectionTimedOut"); bad_addr.insert(bad_addr.begin(), string(Name)); return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(), Service,Name); @@ -184,13 +183,13 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, continue; } bad_addr.insert(bad_addr.begin(), Host); - Owner->SetFailExtraMsg("\nFailReason: ResolveFailure"); + Owner->SetFailReason("ResolveFailure"); return _error->Error(_("Could not resolve '%s'"),Host.c_str()); } if (Res == EAI_AGAIN) { - Owner->SetFailExtraMsg("\nFailReason: TmpResolveFailure"); + Owner->SetFailReason("TmpResolveFailure"); return _error->Error(_("Temporary failure resolving '%s'"), Host.c_str()); } diff --git a/methods/copy.cc b/methods/copy.cc index 72896b4c0..027b59f46 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -84,6 +84,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) FileFd Fd(Res.Filename, FileFd::ReadOnly); Hash.AddFD(Fd.Fd(), Fd.Size()); Res.TakeHashes(Hash); + URIDone(Res); return true; } diff --git a/methods/http.cc b/methods/http.cc index d43dd14c8..3e2227f2b 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -953,6 +953,9 @@ HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) failure */ if (Srv->Result < 200 || Srv->Result >= 300) { + char err[255]; + snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result); + SetFailReason(err); _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) return ERROR_WITH_CONTENT_PAGE; @@ -1366,15 +1369,4 @@ bool HttpMethod::AutoDetectProxy() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - - HttpMethod Mth; - return Mth.Loop(); -} - diff --git a/methods/http.h b/methods/http.h index af0f5e033..d0677bdaa 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,7 +13,7 @@ #define MAXLEN 360 -#include <iostream> + using std::cout; using std::endl; @@ -167,7 +167,6 @@ class HttpMethod : public pkgAcqMethod /** \brief Try to AutoDetect the proxy */ bool AutoDetectProxy(); - virtual bool Fetch(FetchItem *); virtual bool Configuration(string Message); // In the event of a fatal signal this file will be closed and timestamped. @@ -175,6 +174,9 @@ class HttpMethod : public pkgAcqMethod static int FailFd; static time_t FailTime; static void SigTerm(int); + + protected: + virtual bool Fetch(FetchItem *); string NextURI; string AutoDetectProxyCmd; diff --git a/methods/http_main.cc b/methods/http_main.cc new file mode 100644 index 000000000..7815c2fc1 --- /dev/null +++ b/methods/http_main.cc @@ -0,0 +1,20 @@ +#include <apt-pkg/fileutl.h> +#include <apt-pkg/acquire-method.h> +#include <signal.h> + +#include "connect.h" +#include "rfc2553emu.h" +#include "http.h" + + +int main() +{ + setlocale(LC_ALL, ""); + + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + + HttpMethod Mth; + return Mth.Loop(); +} diff --git a/methods/makefile b/methods/makefile index 7bcae6b9b..eabe85cfd 100644 --- a/methods/makefile +++ b/methods/makefile @@ -48,7 +48,7 @@ include $(PROGRAM_H) PROGRAM=http SLIBS = -lapt-pkg $(SOCKETLIBS) $(INTLLIBS) LIB_MAKES = apt-pkg/makefile -SOURCE = http.cc rfc2553emu.cc connect.cc +SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc include $(PROGRAM_H) # The https method @@ -79,9 +79,17 @@ LIB_MAKES = apt-pkg/makefile SOURCE = rsh.cc include $(PROGRAM_H) +# The mirror method +PROGRAM=mirror +SLIBS = -lapt-pkg $(SOCKETLIBS) +LIB_MAKES = apt-pkg/makefile +SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc +include $(PROGRAM_H) + # SSH and bzip2 method symlink binary: $(BIN)/ssh $(BIN)/bzip2 $(BIN)/lzma veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 clean-$(BIN)/lzma + $(BIN)/ssh: echo "Installing ssh method link" ln -fs rsh $(BIN)/ssh diff --git a/methods/mirror.cc b/methods/mirror.cc new file mode 100644 index 000000000..b3a956b95 --- /dev/null +++ b/methods/mirror.cc @@ -0,0 +1,330 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ +/* ###################################################################### + + Mirror Aquire Method - This is the Mirror aquire method for APT. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <apt-pkg/fileutl.h> +#include <apt-pkg/acquire-method.h> +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire.h> +#include <apt-pkg/error.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/sourcelist.h> + +#include <fstream> +#include <iostream> +#include <stdarg.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <dirent.h> + +using namespace std; + +#include "mirror.h" +#include "http.h" +#include "apti18n.h" + /*}}}*/ + +/* Done: + * - works with http (only!) + * - always picks the first mirror from the list + * - call out to problem reporting script + * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * - uses pkgAcqMethod::FailReason() to have a string representation + * of the failure that is also send to LP + * + * TODO: + * - deal with runing as non-root because we can't write to the lists + dir then -> use the cached mirror file + * - better method to download than having a pkgAcquire interface here + * and better error handling there! + * - support more than http + * - testing :) + */ + +MirrorMethod::MirrorMethod() + : HttpMethod(), DownloadedMirrorFile(false) +{ +}; + +// HttpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool MirrorMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + Debug = _config->FindB("Debug::Acquire::mirror",false); + + return true; +} + /*}}}*/ + +// clean the mirrors dir based on ttl information +bool MirrorMethod::Clean(string Dir) +{ + vector<metaIndex *>::const_iterator I; + + if(Debug) + clog << "MirrorMethod::Clean(): " << Dir << endl; + + if(Dir == "/") + return _error->Error("will not clean: '/'"); + + // read sources.list + pkgSourceList list; + list.ReadMainList(); + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + string StartDir = SafeGetCWD(); + if (chdir(Dir.c_str()) != 0) + { + closedir(D); + return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str()); + } + + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,"lock") == 0 || + strcmp(Dir->d_name,"partial") == 0 || + strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0) + continue; + + // see if we have that uri + for(I=list.begin(); I != list.end(); I++) + { + string uri = (*I)->GetURI(); + if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + string BaseUri = uri.substr(0,uri.size()-1); + if (URItoFileName(BaseUri) == Dir->d_name) + break; + } + // nothing found, nuke it + if (I == list.end()) + unlink(Dir->d_name); + }; + + chdir(StartDir.c_str()); + closedir(D); + return true; +} + + +bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) +{ + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile(): " << endl; + + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + DownloadedMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); + + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + if(res) + DownloadedMirrorFile = true; + Fetcher.Shutdown(); + return res; +} + +bool MirrorMethod::SelectMirror() +{ + // if we do not have a MirrorFile, fallback + if(!FileExists(MirrorFile)) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + } + + // FIXME: make the mirror selection more clever, do not + // just use the first one! + // BUT: we can not make this random, the mirror has to be + // stable accross session, because otherwise we can + // get into sync issues (got indexfiles from mirror A, + // but packages from mirror B - one might be out of date etc) + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +string MirrorMethod::GetMirrorFileName(string mirror_uri_str) +{ + /* + - a mirror_uri_str looks like this: + mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg + + - the matching source.list entry + deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main + + - we actually want to go after: + http://people.ubuntu.com/~mvo/apt/mirror/mirrors + + And we need to save the BaseUri for later: + - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors + + FIXME: what if we have two similar prefixes? + mirror://people.ubuntu.com/~mvo/mirror + mirror://people.ubuntu.com/~mvo/mirror2 + then mirror_uri_str looks like: + mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg + mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg + we search sources.list and find: + mirror://people.ubuntu.com/~mvo/apt/mirror + in both cases! So we need to apply some domain knowledge here :( and + check for /dists/ or /Release.gpg as suffixes + */ + string name; + if(Debug) + std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; + + // read sources.list and find match + vector<metaIndex *>::const_iterator I; + pkgSourceList list; + list.ReadMainList(); + for(I=list.begin(); I != list.end(); I++) + { + string uristr = (*I)->GetURI(); + if(Debug) + std::cerr << "Checking: " << uristr << std::endl; + if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + // find matching uri in sources.list + if(mirror_uri_str.substr(0,uristr.size()) == uristr) + { + if(Debug) + std::cerr << "found BaseURI: " << uristr << std::endl; + BaseUri = uristr.substr(0,uristr.size()-1); + } + } + // get new file + name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << name << endl; + } + return name; +} + +// MirrorMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool MirrorMethod::Fetch(FetchItem *Itm) +{ + if(Debug) + clog << "MirrorMethod::Fetch()" << endl; + + // the http method uses Fetch(0) as a way to update the pipeline, + // just let it do its work in this case - Fetch() with a valid + // Itm will always run before the first Fetch(0) + if(Itm == NULL) + return HttpMethod::Fetch(Itm); + + // if we don't have the name of the mirror file on disk yet, + // calculate it now (can be derived from the uri) + if(MirrorFile.empty()) + MirrorFile = GetMirrorFileName(Itm->Uri); + + // download mirror file once (if we are after index files) + if(Itm->IndexFile && !DownloadedMirrorFile) + { + Clean(_config->FindDir("Dir::State::mirrors")); + DownloadMirrorFile(Itm->Uri); + } + + if(Mirror.empty()) { + if(!SelectMirror()) { + // no valid mirror selected, something went wrong downloading + // from the master mirror site most likely and there is + // no old mirror file availalbe + return false; + } + } + if(Debug) + clog << "selected mirror: " << Mirror << endl; + + + for (FetchItem *I = Queue; I != 0; I = I->Next) + { + if(I->Uri.find("mirror://") != string::npos) + I->Uri.replace(0,BaseUri.size(), Mirror); + } + + // now run the real fetcher + return HttpMethod::Fetch(Itm); +}; + +void MirrorMethod::Fail(string Err,bool Transient) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::Fail(Err, Transient); +} + +void MirrorMethod::URIStart(FetchResult &Res) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIStart(Res); +} + +void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIDone(Res, Alt); +} + + +int main() +{ + setlocale(LC_ALL, ""); + + MirrorMethod Mth; + + return Mth.Loop(); +} + + diff --git a/methods/mirror.h b/methods/mirror.h new file mode 100644 index 000000000..ed817806b --- /dev/null +++ b/methods/mirror.h @@ -0,0 +1,52 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +/* ###################################################################### + + MIRROR Aquire Method - This is the MIRROR aquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_MIRROR_H +#define APT_MIRROR_H + + +#include <iostream> + +using std::cout; +using std::cerr; +using std::endl; + +#include "http.h" + +class MirrorMethod : public HttpMethod +{ + FetchResult Res; + // we simply transform between BaseUri and Mirror + string BaseUri; // the original mirror://... url + string Mirror; // the selected mirror uri (http://...) + string MirrorFile; // the file that contains the list of mirrors + bool DownloadedMirrorFile; // already downloaded this session + + bool Debug; + + protected: + bool DownloadMirrorFile(string uri); + string GetMirrorFileName(string uri); + bool SelectMirror(); + bool Clean(string dir); + + // we need to overwrite those to transform the url back + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual bool Configuration(string Message); + + public: + MirrorMethod(); + virtual bool Fetch(FetchItem *Itm); +}; + + +#endif diff --git a/mirror-failure.py b/mirror-failure.py new file mode 100644 index 000000000..e7d2bbf54 --- /dev/null +++ b/mirror-failure.py @@ -0,0 +1,23 @@ +# File: cgihttpserver-example-1.py + +import CGIHTTPServer +import BaseHTTPServer + +class Handler(CGIHTTPServer.CGIHTTPRequestHandler): + #cgi_directories = ["/cgi"] + def do_POST(self): + print "do_POST" + #print self.command + #print self.path + #print self.headers + print self.client_address + data = self.rfile.read(int(self.headers["content-length"])) + print data + self.wfile.write("200 Ok\n"); + +PORT = 8000 + +httpd = BaseHTTPServer.HTTPServer(("", PORT), Handler) +print "serving at port", PORT +httpd.serve_forever() + diff --git a/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages new file mode 100644 index 000000000..3e7265438 --- /dev/null +++ b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages @@ -0,0 +1,25 @@ +Package: libglib2.0-data +Priority: optional +Section: misc +Installed-Size: 2288 +Maintainer: Ubuntu Desktop Team <ubuntu-desktop@lists.ubuntu.com> +Original-Maintainer: Loic Minier <lool@dooz.org> +Architecture: all +Source: glib2.0 +Version: 2.13.6-1ubuntu1 +Replaces: libglib1.3, libglib1.3-data +Depends: libglib2.0-0 (>= 2.13.6-1ubuntu1) +Conflicts: libglib1.3-data +Filename: ./libglib2.0-data_2.13.6-1ubuntu1_all.deb +Size: 958 +MD5sum: 803fc5e2e31a4345b3e9c771e1eae49f +SHA1: 75b2c62b21bae60c58e694dd40ed6d4df946e304 +SHA256: 142d8466eac252f06bc957d76fe1bb87f86f2d3512b99c8d4b08c1ad79fbe59e +Description: Common files for GLib library + GLib is a library containing many useful C routines for things such + as trees, hashes, lists, and strings. It is a useful general-purpose + C library used by projects such as GTK+, GIMP, and GNOME. + . + This package is needed for the runtime libraries to display messages in + languages other than English. + diff --git a/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release new file mode 100644 index 000000000..7ecd4cd19 --- /dev/null +++ b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release @@ -0,0 +1,13 @@ +Date: Fri, 27 Jul 2007 14:39:41 UTC +MD5Sum: + 4672dadea6a144839f823c9f3d5fd44b 934 Packages + 82ebcf09a8d78a2b9cf7759349da4936 603 Packages.gz + d41d8cd98f00b204e9800998ecf8427e 0 Release +SHA1: + fa0f294aa30789529371066b10e9497be1284d26 934 Packages + f4032808663b2810d87b4a4dab6f5ae4a1e8fa8e 603 Packages.gz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 Release +SHA256: + 92c9b605480dc74e6be79c0ddc24738bfcbd6dd3148af531acd68717de528049 934 Packages + 659ccc0d07ff21f0247f9fa5abe149221c90d5e17da52c7afddb035b93c23d39 603 Packages.gz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 Release diff --git a/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg new file mode 100644 index 000000000..85c356e6f --- /dev/null +++ b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg @@ -0,0 +1,7 @@ +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQBGqgOwliSD4VZixzQRAs6jAJ9p7Aiob9gzkUNCtoW8UPrBo0E/YwCdEaz0 +CQJszU6fRYX5jGWXSWzfc5c= +=ugH0 +-----END PGP SIGNATURE----- diff --git a/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages new file mode 100644 index 000000000..3e7265438 --- /dev/null +++ b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages @@ -0,0 +1,25 @@ +Package: libglib2.0-data +Priority: optional +Section: misc +Installed-Size: 2288 +Maintainer: Ubuntu Desktop Team <ubuntu-desktop@lists.ubuntu.com> +Original-Maintainer: Loic Minier <lool@dooz.org> +Architecture: all +Source: glib2.0 +Version: 2.13.6-1ubuntu1 +Replaces: libglib1.3, libglib1.3-data +Depends: libglib2.0-0 (>= 2.13.6-1ubuntu1) +Conflicts: libglib1.3-data +Filename: ./libglib2.0-data_2.13.6-1ubuntu1_all.deb +Size: 958 +MD5sum: 803fc5e2e31a4345b3e9c771e1eae49f +SHA1: 75b2c62b21bae60c58e694dd40ed6d4df946e304 +SHA256: 142d8466eac252f06bc957d76fe1bb87f86f2d3512b99c8d4b08c1ad79fbe59e +Description: Common files for GLib library + GLib is a library containing many useful C routines for things such + as trees, hashes, lists, and strings. It is a useful general-purpose + C library used by projects such as GTK+, GIMP, and GNOME. + . + This package is needed for the runtime libraries to display messages in + languages other than English. + diff --git a/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release new file mode 100644 index 000000000..7ecd4cd19 --- /dev/null +++ b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release @@ -0,0 +1,13 @@ +Date: Fri, 27 Jul 2007 14:39:41 UTC +MD5Sum: + 4672dadea6a144839f823c9f3d5fd44b 934 Packages + 82ebcf09a8d78a2b9cf7759349da4936 603 Packages.gz + d41d8cd98f00b204e9800998ecf8427e 0 Release +SHA1: + fa0f294aa30789529371066b10e9497be1284d26 934 Packages + f4032808663b2810d87b4a4dab6f5ae4a1e8fa8e 603 Packages.gz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 Release +SHA256: + 92c9b605480dc74e6be79c0ddc24738bfcbd6dd3148af531acd68717de528049 934 Packages + 659ccc0d07ff21f0247f9fa5abe149221c90d5e17da52c7afddb035b93c23d39 603 Packages.gz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 Release diff --git a/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg new file mode 100644 index 000000000..85c356e6f --- /dev/null +++ b/test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg @@ -0,0 +1,7 @@ +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQBGqgOwliSD4VZixzQRAs6jAJ9p7Aiob9gzkUNCtoW8UPrBo0E/YwCdEaz0 +CQJszU6fRYX5jGWXSWzfc5c= +=ugH0 +-----END PGP SIGNATURE----- diff --git a/test/authReliability/sources.list.failure b/test/authReliability/sources.list.failure new file mode 100644 index 000000000..110f31884 --- /dev/null +++ b/test/authReliability/sources.list.failure @@ -0,0 +1,2 @@ +deb http://people.ubuntu.com/~mvo/apt/auth-test-suit/gpg-package-broken/ / + diff --git a/test/authReliability/sources.list.good b/test/authReliability/sources.list.good new file mode 100644 index 000000000..2e9a4458a --- /dev/null +++ b/test/authReliability/sources.list.good @@ -0,0 +1,2 @@ +deb http://people.ubuntu.com/~mvo/apt/auth-test-suit/gpg-package-ok/ / + diff --git a/test/pre-upload-check.py b/test/pre-upload-check.py index 268b3d672..97a0ec4c4 100755 --- a/test/pre-upload-check.py +++ b/test/pre-upload-check.py @@ -4,6 +4,8 @@ import sys import os import glob import os.path +import shutil +import time from subprocess import call, PIPE import unittest @@ -11,7 +13,102 @@ import unittest stdout = os.open("/dev/null",0) #sys.stdout stderr = os.open("/dev/null",0) # sys.stderr -apt_args = [] # ["-o","Debug::pkgAcquire::Auth=true"] +apt_args = [] +#apt_args = ["-o","Debug::pkgAcquire::Auth=true"] + +class testAptAuthenticationReliability(unittest.TestCase): + """ + test if the spec https://wiki.ubuntu.com/AptAuthenticationReliability + is properly implemented + """ + #apt = "../bin/apt-get" + apt = "apt-get" + + def setUp(self): + if os.path.exists("/tmp/autFailure"): + os.unlink("/tmp/authFailure"); + if os.path.exists("/tmp/autFailure2"): + os.unlink("/tmp/authFailure2"); + def testRepositorySigFailure(self): + """ + test if a repository that used to be authenticated and fails on + apt-get update refuses to update and uses the old state + """ + # copy valid signatures into lists (those are ok, even + # if the name is "-broken-" ... + for f in glob.glob("./authReliability/lists/*"): + shutil.copy(f,"/var/lib/apt/lists") + # ensure we do *not* get a I-M-S hit + os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (0,0)) + res = call([self.apt, + "update", + "-o","Dir::Etc::sourcelist=./authReliability/sources.list.failure", + "-o",'APT::Update::Auth-Failure::=touch /tmp/authFailure', + ] + apt_args, + stdout=stdout, stderr=stderr) + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg"), + "The gpg file disappeared, this should not happen") + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages"), + "The Packages file disappeared, this should not happen") + self.assert_(os.path.exists("/tmp/authFailure"), + "The APT::Update::Auth-Failure script did not run (1)") + # the same with i-m-s hit this time + for f in glob.glob("./authReliability/lists/*"): + shutil.copy(f,"/var/lib/apt/lists") + os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (time.time(),time.time())) + res = call([self.apt, + "update", + "-o","Dir::Etc::sourcelist=./authReliability/sources.list.failure", + "-o",'APT::Update::Auth-Failure::=touch /tmp/authFailure2', + ] + apt_args, + stdout=stdout, stderr=stderr) + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg"), + "The gpg file disappeared, this should not happen") + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages"), + "The Packages file disappeared, this should not happen") + self.assert_(os.path.exists("/tmp/authFailure2"), + "The APT::Update::Auth-Failure script did not run (2)") + def testRepositorySigGood(self): + """ + test that a regular repository with good data stays good + """ + res = call([self.apt, + "update", + "-o","Dir::Etc::sourcelist=./authReliability/sources.list.good" + ] + apt_args, + stdout=stdout, stderr=stderr) + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg"), + "The gpg file disappeared after a regular download, this should not happen") + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages"), + "The Packages file disappeared, this should not happen") + # test good is still good after non I-M-S hit and a previous files in lists/ + for f in glob.glob("./authReliability/lists/*"): + shutil.copy(f,"/var/lib/apt/lists") + # ensure we do *not* get a I-M-S hit + os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (0,0)) + res = call([self.apt, + "update", + "-o","Dir::Etc::sourcelist=./authReliability/sources.list.good" + ] + apt_args, + stdout=stdout, stderr=stderr) + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg"), + "The gpg file disappeared after a I-M-S hit, this should not happen") + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages"), + "The Packages file disappeared, this should not happen") + # test good is still good after I-M-S hit + for f in glob.glob("./authReliability/lists/*"): + shutil.copy(f,"/var/lib/apt/lists") + # ensure we do get a I-M-S hit + os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (time.time(),time.time())) + res = call([self.apt, + "update", + "-o","Dir::Etc::sourcelist=./authReliability/sources.list.good" + ] + apt_args, + stdout=stdout, stderr=stderr) + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg"), + "The gpg file disappeared, this should not happen") + self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages"), + "The Packages file disappeared, this should not happen") class testAuthentication(unittest.TestCase): @@ -149,6 +246,7 @@ if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "-v": stdout = sys.stdout stderr = sys.stderr + + # run only one for now + #unittest.main(defaultTest="testAptAuthenticationReliability") unittest.main() - - |