From 196fd13629fc0f673b18735d633bdf69a3272590 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Mar 2011 11:34:32 +0100 Subject: * mirror method: - when downloading data, show the mirror being used --- apt-pkg/acquire-method.cc | 17 ++++++++++++----- apt-pkg/acquire-worker.cc | 11 +++++++++++ debian/changelog | 2 ++ methods/mirror.cc | 4 ++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 17d52cf51..29d53f2ac 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -149,6 +149,8 @@ void pkgAcqMethod::URIStart(FetchResult &Res) if (Res.ResumePoint != 0) End += snprintf(End,sizeof(S)-4 - (End - S),"Resume-Point: %lu\n", Res.ResumePoint); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); strcat(End,"\n"); if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) @@ -418,9 +420,11 @@ void pkgAcqMethod::Log(const char *Format,...) // sprintf the description char S[1024]; - unsigned int Len = snprintf(S,sizeof(S)-4,"101 Log\nURI: %s\n" - "Message: ",CurrentURI.c_str()); - + unsigned int Len = snprintf(S,sizeof(S)-4,"101 Log\n" + "URI: %s\n" + "UsedMirror: %s\n" + "Message: ", UsedMirror.c_str(), + CurrentURI.c_str()); vsnprintf(S+Len,sizeof(S)-4-Len,Format,args); strcat(S,"\n\n"); @@ -442,8 +446,11 @@ void pkgAcqMethod::Status(const char *Format,...) // sprintf the description char S[1024]; - unsigned int Len = snprintf(S,sizeof(S)-4,"102 Status\nURI: %s\n" - "Message: ",CurrentURI.c_str()); + unsigned int Len = snprintf(S,sizeof(S)-4,"102 Status\n" + "URI: %s\n" + "UsedMirror: %s\n" + "Message: ",UsedMirror.c_str(), + CurrentURI.c_str()); vsnprintf(S+Len,sizeof(S)-4-Len,Format,args); strcat(S,"\n\n"); diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 4f0b52af9..ddd8e3101 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -199,6 +199,17 @@ bool pkgAcquire::Worker::RunMessages() pkgAcquire::Queue::QItem *Itm = 0; if (URI.empty() == false) Itm = OwnerQ->FindItem(URI,this); + + // update used mirror + string UsedMirror = LookupTag(Message,"UsedMirror", ""); + if (!UsedMirror.empty() && + Itm && + Itm->Description.find(" ") != string::npos) + { + Itm->Description.replace(0, Itm->Description.find(" "), UsedMirror); + // FIXME: will we need this as well? + //Itm->ShortDesc = UsedMirror; + } // Determine the message number and dispatch switch (Number) diff --git a/debian/changelog b/debian/changelog index 6176aa87e..efa7be763 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,8 @@ apt (0.8.13) unstable; urgency=low * apt-pkg/acquire-item.cc: - mark pkgAcqIndexTrans as Index-File to avoid asking the user to insert the CD on each apt-get update + * mirror method: + - when downloading data, show the mirror being used -- Michael Vogt Thu, 10 Mar 2011 15:56:54 +0100 diff --git a/methods/mirror.cc b/methods/mirror.cc index d092cc657..5a53d3c81 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -186,6 +186,10 @@ bool MirrorMethod::TryNextMirror() Queue->Uri.replace(0, mirror->length(), *nextmirror); if (Debug) clog << "TryNextMirror: " << Queue->Uri << endl; + + // inform parent + UsedMirror = *nextmirror; + Log("Switching mirror"); return true; } -- cgit v1.2.3 From 8d6c583900b81bf996afd104571580c6f3e55e1c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Mar 2011 15:08:52 +0100 Subject: apt-pkg/acquire-item.{cc,h}: mark InRelease with Fail-Ignore to ensure the mirror methods does not retry on each mirror --- apt-pkg/acquire-item.cc | 15 +++++++++++++++ apt-pkg/acquire-item.h | 1 + 2 files changed, 16 insertions(+) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 497edbaac..6912eae87 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1538,6 +1538,21 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ SigFile = DestFile; } /*}}}*/ +// pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +// FIXME: this can go away once the InRelease file is used widely +string pkgAcqMetaClearSig::Custom600Headers() +{ + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(RealURI); + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true\nFail-Ignore: true\n"; + + return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} + /*}}}*/ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { if (AuthPass == false) diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 2ae7bf27c..f763577ee 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -854,6 +854,7 @@ class pkgAcqMetaClearSig : public pkgAcqMetaIndex public: void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual string Custom600Headers(); /** \brief Create a new pkgAcqMetaClearSig. */ pkgAcqMetaClearSig(pkgAcquire *Owner, -- cgit v1.2.3 From 01a695e260429b4ed834a52fb55ba49a8cb42e6a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Mar 2011 15:44:08 +0100 Subject: randomize mirror list to ensure more even load --- debian/changelog | 1 + methods/mirror.cc | 30 ++++++++++++++++++++++++++++++ methods/mirror.h | 1 + 3 files changed, 32 insertions(+) diff --git a/debian/changelog b/debian/changelog index efa7be763..d9a20765f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -13,6 +13,7 @@ apt (0.8.13) unstable; urgency=low user to insert the CD on each apt-get update * mirror method: - when downloading data, show the mirror being used + - randomize mirror list to ensure more even load -- Michael Vogt Thu, 10 Mar 2011 15:56:54 +0100 diff --git a/methods/mirror.cc b/methods/mirror.cc index 5a53d3c81..08a603207 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -147,6 +148,34 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } +// Randomizes the lines in the mirror file, this is used so that +// we spread the load on the mirrors evenly +bool MirrorMethod::RandomizeMirrorFile(string mirror_file) +{ + vector content; + string line; + + // read + ifstream in(mirror_file.c_str()); + while ( ! in.eof() ) { + getline(in, line); + content.push_back(line); + } + + // randomize + random_shuffle(content.begin(), content.end()); + + // write + ofstream out(mirror_file.c_str()); + while ( !content.empty()) { + line = content.back(); + content.pop_back(); + out << line << "\n"; + } + + return true; +} + /* convert a the Queue->Uri back to the mirror base uri and look * at all mirrors we have for this, this is needed as queue->uri * may point to different mirrors (if TryNextMirror() was run) @@ -313,6 +342,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) { Clean(_config->FindDir("Dir::State::mirrors")); DownloadMirrorFile(Itm->Uri); + RandomizeMirrorFile(MirrorFile); } if(AllMirrors.empty()) { diff --git a/methods/mirror.h b/methods/mirror.h index 0a3ea6e92..bd807e122 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -34,6 +34,7 @@ class MirrorMethod : public HttpMethod protected: bool DownloadMirrorFile(string uri); + bool RandomizeMirrorFile(string file); string GetMirrorFileName(string uri); bool InitMirrors(); bool TryNextMirror(); -- cgit v1.2.3 From 97b65b1041ad684a6dafb601a0926dcd98b5d6a0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Mar 2011 16:27:24 +0100 Subject: apt-pkg/acquire-item.cc: add some more missing Fail-Ignore --- apt-pkg/acquire-item.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 6912eae87..2ecb8ed6e 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -220,8 +220,8 @@ string pkgAcqSubIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) - return "\nIndex-File: true"; - return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + return "\nIndex-File: true\nFail-Ignore: true\n"; + return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ @@ -835,10 +835,16 @@ string pkgAcqIndex::Custom600Headers() if (_config->FindB("Acquire::GzipIndexes",false)) Final += ".gz"; + string msg = "\nIndex-File: true"; + // FIXME: this really should use "IndexTarget::IsOptional()" but that + // seems to be difficult without breaking ABI + if (ShortDesc().find("Translation") != 0) + msg += "\nFail-Ignore: true"; struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) - return "\nIndex-File: true"; - return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + + return msg; } /*}}}*/ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ -- cgit v1.2.3 From 0190e315a34377c658fa3acb5c658a6b4a52ae1e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Mar 2011 16:40:29 +0100 Subject: methods/mirror.cc: init random seed at startup --- methods/mirror.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/methods/mirror.cc b/methods/mirror.cc index 08a603207..ed42cdbfb 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -157,7 +157,7 @@ bool MirrorMethod::RandomizeMirrorFile(string mirror_file) // read ifstream in(mirror_file.c_str()); - while ( ! in.eof() ) { + while ( !in.eof() ) { getline(in, line); content.push_back(line); } @@ -403,6 +403,8 @@ int main() { setlocale(LC_ALL, ""); + srand ( time(NULL) ); + MirrorMethod Mth; return Mth.Loop(); -- cgit v1.2.3 From 78c8f3cd7e0d2a61ee564cb010277bccb232b523 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Mar 2011 18:03:14 +0100 Subject: methods/mirror.cc: randomize only based on hostname --- methods/mirror.cc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/methods/mirror.cc b/methods/mirror.cc index ed42cdbfb..e499b054b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -16,12 +16,15 @@ #include #include -#include + #include +#include #include + #include #include #include +#include #include using namespace std; @@ -162,7 +165,17 @@ bool MirrorMethod::RandomizeMirrorFile(string mirror_file) content.push_back(line); } - // randomize + // we want the file to be random for each different machine, but also + // "stable" on the same machine. this is to avoid running into out-of-sync + // issues (i.e. Release/Release.gpg different on each mirror) + struct utsname buf; + int seed=1, i; + if(uname(&buf) == 0) { + for(i=0,seed=1; buf.nodename[i] != 0; i++) { + seed = seed * 31 + buf.nodename[i]; + } + } + srand( seed ); random_shuffle(content.begin(), content.end()); // write @@ -403,8 +416,6 @@ int main() { setlocale(LC_ALL, ""); - srand ( time(NULL) ); - MirrorMethod Mth; return Mth.Loop(); -- cgit v1.2.3 From ca0a660c3b63717ffec0305bba95af6930e62c34 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 15 Mar 2011 09:07:02 +0100 Subject: apt-pkg/acquire-method.cc: fix copy/paste error --- apt-pkg/acquire-method.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 29d53f2ac..fb45d9ee7 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -150,7 +150,7 @@ void pkgAcqMethod::URIStart(FetchResult &Res) End += snprintf(End,sizeof(S)-4 - (End - S),"Resume-Point: %lu\n", Res.ResumePoint); if (UsedMirror.empty() == false) - End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); + End += snprintf(End,sizeof(S)-4 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); strcat(End,"\n"); if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) -- cgit v1.2.3