diff options
-rw-r--r-- | apt-pkg/acquire-item.cc | 306 | ||||
-rw-r--r-- | apt-pkg/acquire-item.h | 43 | ||||
-rw-r--r-- | apt-pkg/acquire.cc | 4 | ||||
-rw-r--r-- | configure.in | 2 | ||||
-rwxr-xr-x | debian/rules | 2 | ||||
-rw-r--r-- | methods/http.cc | 41 | ||||
-rw-r--r-- | methods/http.h | 5 | ||||
-rw-r--r-- | po/apt-all.pot | 56 |
8 files changed, 422 insertions, 37 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b2f896627..81636902e 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -24,6 +24,8 @@ #include <apt-pkg/strutl.h> #include <apt-pkg/fileutl.h> #include <apt-pkg/md5.h> +#include <apt-pkg/sha1.h> +#include <apt-pkg/tagfile.h> #include <apti18n.h> @@ -31,6 +33,7 @@ #include <unistd.h> #include <errno.h> #include <string> +#include <sstream> #include <stdio.h> /*}}}*/ @@ -131,14 +134,309 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ +// AcqIndexDiffs::AcqIndexDiffs - Constructor +// --------------------------------------------------------------------- +/* The package diff is added to the queue. one object is constructed + * for each diff and the index + */ +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5, vector<DiffInfo> diffs) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), available_patches(diffs) +{ + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + CurrentPackagesFile = _config->FindDir("Dir::State::lists"); + CurrentPackagesFile += URItoFileName(RealURI); + + if(Debug) { + std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " + << CurrentPackagesFile << std::endl; + } + + if(!FileExists(CurrentPackagesFile) || + !_config->FindB("Acquire::Diffs",true)) { + // we don't have a pkg file or we don't want to queue + if(Debug) + std::clog << "No index file or canceld by user" << std::endl; + Failed("", NULL); + return; + } + + if(available_patches.size() == 0) + QueueDiffIndex(URI); + else + QueueNextDiff(); +} + +void pkgAcqIndexDiffs::QueueDiffIndex(string URI) +{ + Desc.URI = URI + ".diff/Index"; + Desc.Description = Description + "IndexDiff"; + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI) + string(".IndexDiff"); + + if(Debug) + std::clog << "QueueDiffIndex: " << Desc.URI << std::endl; + + QueueURI(Desc); +} + +// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqIndexDiffs::Custom600Headers() +{ + if(DestFile.rfind(".IndexDiff") == string::npos) + return string(""); + + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(RealURI) + string(".IndexDiff"); + + if(Debug) + std::clog << "Custom600Header-IMS: " << Final << std::endl; + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true"; + + return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} + +void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "Failed(): " << Desc.URI << std::endl + << "Falling back to big package file" << std::endl; + new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, + ExpectedMD5); + Finish(); +} + + +// helper that cleans the item out of the fetcher queue +void pkgAcqIndexDiffs::Finish(bool allDone) +{ + // we restore the original name, this is required, otherwise + // the file will be cleaned + if(allDone) { + // this is for the "real" finish + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + Complete = true; + Dequeue(); + if(Debug) + std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; + return; + + } + + if(Debug) + std::clog << "Finishing: " << Desc.URI << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + return; +} + + +// this needs to be rewriten to not depend on the external ed +bool pkgAcqIndexDiffs::ApplyDiff(string PatchFile) +{ + char *error; + int res=0; + + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + int Process = ExecFork(); + if (Process == 0) + { + chdir(_config->FindDir("Dir::State::lists").c_str()); + // for some reason "red" fails with the pdiffs from p.d.o/~aba ?!? + string cmd = "(zcat " + PatchFile + "; echo \"wq\" ) | /bin/ed " + FinalFile + " >/dev/null 2>/dev/null"; + if(Debug) + std::clog << "Runing: " << cmd << std::endl; + res = system(cmd.c_str()); + _exit(WEXITSTATUS(res)); + } + if(!ExecWait(Process, error, true)) { + //_error->Error("Patch failed: %s ", error); + return false; + } + + return true; +} + +bool pkgAcqIndexDiffs::QueueNextDiff() +{ + // calc sha1 of the just patched file + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + FileFd fd(FinalFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + string local_sha1 = string(SHA1.Result()); + + // see if we have a patch for it, the patch list must be ordered + for(vector<DiffInfo>::iterator I=available_patches.begin(); + I != available_patches.end(); I++) { + // if the patch does not fit, it's not interessting + if((*I).sha1 != local_sha1) + available_patches.erase(I); + } + + // error checking and falling back if no patch was found + if(available_patches.size() == 0) { + Failed("", NULL); + return false; + } + + // queue the right diff + Desc.URI = string(RealURI) + string(".diff/") + available_patches[0].file + string(".gz"); + Desc.Description = available_patches[0].file + string(".pdiff"); + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI + string(".diff/") + available_patches[0].file); + + if(Debug) + std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; + + QueueURI(Desc); + + return true; +} + +bool pkgAcqIndexDiffs::ParseIndexDiff(string IndexDiffFile) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile + << std::endl; + + FileFd Fd(IndexDiffFile,FileFd::ReadOnly); + pkgTagFile TF(&Fd); + if (_error->PendingError() == true) + return false; + pkgTagSection Tags; + if(TF.Step(Tags) == true) + { + string local_sha1; + string tmp = Tags.FindS("SHA1-Current"); + std::stringstream ss(tmp); + ss >> ServerSha1; + + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + local_sha1 = string(SHA1.Result()); + + if(local_sha1 == ServerSha1) { + if(Debug) + std::clog << "Package file is up-to-date" << std::endl; + Finish(true); + return true; + } + if(Debug) + std::clog << "SHA1-Current: " << ServerSha1 << std::endl; + + // check the historie and see what patches we need + string history = Tags.FindS("SHA1-History"); + std::stringstream hist(history); + DiffInfo d; + string size; + bool found = false; + while(hist >> d.sha1 >> size >> d.file) { + d.size = atoi(size.c_str()); + if(d.sha1 == local_sha1) + found=true; + if(found) { + if(Debug) + std::clog << "Need to get diff: " << d.file << std::endl; + available_patches.push_back(d); + } + } + // no information how to get the patches, bail out + if(!found) { + if(Debug) + std::clog << "Can't find a patch in the index file" << std::endl; + // Failed will queue a big package file + Failed("", NULL); + } else { + // queue the diffs + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + Finish(); + return true; + } + } + + return false; +} + + +void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + int len = Desc.URI.size(); + // sucess in downloading the index + if(Desc.URI.substr(len-strlen("Index"),len-1) == "Index") { + + // rename + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI) + string(".IndexDiff"); + if(Debug) + std::clog << "Renaming: " << DestFile << " -> " << FinalFile + << std::endl; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + + if(!ParseIndexDiff(DestFile)) + return Failed("", NULL); + else + return Finish(); + } + + // sucess in downloading a diff + if(Desc.URI.find(".diff") != string::npos) { + ApplyDiff(DestFile); + available_patches.erase(available_patches.begin()); + + if(available_patches.size() > 0) { + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + } else { + Finish(true); + return; + } + } + + Finish(); +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - string ExpectedMD5, string comprExt) : - Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) + string ExpectedMD5, string comprExt) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) { Decompression = false; Erase = false; @@ -599,8 +897,8 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) } // Queue Packages file - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexMD5); + new pkgAcqIndexDiffs(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexMD5); } } diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index ea71153d5..8d58a39ba 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -82,6 +82,47 @@ class pkgAcquire::Item virtual ~Item(); }; +// item for index diffs +class pkgAcqIndexDiffs : public pkgAcquire::Item +{ + protected: + bool Debug; + pkgAcquire::ItemDesc Desc; + string RealURI; + string ExpectedMD5; + // this is the SHA-1 sum we expect after the patching + string ServerSha1; + string CurrentPackagesFile; + string Description; + struct DiffInfo { + string file; + string sha1; + unsigned long size; + }; + vector<DiffInfo> available_patches; + + + public: + + // Specialized action members + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual string DescURI() {return RealURI + "Index";}; + virtual string Custom600Headers(); + + // various helpers + bool ParseIndexDiff(string IndexDiffFile); + void QueueDiffIndex(string URI); + bool QueueNextDiff(); + bool ApplyDiff(string PatchFile); + void Finish(bool allDone=false); + + pkgAcqIndexDiffs(pkgAcquire *Owner,string URI,string URIDesc, + string ShortDesct, string ExpectedMD5, + vector<DiffInfo> diffs=vector<DiffInfo>()); +}; + // Item class for index files class pkgAcqIndex : public pkgAcquire::Item { @@ -92,7 +133,7 @@ class pkgAcqIndex : public pkgAcquire::Item pkgAcquire::ItemDesc Desc; string RealURI; string ExpectedMD5; - + public: // Specialized action members diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 70dce4f54..212c8d52b 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -266,6 +266,10 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access) if (Work.Start() == false) return 0; + /* if a method uses DownloadLimit, we switch to SingleInstance mode */ + if(_config->FindI("Acquire::"+Access+"::DlLimit",0) > 0) + Conf->SingleInstance = true; + return Conf; } /*}}}*/ diff --git a/configure.in b/configure.in index e17a126ba..12d3ecae7 100644 --- a/configure.in +++ b/configure.in @@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib) AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in) dnl -- SET THIS TO THE RELEASE VERSION -- -AC_DEFINE_UNQUOTED(VERSION,"0.6.35ubuntu1") +AC_DEFINE_UNQUOTED(VERSION,"0.6.36") PACKAGE="apt" AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE") AC_SUBST(PACKAGE) diff --git a/debian/rules b/debian/rules index e3254c2d2..fdae46352 100755 --- a/debian/rules +++ b/debian/rules @@ -36,7 +36,7 @@ endif # Default rule build: -DEB_BUILD_PROG:=debuild --preserve-envvar PATH --preserve-envvar CCACHE_DIR -us -uc $(DEB_BUILD_PROG_OPTS) +DEB_BUILD_PROG:=debuild --preserve-envvar PATH --preserve-envvar CCACHE_DIR -us -uc $(DEBUILD_DPKG_BUILDPACKAGE_OPTS) APT_DEBVER=$(shell dpkg-parsechangelog |sed -n -e '/^Version:/s/^Version: //p') APT_CONFVER=$(shell sed -n -e 's/^AC_DEFINE_UNQUOTED(VERSION,"\(.*\)")/\1/p' configure.in) APT_CVSTAG=$(shell echo "$(APT_DEBVER)" | sed -e 's/^/v/' -e 's/\./_/g') diff --git a/methods/http.cc b/methods/http.cc index 81a64d7a3..27f0dafef 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -58,6 +58,12 @@ unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; bool Debug = false; + +unsigned long CircleBuf::BwReadLimit=0; +unsigned long CircleBuf::BwTickReadData=0; +struct timeval CircleBuf::BwReadTick={0,0}; +const unsigned int CircleBuf::BW_HZ=10; + // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -65,6 +71,8 @@ CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0) { Buf = new unsigned char[Size]; Reset(); + + CircleBuf::BwReadLimit = _config->FindI("Acquire::http::DlLimit",0)*1024; } /*}}}*/ // CircleBuf::Reset - Reset to the default state /*{{{*/ @@ -90,16 +98,45 @@ void CircleBuf::Reset() is non-blocking.. */ bool CircleBuf::Read(int Fd) { + unsigned long BwReadMax; + while (1) { // Woops, buffer is full if (InP - OutP == Size) return true; - + + // what's left to read in this tick + BwReadMax = CircleBuf::BwReadLimit/BW_HZ; + + if(CircleBuf::BwReadLimit) { + struct timeval now; + gettimeofday(&now,0); + + unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 + + now.tv_usec-CircleBuf::BwReadTick.tv_usec; + if(d > 1000000/BW_HZ) { + CircleBuf::BwReadTick = now; + CircleBuf::BwTickReadData = 0; + } + + if(CircleBuf::BwTickReadData >= BwReadMax) { + usleep(1000000/BW_HZ); + return true; + } + } + // Write the buffer segment int Res; - Res = read(Fd,Buf + (InP%Size),LeftRead()); + if(CircleBuf::BwReadLimit) { + Res = read(Fd,Buf + (InP%Size), + BwReadMax > LeftRead() ? LeftRead() : BwReadMax); + } else + Res = read(Fd,Buf + (InP%Size),LeftRead()); + if(Res > 0 && BwReadLimit > 0) + CircleBuf::BwTickReadData += Res; + if (Res == 0) return false; if (Res < 0) diff --git a/methods/http.h b/methods/http.h index c5a4d0e86..541e2952c 100644 --- a/methods/http.h +++ b/methods/http.h @@ -31,6 +31,11 @@ class CircleBuf unsigned long MaxGet; struct timeval Start; + static unsigned long BwReadLimit; + static unsigned long BwTickReadData; + static struct timeval BwReadTick; + static const unsigned int BW_HZ; + unsigned long LeftRead() { unsigned long Sz = Size - (InP - OutP); diff --git a/po/apt-all.pot b/po/apt-all.pot index b9232cac4..7c3774ed6 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2005-04-06 13:03+0200\n" +"POT-Creation-Date: 2005-05-07 16:22+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Language-Team: LANGUAGE <LL@li.org>\n" @@ -1271,7 +1271,7 @@ msgid "File %s/%s overwrites the one in the package %s" msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:709 -#: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/acquire.cc:416 apt-pkg/clean.cc:38 +#: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/acquire.cc:420 apt-pkg/clean.cc:38 #, c-format msgid "Unable to read %s" msgstr "" @@ -1561,7 +1561,7 @@ msgstr "" msgid "Unable to accept connection" msgstr "" -#: methods/ftp.cc:864 methods/http.cc:916 methods/rsh.cc:303 +#: methods/ftp.cc:864 methods/http.cc:953 methods/rsh.cc:303 msgid "Problem hashing file" msgstr "" @@ -1655,76 +1655,76 @@ msgstr "" msgid "Read error from %s process" msgstr "" -#: methods/http.cc:344 +#: methods/http.cc:381 msgid "Waiting for headers" msgstr "" -#: methods/http.cc:490 +#: methods/http.cc:527 #, c-format msgid "Got a single header line over %u chars" msgstr "" -#: methods/http.cc:498 +#: methods/http.cc:535 msgid "Bad header line" msgstr "" -#: methods/http.cc:517 methods/http.cc:524 +#: methods/http.cc:554 methods/http.cc:561 msgid "The HTTP server sent an invalid reply header" msgstr "" -#: methods/http.cc:553 +#: methods/http.cc:590 msgid "The HTTP server sent an invalid Content-Length header" msgstr "" -#: methods/http.cc:568 +#: methods/http.cc:605 msgid "The HTTP server sent an invalid Content-Range header" msgstr "" -#: methods/http.cc:570 +#: methods/http.cc:607 msgid "This HTTP server has broken range support" msgstr "" -#: methods/http.cc:594 +#: methods/http.cc:631 msgid "Unknown date format" msgstr "" -#: methods/http.cc:737 +#: methods/http.cc:774 msgid "Select failed" msgstr "" -#: methods/http.cc:742 +#: methods/http.cc:779 msgid "Connection timed out" msgstr "" -#: methods/http.cc:765 +#: methods/http.cc:802 msgid "Error writing to output file" msgstr "" -#: methods/http.cc:793 +#: methods/http.cc:830 msgid "Error writing to file" msgstr "" -#: methods/http.cc:818 +#: methods/http.cc:855 msgid "Error writing to the file" msgstr "" -#: methods/http.cc:832 +#: methods/http.cc:869 msgid "Error reading from server. Remote end closed connection" msgstr "" -#: methods/http.cc:834 +#: methods/http.cc:871 msgid "Error reading from server" msgstr "" -#: methods/http.cc:1065 +#: methods/http.cc:1102 msgid "Bad header data" msgstr "" -#: methods/http.cc:1082 +#: methods/http.cc:1119 msgid "Connection failed" msgstr "" -#: methods/http.cc:1173 +#: methods/http.cc:1210 msgid "Internal error" msgstr "" @@ -1862,7 +1862,7 @@ msgstr "" msgid "Unable to stat the mount point %s" msgstr "" -#: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:422 apt-pkg/clean.cc:44 +#: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:426 apt-pkg/clean.cc:44 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2246,36 +2246,36 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:126 +#: apt-pkg/acquire-item.cc:129 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:235 apt-pkg/acquire-item.cc:900 +#: apt-pkg/acquire-item.cc:502 apt-pkg/acquire-item.cc:1161 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:714 +#: apt-pkg/acquire-item.cc:975 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:767 +#: apt-pkg/acquire-item.cc:1028 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:803 +#: apt-pkg/acquire-item.cc:1064 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:890 +#: apt-pkg/acquire-item.cc:1151 msgid "Size mismatch" msgstr "" |