From ac5b205a831168ee76e8760e19eb7d43aa361851 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 15 Apr 2005 11:27:58 +0000 Subject: * first rewrite of the apt pdiff support patch finished --- apt-pkg/acquire-item.cc | 248 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 244 insertions(+), 4 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 714edd8d8..8c519e3f7 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include @@ -31,6 +33,7 @@ #include #include #include +#include #include /*}}}*/ @@ -131,14 +134,251 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ +// AcqIndexDiffs::AcqIndexDiffs - Constructor +// --------------------------------------------------------------------- +/* The package diff is added to the queue. one object is constructed + * for each diff and the index + */ +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5, vector diffs) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), needed_files(diffs) +{ + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + CurrentPackagesFile = _config->FindDir("Dir::State::lists"); + CurrentPackagesFile += URItoFileName(RealURI); + + if(Debug) { + std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " + << CurrentPackagesFile << std::endl; + } + + if(!FileExists(CurrentPackagesFile) || + !_config->FindB("Acquire::Diffs",true)) { + // we don't have a pkg file or we don't want to queue + if(Debug) + std::clog << "No index file or canceld by user" << std::endl; + Failed("", NULL); + return; + } + + if(needed_files.size() == 0) + QueueDiffIndex(URI); + else + QueueNextDiff(); +} + +void pkgAcqIndexDiffs::QueueDiffIndex(string URI) +{ + Desc.URI = URI + ".diff/Index"; + Desc.Description = Description + "IndexDiff"; + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI) + string(".IndexDiff"); + + if(Debug) + std::clog << "QueueDiffIndex: " << Desc.URI << std::endl; + + QueueURI(Desc); +} + +void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "Failed(): " << Desc.URI << std::endl + << "Falling back to big package file" << std::endl; + new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, + ExpectedMD5); + Finish(); +} + + +// helper that cleans the item out of the fetcher queue +void pkgAcqIndexDiffs::Finish(bool allDone) +{ + // we restore the original name, this is required, otherwise + // the file will be cleaned + if(allDone) { + // this is for the "real" finish + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + Complete = true; + Dequeue(); + if(Debug) + std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; + return; + + } + + if(Debug) + std::clog << "Finishing: " << Desc.URI << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + return; +} + + +// this needs to be rewriten to not depend on the external ed +bool pkgAcqIndexDiffs::ApplyDiff(string PatchFile) +{ + char *error; + int res=0; + + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + int Process = ExecFork(); + if (Process == 0) + { + chdir(_config->FindDir("Dir::State::lists").c_str()); + string cmd = "(zcat " + PatchFile + "; echo \"wq\" ) | ed " + FinalFile + " >/dev/null 2>/dev/null"; + if(Debug) + std::clog << "Runing: " << cmd << std::endl; + res = system(cmd.c_str()); + _exit(WEXITSTATUS(res)); + } + if(!ExecWait(Process, error, true)) { + //_error->Error("Patch failed: %s ", error); + return false; + } + + return true; +} + +bool pkgAcqIndexDiffs::QueueNextDiff() +{ + // queue diff + Desc.URI = string(RealURI) + string(".diff/") + needed_files[0] + string(".gz"); + Desc.Description = Description + string("-diff"); + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI + string(".diff/") + needed_files[0]); + + if(Debug) + std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; + + QueueURI(Desc); + + return true; +} + +bool pkgAcqIndexDiffs::ParseIndexDiff(string IndexDiffFile) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile + << std::endl; + + FileFd Fd(IndexDiffFile,FileFd::ReadOnly); + pkgTagFile TF(&Fd); + if (_error->PendingError() == true) + return false; + pkgTagSection Tags; + if(TF.Step(Tags) == true) + { + string local_sha1; + string tmp = Tags.FindS("SHA1-Current"); + std::stringstream ss(tmp); + ss >> ServerSha1; + + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + local_sha1 = string(SHA1.Result()); + + if(local_sha1 == ServerSha1) { + if(Debug) + std::clog << "Package file is up-to-date" << std::endl; + Finish(true); + return true; + } + if(Debug) + std::clog << "SHA1-Current: " << ServerSha1 << std::endl; + + // check the historie and see what patches we need + string history = Tags.FindS("SHA1-History"); + std::stringstream hist(history); + string sha1, size, file; + bool found = false; + while(hist >> sha1 >> size >> file) { + if(sha1 == local_sha1) + found=true; + if(found) { + if(Debug) + std::clog << "Need to get diff: " << file << std::endl; + needed_files.push_back(file); + } + } + // no information how to get the patches, bail out + if(!found) { + if(Debug) + std::clog << "Can't find a patch in the index file" << std::endl; + // Failed will queue a big package file + Failed("", NULL); + } else { + // queue the diffs + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, needed_files); + Finish(); + return true; + } + } + + return false; +} + + +void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + int len = Desc.URI.size(); + // sucess in downloading the index + if(Desc.URI.substr(len-strlen("Index"),len-1) == "Index") { + if(!ParseIndexDiff(DestFile)) + return Failed("", NULL); + else + return Finish(); + } + + // sucess in downloading a diff + if(Desc.URI.find(".diff") != string::npos) { + ApplyDiff(DestFile); + needed_files.erase(needed_files.begin()); + + if(needed_files.size() > 0) { + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, needed_files); + } else { + Finish(true); + return; + } + } + + Finish(); +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - string ExpectedMD5, string comprExt) : - Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) + string ExpectedMD5, string comprExt) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) { Decompression = false; Erase = false; @@ -593,8 +833,8 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) } // Queue Packages file - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexMD5); + new pkgAcqIndexDiffs(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexMD5); } } -- cgit v1.2.3 From 6cb30d01f8e247e85966ba8ad657453d73598527 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sat, 7 May 2005 15:39:12 +0000 Subject: * use "red" now, print meaningfull pdiff names, use IMS for the IndexDiff --- apt-pkg/acquire-item.cc | 60 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 11 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 8c519e3f7..5d741af6f 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -141,7 +141,7 @@ void pkgAcquire::Item::Rename(string From,string To) */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - string ExpectedMD5, vector diffs) + string ExpectedMD5, vector diffs) : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), needed_files(diffs) { @@ -190,6 +190,27 @@ void pkgAcqIndexDiffs::QueueDiffIndex(string URI) QueueURI(Desc); } +// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqIndexDiffs::Custom600Headers() +{ + if(DestFile.rfind(".IndexDiff") == string::npos) + return string(""); + + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(RealURI) + string(".IndexDiff"); + + if(Debug) + std::clog << "Custom600Header-IMS: " << Final << std::endl; + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true"; + + return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} + void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { if(Debug) @@ -240,7 +261,7 @@ bool pkgAcqIndexDiffs::ApplyDiff(string PatchFile) if (Process == 0) { chdir(_config->FindDir("Dir::State::lists").c_str()); - string cmd = "(zcat " + PatchFile + "; echo \"wq\" ) | ed " + FinalFile + " >/dev/null 2>/dev/null"; + string cmd = "(zcat " + PatchFile + "; echo \"wq\" ) | red " + FinalFile + " >/dev/null 2>/dev/null"; if(Debug) std::clog << "Runing: " << cmd << std::endl; res = system(cmd.c_str()); @@ -256,12 +277,16 @@ bool pkgAcqIndexDiffs::ApplyDiff(string PatchFile) bool pkgAcqIndexDiffs::QueueNextDiff() { + // FIXME: don't use the needed_files[0] but check sha1 and search + // for the right patch in needed_files + // -> and rename needed_files to "available_patches" + // queue diff - Desc.URI = string(RealURI) + string(".diff/") + needed_files[0] + string(".gz"); - Desc.Description = Description + string("-diff"); + Desc.URI = string(RealURI) + string(".diff/") + needed_files[0].file + string(".gz"); + Desc.Description = needed_files[0].file + string(".pdiff"); DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI + string(".diff/") + needed_files[0]); + DestFile += URItoFileName(RealURI + string(".diff/") + needed_files[0].file); if(Debug) std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; @@ -306,15 +331,17 @@ bool pkgAcqIndexDiffs::ParseIndexDiff(string IndexDiffFile) // check the historie and see what patches we need string history = Tags.FindS("SHA1-History"); std::stringstream hist(history); - string sha1, size, file; + DiffInfo d; + string size; bool found = false; - while(hist >> sha1 >> size >> file) { - if(sha1 == local_sha1) + while(hist >> d.sha1 >> size >> d.file) { + d.size = atoi(size.c_str()); + if(d.sha1 == local_sha1) found=true; if(found) { if(Debug) - std::clog << "Need to get diff: " << file << std::endl; - needed_files.push_back(file); + std::clog << "Need to get diff: " << d.file << std::endl; + needed_files.push_back(d); } } // no information how to get the patches, bail out @@ -347,9 +374,20 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, int len = Desc.URI.size(); // sucess in downloading the index if(Desc.URI.substr(len-strlen("Index"),len-1) == "Index") { + + // rename + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI) + string(".IndexDiff"); + if(Debug) + std::clog << "Renaming: " << DestFile << " -> " << FinalFile + << std::endl; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + if(!ParseIndexDiff(DestFile)) return Failed("", NULL); - else + else return Finish(); } -- cgit v1.2.3 From 26d276459901fea7209203ec84403bb7934fb869 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 May 2005 07:59:17 +0000 Subject: * fix a bug QueueNextDiff --- apt-pkg/acquire-item.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 81636902e..fff55f7b5 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -287,13 +287,13 @@ bool pkgAcqIndexDiffs::QueueNextDiff() SHA1.AddFD(fd.Fd(), fd.Size()); string local_sha1 = string(SHA1.Result()); - // see if we have a patch for it, the patch list must be ordered + // remove all patches until the next matching patch is found + // this requires the Index file to be ordered for(vector::iterator I=available_patches.begin(); - I != available_patches.end(); I++) { - // if the patch does not fit, it's not interessting - if((*I).sha1 != local_sha1) - available_patches.erase(I); - } + (*I).sha1 == local_sha1 || I != available_patches.end(); + I++) + available_patches.erase(I); + // error checking and falling back if no patch was found if(available_patches.size() == 0) { -- cgit v1.2.3 From 77a7df0e00b10252abb2fe52a083d53d5878ab11 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 May 2005 09:09:12 +0000 Subject: * another bug in QueueNextDiff fixed --- apt-pkg/acquire-item.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index fff55f7b5..79e0b1898 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -290,10 +290,9 @@ bool pkgAcqIndexDiffs::QueueNextDiff() // remove all patches until the next matching patch is found // this requires the Index file to be ordered for(vector::iterator I=available_patches.begin(); - (*I).sha1 == local_sha1 || I != available_patches.end(); + I != available_patches.end() && (*I).sha1 != local_sha1; I++) available_patches.erase(I); - // error checking and falling back if no patch was found if(available_patches.size() == 0) { @@ -356,14 +355,17 @@ bool pkgAcqIndexDiffs::ParseIndexDiff(string IndexDiffFile) bool found = false; while(hist >> d.sha1 >> size >> d.file) { d.size = atoi(size.c_str()); + // read until the first match is found if(d.sha1 == local_sha1) found=true; + // from that point on, we probably need all diffs if(found) { if(Debug) std::clog << "Need to get diff: " << d.file << std::endl; available_patches.push_back(d); } } + // no information how to get the patches, bail out if(!found) { if(Debug) -- cgit v1.2.3 From 4a0a786f45a78eb8631c0e2d39d804ab9fdea214 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 19 Aug 2005 15:07:53 +0000 Subject: * use the new cool rred method for the patchting --- apt-pkg/acquire-item.cc | 112 ++++++++++++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 47 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 79e0b1898..166845868 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -141,8 +141,9 @@ void pkgAcquire::Item::Rename(string From,string To) */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - string ExpectedMD5, vector diffs) - : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), available_patches(diffs) + string ExpectedMD5, vector diffs) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), + available_patches(diffs) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -171,10 +172,13 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, return; } - if(available_patches.size() == 0) + if(available_patches.size() == 0) { + State = StateFetchIndex; QueueDiffIndex(URI); - else + } else { + State = StateFetchDiff; QueueNextDiff(); + } } void pkgAcqIndexDiffs::QueueDiffIndex(string URI) @@ -195,7 +199,8 @@ void pkgAcqIndexDiffs::QueueDiffIndex(string URI) /* The only header we use is the last-modified header. */ string pkgAcqIndexDiffs::Custom600Headers() { - if(DestFile.rfind(".IndexDiff") == string::npos) + // we only care for the IndexDiff file + if(State != StateFetchIndex) return string(""); string Final = _config->FindDir("Dir::State::lists"); @@ -248,33 +253,6 @@ void pkgAcqIndexDiffs::Finish(bool allDone) } -// this needs to be rewriten to not depend on the external ed -bool pkgAcqIndexDiffs::ApplyDiff(string PatchFile) -{ - char *error; - int res=0; - - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - - int Process = ExecFork(); - if (Process == 0) - { - chdir(_config->FindDir("Dir::State::lists").c_str()); - // for some reason "red" fails with the pdiffs from p.d.o/~aba ?!? - string cmd = "(zcat " + PatchFile + "; echo \"wq\" ) | /bin/ed " + FinalFile + " >/dev/null 2>/dev/null"; - if(Debug) - std::clog << "Runing: " << cmd << std::endl; - res = system(cmd.c_str()); - _exit(WEXITSTATUS(res)); - } - if(!ExecWait(Process, error, true)) { - //_error->Error("Patch failed: %s ", error); - return false; - } - - return true; -} bool pkgAcqIndexDiffs::QueueNextDiff() { @@ -393,13 +371,14 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, Item::Done(Message,Size,Md5Hash,Cnf); - int len = Desc.URI.size(); - // sucess in downloading the index - if(Desc.URI.substr(len-strlen("Index"),len-1) == "Index") { + string FinalFile; + FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); - // rename - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI) + string(".IndexDiff"); + // sucess in downloading the index + if(State == StateFetchIndex) + { + // rename the index + FinalFile += string(".IndexDiff"); if(Debug) std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl; @@ -413,21 +392,60 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, return Finish(); } - // sucess in downloading a diff - if(Desc.URI.find(".diff") != string::npos) { - ApplyDiff(DestFile); + // sucess in downloading a diff, enter ApplyDiff state + if(State == StateFetchDiff) + { + + if(Debug) + std::clog << "Sending to gzip method: " << FinalFile << std::endl; + + string FileName = LookupTag(Message,"Filename"); + State = StateUnzipDiff; + Desc.URI = "gzip:" + FileName; + DestFile += ".decomp"; + QueueURI(Desc); + Mode = "gzip"; + return; + } + + // sucess in downloading a diff, enter ApplyDiff state + if(State == StateUnzipDiff) + { + + // rred excepts the patch as $FinalFile.ed + Rename(DestFile,FinalFile+".ed"); + + if(Debug) + std::clog << "Sending to rred method: " << FinalFile << std::endl; + + State = StateApplyDiff; + Desc.URI = "rred:" + FinalFile; + QueueURI(Desc); + Mode = "rred"; + return; + } + + + // success in download/apply a diff, queue next (if needed) + if(State == StateApplyDiff) + { + // remove the just applied patch available_patches.erase(available_patches.begin()); + // move into place + if(Debug) + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << FinalFile << std::endl; + Rename(DestFile,FinalFile); + + // see if there is more to download if(available_patches.size() > 0) { new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedMD5, available_patches); - } else { - Finish(true); - return; - } + return Finish(); + } else + return Finish(true); } - - Finish(); } -- cgit v1.2.3 From 59a704f0efe9cf243f1b27ac833558b89728aa9c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 22 Aug 2005 08:48:50 +0000 Subject: * rred.cc: use the d'tor of FileFd to close the file-fd; acquuire-item.cc: make the QueueNextDiff() code more robust --- apt-pkg/acquire-item.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 166845868..43d29c737 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -218,9 +218,8 @@ string pkgAcqIndexDiffs::Custom600Headers() void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - if(Debug) - std::clog << "Failed(): " << Desc.URI << std::endl - << "Falling back to big package file" << std::endl; + std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl + << "Falling back to normal index file aquire" << std::endl; new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, ExpectedMD5); Finish(); @@ -268,9 +267,11 @@ bool pkgAcqIndexDiffs::QueueNextDiff() // remove all patches until the next matching patch is found // this requires the Index file to be ordered for(vector::iterator I=available_patches.begin(); - I != available_patches.end() && (*I).sha1 != local_sha1; - I++) + available_patches.size() > 0 && I != available_patches.end() + && (*I).sha1 != local_sha1; + I++) { available_patches.erase(I); + } // error checking and falling back if no patch was found if(available_patches.size() == 0) { @@ -433,9 +434,11 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, available_patches.erase(available_patches.begin()); // move into place - if(Debug) + if(Debug) + { std::clog << "Moving patched file in place: " << std::endl << DestFile << " -> " << FinalFile << std::endl; + } Rename(DestFile,FinalFile); // see if there is more to download -- cgit v1.2.3 From 3de9ff77a811d67632c9948a953e628ed6a2b78a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 22 Aug 2005 10:19:43 +0000 Subject: * bugfix/updates in the rred.cc code (big thanks for helping Robert!) --- apt-pkg/acquire-item.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 43d29c737..b8886a750 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -255,6 +255,7 @@ void pkgAcqIndexDiffs::Finish(bool allDone) bool pkgAcqIndexDiffs::QueueNextDiff() { + // calc sha1 of the just patched file string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); @@ -263,6 +264,9 @@ bool pkgAcqIndexDiffs::QueueNextDiff() SHA1Summation SHA1; SHA1.AddFD(fd.Fd(), fd.Size()); string local_sha1 = string(SHA1.Result()); + if(Debug) + std::clog << "QueueNextDiff: " + << FinalFile << " (" << local_sha1 << ")"< Date: Mon, 29 Aug 2005 23:23:31 +0000 Subject: * split the diff acquire code in a bit that gets the index file and one that gets the patches. that make the code cleaner and the list-cleaner happy --- apt-pkg/acquire-item.cc | 323 ++++++++++++++++++++++++++++-------------------- 1 file changed, 190 insertions(+), 133 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index b8886a750..aab8b7ed1 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -134,35 +134,37 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ -// AcqIndexDiffs::AcqIndexDiffs - Constructor + +// AcqDiffIndex::AcqDiffIndex - Constructor // --------------------------------------------------------------------- -/* The package diff is added to the queue. one object is constructed - * for each diff and the index +/* Get the DiffIndex file first and see if there are patches availabe + * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the + * patches. If anything goes wrong in that process, it will fall back to + * the original packages file */ -pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, - string URI,string URIDesc,string ShortDesc, - string ExpectedMD5, vector diffs) - : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), - available_patches(diffs) +pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), Description(URIDesc) { - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); - Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Desc.Description = URIDesc; + Desc.Description = URIDesc + "/DiffIndex"; Desc.Owner = this; Desc.ShortDesc = ShortDesc; + Desc.URI = URI + ".diff/Index"; + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI) + string(".DiffIndex"); + + if(Debug) + std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; + // look for the current package file CurrentPackagesFile = _config->FindDir("Dir::State::lists"); CurrentPackagesFile += URItoFileName(RealURI); - if(Debug) { - std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " - << CurrentPackagesFile << std::endl; - } - if(!FileExists(CurrentPackagesFile) || !_config->FindB("Acquire::Diffs",true)) { // we don't have a pkg file or we don't want to queue @@ -172,37 +174,20 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, return; } - if(available_patches.size() == 0) { - State = StateFetchIndex; - QueueDiffIndex(URI); - } else { - State = StateFetchDiff; - QueueNextDiff(); + if(Debug) { + std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " + << CurrentPackagesFile << std::endl; } -} - -void pkgAcqIndexDiffs::QueueDiffIndex(string URI) -{ - Desc.URI = URI + ".diff/Index"; - Desc.Description = Description + "IndexDiff"; - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI) + string(".IndexDiff"); - - if(Debug) - std::clog << "QueueDiffIndex: " << Desc.URI << std::endl; QueueURI(Desc); + } // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqIndexDiffs::Custom600Headers() +string pkgAcqDiffIndex::Custom600Headers() { - // we only care for the IndexDiff file - if(State != StateFetchIndex) - return string(""); - string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI) + string(".IndexDiff"); @@ -216,11 +201,170 @@ string pkgAcqIndexDiffs::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } + +bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile + << std::endl; + + pkgTagSection Tags; + string ServerSha1; + vector available_patches; + + FileFd Fd(IndexDiffFile,FileFd::ReadOnly); + pkgTagFile TF(&Fd); + if (_error->PendingError() == true) + return false; + + if(TF.Step(Tags) == true) + { + string local_sha1; + string tmp = Tags.FindS("SHA1-Current"); + std::stringstream ss(tmp); + ss >> ServerSha1; + + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + local_sha1 = string(SHA1.Result()); + + if(local_sha1 == ServerSha1) { + if(Debug) + std::clog << "Package file is up-to-date" << std::endl; + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + Complete = false; + Status = StatDone; + Dequeue(); + return true; + } + if(Debug) + std::clog << "SHA1-Current: " << ServerSha1 << std::endl; + + // check the historie and see what patches we need + string history = Tags.FindS("SHA1-History"); + std::stringstream hist(history); + DiffInfo d; + string size; + bool found = false; + while(hist >> d.sha1 >> size >> d.file) { + d.size = atoi(size.c_str()); + // read until the first match is found + if(d.sha1 == local_sha1) + found=true; + // from that point on, we probably need all diffs + if(found) { + if(Debug) + std::clog << "Need to get diff: " << d.file << std::endl; + available_patches.push_back(d); + } + } + + // no information how to get the patches, bail out + if(!found) { + if(Debug) + std::clog << "Can't find a patch in the index file" << std::endl; + // Failed will queue a big package file + Failed("", NULL); + } else { + // queue the diffs + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + Complete = false; + Status = StatDone; + Dequeue(); + return true; + } + } + + return false; +} + +void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl + << "Falling back to normal index file aquire" << std::endl; + + new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, + ExpectedMD5); + + Complete = false; + Status = StatDone; + Dequeue(); +} + +void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + string FinalFile; + FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); + + // sucess in downloading the index + // rename the index + FinalFile += string(".IndexDiff"); + if(Debug) + std::clog << "Renaming: " << DestFile << " -> " << FinalFile + << std::endl; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + + if(!ParseDiffIndex(DestFile)) + return Failed("", NULL); + + Complete = true; + Status = StatDone; + Dequeue(); + return; +} + + + +// AcqIndexDiffs::AcqIndexDiffs - Constructor +// --------------------------------------------------------------------- +/* The package diff is added to the queue. one object is constructed + * for each diff and the index + */ +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5, vector diffs) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), + available_patches(diffs) +{ + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + if(available_patches.size() == 0) { + // we are done (yeah!) + Finish(true); + } else { + // get the next diff + State = StateFetchDiff; + QueueNextDiff(); + } +} + + void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl - << "Falling back to normal index file aquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, + if(Debug) + std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl + << "Falling back to normal index file aquire" << std::endl; + new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, ExpectedMD5); Finish(); } @@ -240,7 +384,6 @@ void pkgAcqIndexDiffs::Finish(bool allDone) if(Debug) std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; return; - } if(Debug) @@ -284,11 +427,11 @@ bool pkgAcqIndexDiffs::QueueNextDiff() } // queue the right diff - Desc.URI = string(RealURI) + string(".diff/") + available_patches[0].file + string(".gz"); + Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz"; Desc.Description = available_patches[0].file + string(".pdiff"); DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI + string(".diff/") + available_patches[0].file); + DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file); if(Debug) std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; @@ -298,74 +441,6 @@ bool pkgAcqIndexDiffs::QueueNextDiff() return true; } -bool pkgAcqIndexDiffs::ParseIndexDiff(string IndexDiffFile) -{ - if(Debug) - std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile - << std::endl; - - FileFd Fd(IndexDiffFile,FileFd::ReadOnly); - pkgTagFile TF(&Fd); - if (_error->PendingError() == true) - return false; - pkgTagSection Tags; - if(TF.Step(Tags) == true) - { - string local_sha1; - string tmp = Tags.FindS("SHA1-Current"); - std::stringstream ss(tmp); - ss >> ServerSha1; - - FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); - SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); - local_sha1 = string(SHA1.Result()); - - if(local_sha1 == ServerSha1) { - if(Debug) - std::clog << "Package file is up-to-date" << std::endl; - Finish(true); - return true; - } - if(Debug) - std::clog << "SHA1-Current: " << ServerSha1 << std::endl; - - // check the historie and see what patches we need - string history = Tags.FindS("SHA1-History"); - std::stringstream hist(history); - DiffInfo d; - string size; - bool found = false; - while(hist >> d.sha1 >> size >> d.file) { - d.size = atoi(size.c_str()); - // read until the first match is found - if(d.sha1 == local_sha1) - found=true; - // from that point on, we probably need all diffs - if(found) { - if(Debug) - std::clog << "Need to get diff: " << d.file << std::endl; - available_patches.push_back(d); - } - } - - // no information how to get the patches, bail out - if(!found) { - if(Debug) - std::clog << "Can't find a patch in the index file" << std::endl; - // Failed will queue a big package file - Failed("", NULL); - } else { - // queue the diffs - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedMD5, available_patches); - Finish(); - return true; - } - } - - return false; -} void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, @@ -379,24 +454,6 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); - // sucess in downloading the index - if(State == StateFetchIndex) - { - // rename the index - FinalFile += string(".IndexDiff"); - if(Debug) - std::clog << "Renaming: " << DestFile << " -> " << FinalFile - << std::endl; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - DestFile = FinalFile; - - if(!ParseIndexDiff(DestFile)) - return Failed("", NULL); - else - return Finish(); - } - // sucess in downloading a diff, enter ApplyDiff state if(State == StateFetchDiff) { @@ -924,8 +981,8 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) } // Queue Packages file - new pkgAcqIndexDiffs(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexMD5); + new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexMD5); } } -- cgit v1.2.3 From 002d99439f800af38ee08fb89ef9651db727f36b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Aug 2005 23:48:15 +0000 Subject: * fixed apt-pkg/acquire-item.h (looks like the last commit didn't include it?) --- apt-pkg/acquire-item.cc | 53 ++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index aab8b7ed1..a1bd2336f 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -220,6 +220,10 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) if(TF.Step(Tags) == true) { string local_sha1; + bool found = false; + DiffInfo d; + string size; + string tmp = Tags.FindS("SHA1-Current"); std::stringstream ss(tmp); ss >> ServerSha1; @@ -232,32 +236,27 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) if(local_sha1 == ServerSha1) { if(Debug) std::clog << "Package file is up-to-date" << std::endl; - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedMD5, available_patches); - Complete = false; - Status = StatDone; - Dequeue(); - return true; - } - if(Debug) - std::clog << "SHA1-Current: " << ServerSha1 << std::endl; - - // check the historie and see what patches we need - string history = Tags.FindS("SHA1-History"); - std::stringstream hist(history); - DiffInfo d; - string size; - bool found = false; - while(hist >> d.sha1 >> size >> d.file) { - d.size = atoi(size.c_str()); - // read until the first match is found - if(d.sha1 == local_sha1) - found=true; - // from that point on, we probably need all diffs - if(found) { - if(Debug) - std::clog << "Need to get diff: " << d.file << std::endl; - available_patches.push_back(d); + // set found to true, this will queue a pkgAcqIndexDiffs with + // a empty availabe_patches + found = true; + } else { + if(Debug) + std::clog << "SHA1-Current: " << ServerSha1 << std::endl; + + // check the historie and see what patches we need + string history = Tags.FindS("SHA1-History"); + std::stringstream hist(history); + while(hist >> d.sha1 >> size >> d.file) { + d.size = atoi(size.c_str()); + // read until the first match is found + if(d.sha1 == local_sha1) + found=true; + // from that point on, we probably need all diffs + if(found) { + if(Debug) + std::clog << "Need to get diff: " << d.file << std::endl; + available_patches.push_back(d); + } } } @@ -287,7 +286,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl << "Falling back to normal index file aquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, + new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, ExpectedMD5); Complete = false; -- cgit v1.2.3 From cffc2ddd412bb0af8e0fe716dacb2da07fe6ddbc Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 30 Aug 2005 00:23:07 +0000 Subject: * fix a missing StatDone if nothing needs to be fetched --- apt-pkg/acquire-item.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index a1bd2336f..5265ef3a6 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -379,6 +379,7 @@ void pkgAcqIndexDiffs::Finish(bool allDone) DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); Complete = true; + Status = StatDone; Dequeue(); if(Debug) std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; -- cgit v1.2.3 From 2d4722e202421007bc34f5439ab4a7cd0bd1148f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 9 Sep 2005 00:06:10 +0000 Subject: * don't forget the final md5sum checking! --- apt-pkg/acquire-item.cc | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'apt-pkg/acquire-item.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c4c0b73a9..8a9a4c5bb 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -375,9 +375,26 @@ void pkgAcqIndexDiffs::Finish(bool allDone) // we restore the original name, this is required, otherwise // the file will be cleaned if(allDone) { - // this is for the "real" finish DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); + + // do the final md5sum checking + MD5Summation sum; + FileFd Fd(DestFile, FileFd::ReadOnly); + sum.AddFD(Fd.Fd(), Fd.Size()); + Fd.Close(); + string MD5 = (string)sum.Result(); + + if (!ExpectedMD5.empty() && MD5 != ExpectedMD5) + { + Status = StatAuthError; + ErrorText = _("MD5Sum mismatch"); + Rename(DestFile,DestFile + ".FAILED"); + Dequeue(); + return; + } + + // this is for the "real" finish Complete = true; Status = StatDone; Dequeue(); -- cgit v1.2.3