* apt-pkg/deb/dpkgpm.cc:

  - apport integration added, this means that a apport
    report is written on dpkg failures
* merged from debian/unstable, remaining changes:
  - maintainer field changed

* merged the debian/experimental changes back
  into the debian/sid branch
* merged from Christian Perrier:
  * mr.po: New Marathi translation  Closes: #416806
  * zh_CN.po: Updated by Eric Pareja  Closes: #416822
  * tl.po: Updated by Eric Pareja   Closes: #416638
  * gl.po: Updated by Jacobo Tarrio
  * da.po: Updated by Claus Hindsgaul
  * fr.po: Remove a non-breakable space for usability
  * ru.po: Updated Russian translation. Closes: #405476
  * *.po: Unfuzzy after upstream typo corrections
* buildlib/archtable:
  - added support for sh3/sh4 (closes: #424870)
  - added support for m32r (closes: #394096)
* buildlib/systemtable:
  - added support for lpia
* configure.in:
  - check systemtable for architecture mapping too
* fix error in AutocleanInterval, closes: #319339
  (thanks to Israel G. Lugo for the patch)
* add "purge" commandline argument, closes: #133421)
  (thanks to Julien Danjou for the patch)
* add "purge" commandline argument, closes: #133421)
  (thanks to Julien Danjou for the patch)
* fix FTBFS with gcc 4.3, closes: #417090
  (thanks to Martin Michlmayr for the patch)
* add --dsc-only option, thanks to K. Richard Pixley
* Removed the more leftover #pragma interface/implementation
  closes: #306937 (thanks to Andreas Henriksson for the patch)

* ABI library name change because its build against
  new glibc
* implement SourceVer() in pkgRecords 
   (thanks to Daniel Burrows for the patch!)
* apt-pkg/algorithm.cc:
  - use clog for all debugging
  - only increase the score of installed applications if they 
    are not obsolete 
  - fix resolver bug on removal triggered by weak-dependencies 
    with or-groups
* methods/http.cc:
  - send apt version in User-Agent
* apt-pkg/deb/debrecords.cc:
  - fix SHA1Hash() return value
* apt-pkg/cdrom.cc:
  - only unmount if APT::CDROM::NoMount is false
* methods/cdrom.cc:  
  - only umount if it was mounted by the method before
  - if decompression of a index fails, delete the index 
  * vi.po: Updated to 515t. Closes: #426976
  * eu.po: Updated to 515t. Closes: #423766
  * pt.po: 515t. Closes: #423111
  * fr.po: Updated by Christian Perrier
  * Update all PO and the POT. Gives 513t2f for formerly
    complete translations
* Package that contains tall the new features
* Removed all #pragma interface/implementation
* Branch that contains tall the new features:
* translated package descriptions
* task install support
* automatic dependency removal (thanks to Daniel Burrows)
* merged support for the new dpkg "Breaks" field 
  (thanks to Ian Jackson)
* handle network failures more gracefully on "update"
* support for unattended-upgrades (via unattended-upgrades
  package)
* added apt-transport-https method
  - ca.po: Updated to 514t
  - be.po: Updated to 514t
  - it.po: Updated to 514t
  - hu.po: Updated to 514t
  - zh_TW.po: Updated to 514t
  - ar.po: Updated to 293t221u.
  - ru.po: Updated to 514t. Closes: #392466
  - nb.po: Updated to 514t. Closes: #392466
  - pt.po: Updated to 514t. Closes: #393199
  - fr.po: One spelling error corrected: s/accèder/accéder
  - km.po: Updated to 514t.
  - ko.po: Updated to 514t.
  - bg.po: Updated to 514t.
  - de.po: Updated to 514t.
  - en_GB.po: Updated to 514t.
* debian/control:
  - depend on debian-archive-keyring to offer clean upgrade path 
    (closes: #386800)
* merged "install-recommends" branch (ABI break): 
  - new "--install-recommends"
  - install new recommends on "upgrade" if --install-recommends is 
    given
  - new "--fix-policy" option to install all packages with unmet
    important dependencies (usefull with --install-recommends to
    see what not-installed recommends are on the system)
  - fix of recommended packages display (only show CandidateVersion
    fix or-group handling)
* merged "install-task" branch (use with "apt-get install taskname^")
* Applied patch from Daniel Schepler to make apt bin-NMU able.
* apt-pkg/acquire-item.cc:
  - fix reversed logic of the "Acquire::PDiffs" option
* apt-pkg/contrib/sha256.cc:
  - applied patch to fix unaligned access problem. Closes: #367417
    (thanks to David Mosberger)
* apt-pkg/contrib/sha256.{cc,h},hashes.{cc,h}: support for sha256 
  (thanks to Anthony Towns)
* ftparchive/cachedb.{cc,h},writer.{cc,h}: optimizations 
  (thanks to Anthony Towns)
* apt pdiff support from experimental merged
* apt-pkg/deb/dpkgpm.cc: wording fixes (thanks to Matt Zimmerman)
  - fix error in dpkg interaction (closes: #364513, thanks to Martin Dickopp)

1 files changed, 434 insertions, 8 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index c42c8af24..b7b16b9a2 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -21,6 +21,8 @@
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/md5.h>
+#include <apt-pkg/sha1.h>
+#include <apt-pkg/tagfile.h>
 
 #include <apti18n.h>
     
@@ -28,6 +30,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <string>
+#include <sstream>
 #include <stdio.h>
 									/*}}}*/
 
@@ -106,7 +109,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string,
    // We just downloaded something..
    string FileName = LookupTag(Message,"Filename");
    UsedMirror =  LookupTag(Message,"UsedMirror");
-   if (Complete == false && FileName == DestFile)
+   if (Complete == false && !Local && FileName == DestFile)
    {
       if (Owner->Log != 0)
 	 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
@@ -179,14 +182,432 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 }
 
 
+
+// AcqDiffIndex::AcqDiffIndex - Constructor			
+// ---------------------------------------------------------------------
+/* Get the DiffIndex file first and see if there are patches availabe 
+ * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
+ * patches. If anything goes wrong in that process, it will fall back to
+ * the original packages file
+ */
+pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
+				 string URI,string URIDesc,string ShortDesc,
+				 string ExpectedMD5)
+   : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), Description(URIDesc)
+{
+   
+   Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+
+   Desc.Description = URIDesc + "/DiffIndex";
+   Desc.Owner = this;
+   Desc.ShortDesc = ShortDesc;
+   Desc.URI = URI + ".diff/Index";
+
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(URI) + string(".DiffIndex");
+
+   if(Debug)
+      std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
+
+   // look for the current package file
+   CurrentPackagesFile = _config->FindDir("Dir::State::lists");
+   CurrentPackagesFile += URItoFileName(RealURI);
+
+   // FIXME: this file:/ check is a hack to prevent fetching
+   //        from local sources. this is really silly, and
+   //        should be fixed cleanly as soon as possible
+   if(!FileExists(CurrentPackagesFile) || 
+      Desc.URI.substr(0,strlen("file:/")) == "file:/")
+   {
+      // we don't have a pkg file or we don't want to queue
+      if(Debug)
+	 std::clog << "No index file, local or canceld by user" << std::endl;
+      Failed("", NULL);
+      return;
+   }
+
+   if(Debug) 
+      std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " 
+		<< CurrentPackagesFile << std::endl;
+   
+   QueueURI(Desc);
+
+}
+
+// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
+// ---------------------------------------------------------------------
+/* The only header we use is the last-modified header. */
+string pkgAcqDiffIndex::Custom600Headers()
+{
+   string Final = _config->FindDir("Dir::State::lists");
+   Final += URItoFileName(RealURI) + string(".IndexDiff");
+   
+   if(Debug)
+      std::clog << "Custom600Header-IMS: " << Final << std::endl;
+
+   struct stat Buf;
+   if (stat(Final.c_str(),&Buf) != 0)
+      return "\nIndex-File: true";
+   
+   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+}
+
+
+bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)
+{
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile 
+		<< std::endl;
+
+   pkgTagSection Tags;
+   string ServerSha1;
+   vector<DiffInfo> available_patches;
+   
+   FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
+   pkgTagFile TF(&Fd);
+   if (_error->PendingError() == true)
+      return false;
+
+   if(TF.Step(Tags) == true)
+   {
+      string local_sha1;
+      bool found = false;
+      DiffInfo d;
+      string size;
+
+      string tmp = Tags.FindS("SHA1-Current");
+      std::stringstream ss(tmp);
+      ss >> ServerSha1;
+
+      FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
+      SHA1Summation SHA1;
+      SHA1.AddFD(fd.Fd(), fd.Size());
+      local_sha1 = string(SHA1.Result());
+
+      if(local_sha1 == ServerSha1) 
+      {
+	 // we have the same sha1 as the server
+	 if(Debug)
+	    std::clog << "Package file is up-to-date" << std::endl;
+	 // set found to true, this will queue a pkgAcqIndexDiffs with
+	 // a empty availabe_patches
+	 found = true;
+      } 
+      else 
+      {
+	 if(Debug)
+	    std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
+
+	 // check the historie and see what patches we need
+	 string history = Tags.FindS("SHA1-History");     
+	 std::stringstream hist(history);
+	 while(hist >> d.sha1 >> size >> d.file) 
+	 {
+	    d.size = atoi(size.c_str());
+	    // read until the first match is found
+	    if(d.sha1 == local_sha1) 
+	       found=true;
+	    // from that point on, we probably need all diffs
+	    if(found) 
+	    {
+	       if(Debug)
+		  std::clog << "Need to get diff: " << d.file << std::endl;
+	       available_patches.push_back(d);
+	    }
+	 }
+      }
+
+      // we have something, queue the next diff
+      if(found) 
+      {
+	 // queue the diffs
+	 int last_space = Description.rfind(" ");
+	 if(last_space != string::npos)
+	    Description.erase(last_space, Description.size()-last_space);
+	 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
+			      ExpectedMD5, available_patches);
+	 Complete = false;
+	 Status = StatDone;
+	 Dequeue();
+	 return true;
+      }
+   }
+   
+   // Nothing found, report and return false
+   // Failing here is ok, if we return false later, the full
+   // IndexFile is queued
+   if(Debug)
+      std::clog << "Can't find a patch in the index file" << std::endl;
+   return false;
+}
+
+void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
+		<< "Falling back to normal index file aquire" << std::endl;
+
+   new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, 
+		   ExpectedMD5);
+
+   Complete = false;
+   Status = StatDone;
+   Dequeue();
+}
+
+void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,
+			   pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
+
+   Item::Done(Message,Size,Md5Hash,Cnf);
+
+   string FinalFile;
+   FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
+
+   // sucess in downloading the index
+   // rename the index
+   FinalFile += string(".IndexDiff");
+   if(Debug)
+      std::clog << "Renaming: " << DestFile << " -> " << FinalFile 
+		<< std::endl;
+   Rename(DestFile,FinalFile);
+   chmod(FinalFile.c_str(),0644);
+   DestFile = FinalFile;
+
+   if(!ParseDiffIndex(DestFile))
+      return Failed("", NULL);
+
+   Complete = true;
+   Status = StatDone;
+   Dequeue();
+   return;
+}
+
+
+
+// AcqIndexDiffs::AcqIndexDiffs - Constructor			
+// ---------------------------------------------------------------------
+/* The package diff is added to the queue. one object is constructed
+ * for each diff and the index
+ */
+pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
+				   string URI,string URIDesc,string ShortDesc,
+				   string ExpectedMD5, vector<DiffInfo> diffs)
+   : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), 
+     available_patches(diffs)
+{
+   
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(URI);
+
+   Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+
+   Description = URIDesc;
+   Desc.Owner = this;
+   Desc.ShortDesc = ShortDesc;
+
+   if(available_patches.size() == 0) 
+   {
+      // we are done (yeah!)
+      Finish(true);
+   }
+   else
+   {
+      // get the next diff
+      State = StateFetchDiff;
+      QueueNextDiff();
+   }
+}
+
+
+void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
+		<< "Falling back to normal index file aquire" << std::endl;
+   new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, 
+		   ExpectedMD5);
+   Finish();
+}
+
+
+// helper that cleans the item out of the fetcher queue
+void pkgAcqIndexDiffs::Finish(bool allDone)
+{
+   // we restore the original name, this is required, otherwise
+   // the file will be cleaned
+   if(allDone) 
+   {
+      DestFile = _config->FindDir("Dir::State::lists");
+      DestFile += URItoFileName(RealURI);
+
+      // do the final md5sum checking
+      MD5Summation sum;
+      FileFd Fd(DestFile, FileFd::ReadOnly);
+      sum.AddFD(Fd.Fd(), Fd.Size());
+      Fd.Close();
+      string MD5 = (string)sum.Result();
+
+      if (!ExpectedMD5.empty() && MD5 != ExpectedMD5)
+      {
+	 Status = StatAuthError;
+	 ErrorText = _("MD5Sum mismatch");
+	 Rename(DestFile,DestFile + ".FAILED");
+	 Dequeue();
+	 return;
+      }
+
+      // this is for the "real" finish
+      Complete = true;
+      Status = StatDone;
+      Dequeue();
+      if(Debug)
+	 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
+      return;
+   }
+
+   if(Debug)
+      std::clog << "Finishing: " << Desc.URI << std::endl;
+   Complete = false;
+   Status = StatDone;
+   Dequeue();
+   return;
+}
+
+
+
+bool pkgAcqIndexDiffs::QueueNextDiff()
+{
+
+   // calc sha1 of the just patched file
+   string FinalFile = _config->FindDir("Dir::State::lists");
+   FinalFile += URItoFileName(RealURI);
+
+   FileFd fd(FinalFile, FileFd::ReadOnly);
+   SHA1Summation SHA1;
+   SHA1.AddFD(fd.Fd(), fd.Size());
+   string local_sha1 = string(SHA1.Result());
+   if(Debug)
+      std::clog << "QueueNextDiff: " 
+		<< FinalFile << " (" << local_sha1 << ")"<<std::endl;
+
+   // remove all patches until the next matching patch is found
+   // this requires the Index file to be ordered
+   for(vector<DiffInfo>::iterator I=available_patches.begin();
+       available_patches.size() > 0 && 
+	  I != available_patches.end() &&
+	  (*I).sha1 != local_sha1; 
+       I++) 
+   {
+      available_patches.erase(I);
+   }
+
+   // error checking and falling back if no patch was found
+   if(available_patches.size() == 0) 
+   { 
+      Failed("", NULL);
+      return false;
+   }
+
+   // queue the right diff
+   Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
+   Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
+
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
+   
+   QueueURI(Desc);
+
+   return true;
+}
+
+
+
+void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,
+			    pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
+
+   Item::Done(Message,Size,Md5Hash,Cnf);
+
+   string FinalFile;
+   FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
+
+   // sucess in downloading a diff, enter ApplyDiff state
+   if(State == StateFetchDiff) 
+   {
+
+      if(Debug)
+	 std::clog << "Sending to gzip method: " << FinalFile << std::endl;
+
+      string FileName = LookupTag(Message,"Filename");
+      State = StateUnzipDiff;
+      Local = true;
+      Desc.URI = "gzip:" + FileName;
+      DestFile += ".decomp";
+      QueueURI(Desc);
+      Mode = "gzip";
+      return;
+   } 
+
+   // sucess in downloading a diff, enter ApplyDiff state
+   if(State == StateUnzipDiff) 
+   {
+
+      // rred excepts the patch as $FinalFile.ed
+      Rename(DestFile,FinalFile+".ed");
+
+      if(Debug)
+	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
+
+      State = StateApplyDiff;
+      Local = true;
+      Desc.URI = "rred:" + FinalFile;
+      QueueURI(Desc);
+      Mode = "rred";
+      return;
+   } 
+
+
+   // success in download/apply a diff, queue next (if needed)
+   if(State == StateApplyDiff)
+   {
+      // remove the just applied patch
+      available_patches.erase(available_patches.begin());
+
+      // move into place
+      if(Debug) 
+      {
+	 std::clog << "Moving patched file in place: " << std::endl
+		   << DestFile << " -> " << FinalFile << std::endl;
+      }
+      Rename(DestFile,FinalFile);
+      chmod(FinalFile.c_str(),0644);
+
+      // see if there is more to download
+      if(available_patches.size() > 0) {
+	 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
+			      ExpectedMD5, available_patches);
+	 return Finish();
+      } else 
+	 return Finish(true);
+   }
+}
+
+
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* The package file is added to the queue and a second class is 
    instantiated to fetch the revision file */   
 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 			 string URI,string URIDesc,string ShortDesc,
-			 string ExpectedMD5, string comprExt) :
-   Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5)
+			 string ExpectedMD5, string comprExt)
+   : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5)
 {
    Decompression = false;
    Erase = false;
@@ -230,7 +651,6 @@ string pkgAcqIndex::Custom600Headers()
 
 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
-
    // no .bz2 found, retry with .gz
    if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") {
       Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz"; 
@@ -480,7 +900,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
    if(Status == StatTransientNetworkError)
    {
       Item::Failed(Message,Cnf);
-      // move the sigfile back on network failures (and re-authenticated?)
+      // move the sigfile back on transient network failures 
       if(FileExists(DestFile))
  	 Rename(DestFile,Final);
 
@@ -622,6 +1042,7 @@ void pkgAcqMetaIndex::RetrievalDone(string Message)
       // Move it into position
       Rename(DestFile,FinalFile);
    }
+   chmod(FinalFile.c_str(),0644);
    DestFile = FinalFile;
 }
 
@@ -691,9 +1112,14 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)
          }
       }
       
-      // Queue Packages file
-      new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
-                      (*Target)->ShortDesc, ExpectedIndexMD5);
+      // Queue Packages file (either diff or full packages files, depending
+      // on the users option)
+      if(_config->FindB("Acquire::PDiffs",false) == true) 
+	 new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
+			     (*Target)->ShortDesc, ExpectedIndexMD5);
+      else 
+	 new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
+			    (*Target)->ShortDesc, ExpectedIndexMD5);
    }
 }