summaryrefslogtreecommitdiff
path: root/apt-pkg/acquire-item.cc
diff options
context:
space:
mode:
authorDavid Kalnischkies <david@kalnischkies.de>2016-01-07 20:32:09 +0100
committerDavid Kalnischkies <david@kalnischkies.de>2016-01-08 15:40:01 +0100
commit0179cfa83cf0042235eda41db7f35c420781c63e (patch)
treea1b49bb2d40ae34a5d06139e7ecd203ba0733c8f /apt-pkg/acquire-item.cc
parent912a61312a0463b46d6560756c89146f59daaab6 (diff)
keep compressed indexes in a low-cost format
Downloading and storing are two different operations were different compression types can be preferred. For downloading we provide the choice via Acquire::CompressionTypes::Order as there is a choice to be made between download size and speed – and limited by whats available in the repository. Storage on the other hand has all compressions currently supported by apt available and to reduce runtime of tools accessing these files the compression type should be a low-cost format in terms of decompression. apt traditionally stores its indexes uncompressed on disk, but has options to keep them compressed. Now that apt downloads additional files we also deal with files which simply can't be stored uncompressed as they are just too big (like Contents for apt-file). Traditionally they are downloaded in a low-cost format (gz) as repositories do not provide other formats, but there might be even lower-cost formats and for download we could introduce higher-cost in the repositories. Downloading an entire index potentially requires recompression to another format, so an update takes potentially longer – but big files are usually updated via pdiffs which has to de- and re-compress anyhow and does it on the fly anyhow, so there is no extra time needed and in general it seems to be benefitial to invest the time in update to save time later on file access.
Diffstat (limited to 'apt-pkg/acquire-item.cc')
-rw-r--r--apt-pkg/acquire-item.cc177
1 files changed, 78 insertions, 99 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 82ccaff64..7c7a204c4 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -85,32 +85,16 @@ static std::string GetKeepCompressedFileName(std::string file, IndexTarget const
if (Target.KeepCompressed == false)
return file;
- std::string const CompressionTypes = Target.Option(IndexTarget::COMPRESSIONTYPES);
- if (CompressionTypes.empty() == false)
+ std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
+ if (KeepCompressedAs.empty() == false)
{
- std::string const ext = CompressionTypes.substr(0, CompressionTypes.find(' '));
+ std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
if (ext != "uncompressed")
file.append(".").append(ext);
}
return file;
}
/*}}}*/
-static std::string GetCompressedFileName(IndexTarget const &Target, std::string const &Name, std::string const &Ext) /*{{{*/
-{
- if (Ext.empty() || Ext == "uncompressed")
- return Name;
-
- // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
- // file when its doing the indexcopy
- if (Target.URI.substr(0,6) == "cdrom:")
- return Name;
-
- // adjust DestFile if its compressed on disk
- if (Target.KeepCompressed == true)
- return Name + '.' + Ext;
- return Name;
-}
- /*}}}*/
static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
{
// rred expects the patch as $FinalFile.ed.$patchname.gz
@@ -326,7 +310,7 @@ std::string pkgAcqDiffIndex::GetFinalFilename() const
std::string pkgAcqIndex::GetFinalFilename() const
{
std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
- return GetCompressedFileName(Target, FinalFile, CurrentCompressionExtension);
+ return GetKeepCompressedFileName(FinalFile, Target);
}
std::string pkgAcqMetaSig::GetFinalFilename() const
{
@@ -379,7 +363,30 @@ bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
case TransactionCommit:
if(PartialFile.empty() == false)
{
- if (PartialFile != DestFile)
+ bool sameFile = (PartialFile == DestFile);
+ // we use symlinks on IMS-Hit to avoid copies
+ if (RealFileExists(DestFile))
+ {
+ struct stat Buf;
+ if (lstat(PartialFile.c_str(), &Buf) != -1)
+ {
+ if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
+ {
+ char partial[Buf.st_size + 1];
+ ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
+ if (sp == -1)
+ _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
+ else
+ {
+ partial[sp] = '\0';
+ sameFile = (DestFile == partial);
+ }
+ }
+ }
+ else
+ _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
+ }
+ if (sameFile == false)
{
// ensure that even without lists-cleanup all compressions are nuked
std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
@@ -2621,20 +2628,6 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
TransactionManager->AbortTransaction();
}
/*}}}*/
-// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/
-void pkgAcqIndex::ReverifyAfterIMS()
-{
- // update destfile to *not* include the compression extension when doing
- // a reverify (as its uncompressed on disk already)
- DestFile = GetCompressedFileName(Target, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);
-
- // copy FinalFile into partial/ so that we check the hash again
- string FinalFile = GetFinalFilename();
- Stage = STAGE_DECOMPRESS_AND_VERIFY;
- Desc.URI = "copy:" + FinalFile;
- QueueURI(Desc);
-}
- /*}}}*/
// AcqIndex::Done - Finished a fetch /*{{{*/
// ---------------------------------------------------------------------
/* This goes through a number of states.. On the initial fetch the
@@ -2651,107 +2644,93 @@ void pkgAcqIndex::Done(string const &Message,
switch(Stage)
{
case STAGE_DOWNLOAD:
- StageDownloadDone(Message, Hashes, Cfg);
+ StageDownloadDone(Message);
break;
case STAGE_DECOMPRESS_AND_VERIFY:
- StageDecompressDone(Message, Hashes, Cfg);
+ StageDecompressDone();
break;
}
}
/*}}}*/
// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/
-void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const &,
- pkgAcquire::MethodConfig const * const)
+void pkgAcqIndex::StageDownloadDone(string const &Message)
{
+ Local = true;
Complete = true;
- // Handle the unzipd case
- std::string FileName = LookupTag(Message,"Alt-Filename");
- if (FileName.empty() == false)
+ std::string const AltFilename = LookupTag(Message,"Alt-Filename");
+ std::string Filename = LookupTag(Message,"Filename");
+
+ // we need to verify the file against the current Release file again
+ // on if-modfied-since hit to avoid a stale attack against us
+ if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
{
+ // copy FinalFile into partial/ so that we check the hash again
+ string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
+ if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0)
+ _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str());
+ else
+ {
+ EraseFileName = DestFile;
+ Filename = DestFile;
+ }
Stage = STAGE_DECOMPRESS_AND_VERIFY;
- Local = true;
- if (CurrentCompressionExtension != "uncompressed")
- DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
- Desc.URI = "copy:" + FileName;
+ Desc.URI = "store:" + Filename;
QueueURI(Desc);
- SetActiveSubprocess("copy");
+ SetActiveSubprocess(::URI(Desc.URI).Access);
return;
}
- FileName = LookupTag(Message,"Filename");
-
+ // methods like file:// give us an alternative (uncompressed) file
+ else if (Target.KeepCompressed == false && AltFilename.empty() == false)
+ {
+ if (CurrentCompressionExtension != "uncompressed")
+ DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
+ Filename = AltFilename;
+ }
// Methods like e.g. "file:" will give us a (compressed) FileName that is
// not the "DestFile" we set, in this case we uncompress from the local file
- if (FileName != DestFile && RealFileExists(DestFile) == false)
+ else if (Filename != DestFile && RealFileExists(DestFile) == false)
{
- Local = true;
- if (Target.KeepCompressed == true)
- {
- // but if we don't keep the uncompress we copy the compressed file first
- Stage = STAGE_DOWNLOAD;
- Desc.URI = "copy:" + FileName;
- QueueURI(Desc);
- SetActiveSubprocess("copy");
- return;
- }
+ // symlinking ensures that the filename can be used for compression detection
+ // that is e.g. needed for by-hash which has no extension over file
+ if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
+ _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
else
{
- // symlinking ensures that the filename can be used for compression detection
- // that is e.g. needed for by-hash over file
- if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
- _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", FileName.c_str(), DestFile.c_str());
- else
- {
- EraseFileName = DestFile;
- FileName = DestFile;
- }
+ EraseFileName = DestFile;
+ Filename = DestFile;
}
}
- else
- EraseFileName = FileName;
-
- // we need to verify the file against the current Release file again
- // on if-modfied-since hit to avoid a stale attack against us
- if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
- {
- // The files timestamp matches, reverify by copy into partial/
- EraseFileName = "";
- ReverifyAfterIMS();
- return;
- }
- string decompProg = "store";
- if (Target.KeepCompressed == true)
- {
- DestFile = "/dev/null";
- EraseFileName.clear();
- }
+ Stage = STAGE_DECOMPRESS_AND_VERIFY;
+ DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
+ if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
+ Desc.URI = "copy:" + Filename;
else
+ Desc.URI = "store:" + Filename;
+ if (DestFile == Filename)
{
if (CurrentCompressionExtension == "uncompressed")
- decompProg = "copy";
- else
- DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
+ return StageDecompressDone();
+ DestFile = "/dev/null";
}
+ if (EraseFileName.empty())
+ EraseFileName = Filename;
+
// queue uri for the next stage
- Stage = STAGE_DECOMPRESS_AND_VERIFY;
- Desc.URI = decompProg + ":" + FileName;
QueueURI(Desc);
- SetActiveSubprocess(decompProg);
+ SetActiveSubprocess(::URI(Desc.URI).Access);
}
/*}}}*/
// AcqIndex::StageDecompressDone - Final verification /*{{{*/
-void pkgAcqIndex::StageDecompressDone(string const &,
- HashStringList const &,
- pkgAcquire::MethodConfig const * const)
+void pkgAcqIndex::StageDecompressDone()
{
- if (Target.KeepCompressed == true && DestFile == "/dev/null")
- DestFile = GetPartialFileNameFromURI(Target.URI + '.' + CurrentCompressionExtension);
+ if (DestFile == "/dev/null")
+ DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
// Done, queue for rename on transaction finished
TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
- return;
}
/*}}}*/
pkgAcqIndex::~pkgAcqIndex() {}