From 73da43e90be945d3be9b4f3b6e5016fb7bacb59d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 5 Jul 2011 15:58:32 +0200 Subject: * apt-pkg/acquire*.{cc,h}: - try even harder to support really big files in the fetcher by converting (hopefully) everything to 'long long' (Closes: #632271) --- apt-pkg/acquire-item.cc | 30 +++++++++++++++--------------- apt-pkg/acquire-item.h | 22 +++++++++++----------- apt-pkg/acquire-method.h | 4 ++-- apt-pkg/acquire-worker.cc | 4 ++-- apt-pkg/acquire-worker.h | 6 +++--- apt-pkg/acquire.cc | 2 +- apt-pkg/acquire.h | 11 +++-------- debian/changelog | 5 ++++- 8 files changed, 41 insertions(+), 43 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d790bd898..a6698b367 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -94,7 +94,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) // --------------------------------------------------------------------- /* Stash status and the file size. Note that setting Complete means sub-phases of the acquire process such as decompresion are operating */ -void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size) +void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size) { Status = StatFetching; if (FileSize == 0 && Complete == false) @@ -104,7 +104,7 @@ void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size) // Acquire::Item::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash, +void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash, pkgAcquire::MethodConfig *Cnf) { // We just downloaded something.. @@ -245,7 +245,7 @@ void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{* } } /*}}}*/ -void pkgAcqSubIndex::Done(string Message,unsigned long Size,string Md5Hash, /*{{{*/ +void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@ -544,7 +544,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{ Dequeue(); } /*}}}*/ -void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash, /*{{{*/ +void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@ -710,7 +710,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ return true; } /*}}}*/ -void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, /*{{{*/ +void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@ -881,7 +881,7 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a gzip uri. */ -void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, +void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hash,Cfg); @@ -1123,7 +1123,7 @@ string pkgAcqMetaSig::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } -void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5, +void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,MD5,Cfg); @@ -1232,7 +1232,7 @@ string pkgAcqMetaIndex::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ -void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, /*{{{*/ +void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash, /*{{{*/ pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hash,Cfg); @@ -1768,7 +1768,7 @@ bool pkgAcqArchive::QueueNext() if (stat(FinalFile.c_str(),&Buf) == 0) { // Make sure the size matches - if ((unsigned)Buf.st_size == Version->Size) + if ((unsigned long long)Buf.st_size == Version->Size) { Complete = true; Local = true; @@ -1787,7 +1787,7 @@ bool pkgAcqArchive::QueueNext() if (stat(FinalFile.c_str(),&Buf) == 0) { // Make sure the size matches - if ((unsigned)Buf.st_size == Version->Size) + if ((unsigned long long)Buf.st_size == Version->Size) { Complete = true; Local = true; @@ -1807,7 +1807,7 @@ bool pkgAcqArchive::QueueNext() if (stat(DestFile.c_str(),&Buf) == 0) { // Hmm, the partial file is too big, erase it - if ((unsigned)Buf.st_size > Version->Size) + if ((unsigned long long)Buf.st_size > Version->Size) unlink(DestFile.c_str()); else PartialSize = Buf.st_size; @@ -1830,7 +1830,7 @@ bool pkgAcqArchive::QueueNext() // AcqArchive::Done - Finished fetching /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash, +void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,CalcHash,Cfg); @@ -1941,7 +1941,7 @@ void pkgAcqArchive::Finished() // --------------------------------------------------------------------- /* The file is added to the queue */ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, - unsigned long Size,string Dsc,string ShortDesc, + unsigned long long Size,string Dsc,string ShortDesc, const string &DestDir, const string &DestFilename, bool IsIndexFile) : Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile) @@ -1969,7 +1969,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, if (stat(DestFile.c_str(),&Buf) == 0) { // Hmm, the partial file is too big, erase it - if ((unsigned)Buf.st_size > Size) + if ((unsigned long long)Buf.st_size > Size) unlink(DestFile.c_str()); else PartialSize = Buf.st_size; @@ -1981,7 +1981,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, // AcqFile::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash, +void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash, pkgAcquire::MethodConfig *Cnf) { Item::Done(Message,Size,CalcHash,Cnf); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index f763577ee..e6916a834 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -194,7 +194,7 @@ class pkgAcquire::Item : public WeakPointable * * \sa pkgAcqMethod */ - virtual void Done(string Message,unsigned long Size,string Hash, + virtual void Done(string Message,unsigned long long Size,string Hash, pkgAcquire::MethodConfig *Cnf); /** \brief Invoked when the worker starts to fetch this object. @@ -206,7 +206,7 @@ class pkgAcquire::Item : public WeakPointable * * \sa pkgAcqMethod */ - virtual void Start(string Message,unsigned long Size); + virtual void Start(string Message,unsigned long long Size); /** \brief Custom headers to be sent to the fetch process. * @@ -309,7 +309,7 @@ class pkgAcqSubIndex : public pkgAcquire::Item public: // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size,string Md5Hash, + virtual void Done(string Message,unsigned long long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string DescURI() {return Desc.URI;}; virtual string Custom600Headers(); @@ -372,7 +372,7 @@ class pkgAcqDiffIndex : public pkgAcquire::Item public: // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size,string Md5Hash, + virtual void Done(string Message,unsigned long long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string DescURI() {return RealURI + "Index";}; virtual string Custom600Headers(); @@ -508,7 +508,7 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item */ virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size,string Md5Hash, + virtual void Done(string Message,unsigned long long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string DescURI() {return RealURI + "Index";}; @@ -581,7 +581,7 @@ class pkgAcqIndex : public pkgAcquire::Item // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size,string Md5Hash, + virtual void Done(string Message,unsigned long long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string Custom600Headers(); virtual string DescURI() {return Desc.URI;}; @@ -723,7 +723,7 @@ class pkgAcqMetaSig : public pkgAcquire::Item // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size,string Md5Hash, + virtual void Done(string Message,unsigned long long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string Custom600Headers(); virtual string DescURI() {return RealURI; }; @@ -818,7 +818,7 @@ class pkgAcqMetaIndex : public pkgAcquire::Item // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size, string Hash, + virtual void Done(string Message,unsigned long long Size, string Hash, pkgAcquire::MethodConfig *Cnf); virtual string Custom600Headers(); virtual string DescURI() {return RealURI; }; @@ -918,7 +918,7 @@ class pkgAcqArchive : public pkgAcquire::Item public: virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size,string Hash, + virtual void Done(string Message,unsigned long long Size,string Hash, pkgAcquire::MethodConfig *Cnf); virtual string DescURI() {return Desc.URI;}; virtual string ShortDesc() {return Desc.ShortDesc;}; @@ -975,7 +975,7 @@ class pkgAcqFile : public pkgAcquire::Item // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(string Message,unsigned long Size,string CalcHash, + virtual void Done(string Message,unsigned long long Size,string CalcHash, pkgAcquire::MethodConfig *Cnf); virtual string DescURI() {return Desc.URI;}; virtual string HashSum() {return ExpectedHash.toStr(); }; @@ -1012,7 +1012,7 @@ class pkgAcqFile : public pkgAcquire::Item * is the absolute name to which the file should be downloaded. */ - pkgAcqFile(pkgAcquire *Owner, string URI, string Hash, unsigned long Size, + pkgAcqFile(pkgAcquire *Owner, string URI, string Hash, unsigned long long Size, string Desc, string ShortDesc, const string &DestDir="", const string &DestFilename="", bool IsIndexFile=false); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 2d0fa4590..6551170c4 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -51,8 +51,8 @@ class pkgAcqMethod time_t LastModified; bool IMSHit; string Filename; - unsigned long Size; - unsigned long ResumePoint; + unsigned long long Size; + unsigned long long ResumePoint; void TakeHashes(Hashes &Hash); FetchResult(); diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 75e03232a..3e1fd98db 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -287,9 +287,9 @@ bool pkgAcquire::Worker::RunMessages() Log->Pulse(Owner->GetOwner()); OwnerQ->ItemDone(Itm); - unsigned long const ServerSize = atol(LookupTag(Message,"Size","0").c_str()); + unsigned long long const ServerSize = atoll(LookupTag(Message,"Size","0").c_str()); if (TotalSize != 0 && ServerSize != TotalSize) - _error->Warning("Size of file %s is not what the server reported %s %lu", + _error->Warning("Size of file %s is not what the server reported %s %llu", Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize); // see if there is a hash to verify diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h index 62545829a..ce19091e4 100644 --- a/apt-pkg/acquire-worker.h +++ b/apt-pkg/acquire-worker.h @@ -247,17 +247,17 @@ class pkgAcquire::Worker : public WeakPointable /** \brief How many bytes of the file have been downloaded. Zero * if the current progress of the file cannot be determined. */ - unsigned long CurrentSize; + unsigned long long CurrentSize; /** \brief The total number of bytes to be downloaded. Zero if the * total size of the final is unknown. */ - unsigned long TotalSize; + unsigned long long TotalSize; /** \brief How much of the file was already downloaded prior to * starting this worker. */ - unsigned long ResumePoint; + unsigned long long ResumePoint; /** \brief Tell the subprocess to download the given item. * diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 939c7f4c5..fff487dc8 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -912,7 +912,7 @@ void pkgAcquireStatus::Stop() // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/ // --------------------------------------------------------------------- /* This is used to get accurate final transfer rate reporting. */ -void pkgAcquireStatus::Fetched(unsigned long Size,unsigned long Resume) +void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume) { FetchedBytes += Size - Resume; } diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 7db7a9958..8fad91497 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -148,12 +148,7 @@ class pkgAcquire /** \brief The progress indicator for this download. */ pkgAcquireStatus *Log; - /** \brief The total size of the files which are to be fetched. - * - * This is not necessarily the total number of bytes to download - * when, e.g., download resumption and list updates via patches - * are taken into account. - */ + /** \brief The number of files which are to be fetched. */ unsigned long ToFetch; // Configurable parameters for the scheduler @@ -484,7 +479,7 @@ class pkgAcquire::Queue * * \todo Unimplemented. Implement it or remove? */ - bool ItemStart(QItem *Itm,unsigned long Size); + bool ItemStart(QItem *Itm,unsigned long long Size); /** \brief Remove the given item from this queue and set its state * to pkgAcquire::Item::StatDone. @@ -734,7 +729,7 @@ class pkgAcquireStatus * * \param ResumePoint How much of the file was already fetched. */ - virtual void Fetched(unsigned long Size,unsigned long ResumePoint); + virtual void Fetched(unsigned long long Size,unsigned long long ResumePoint); /** \brief Invoked when the user should be prompted to change the * inserted removable media. diff --git a/debian/changelog b/debian/changelog index af6db28b9..cfbea1df7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,8 +4,11 @@ apt (0.8.16~exp3) experimental; urgency=low * apt-pkg/pkgcache.h: - readd All{Foreign,Allowed} as suggested by Julian to remain strictly API compatible + * apt-pkg/acquire*.{cc,h}: + - try even harder to support really big files in the fetcher by + converting (hopefully) everything to 'long long' (Closes: #632271) - -- David Kalnischkies Wed, 29 Jun 2011 19:38:44 +0200 + -- David Kalnischkies Tue, 05 Jul 2011 15:56:42 +0200 apt (0.8.16~exp2) experimental; urgency=low -- cgit v1.2.3