diff options
Diffstat (limited to 'apt-pkg')
31 files changed, 2366 insertions, 121 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index cc16215f5..966126255 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -135,7 +135,6 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ - // AcqDiffIndex::AcqDiffIndex - Constructor // --------------------------------------------------------------------- /* Get the DiffIndex file first and see if there are patches availabe @@ -729,6 +728,35 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5, Mode = decompProg; } +// AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* The Translation file is added to the queue */ +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc) : + pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, "", "") +{ +} + + /*}}}*/ +// AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) + { + // Ignore this + Status = StatDone; + Complete = false; + Dequeue(); + return; + } + + Item::Failed(Message,Cnf); +} + /*}}}*/ + pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, string MetaIndexURI, string MetaIndexURIDesc, diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 3649d7a03..217ddb3ef 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -9,8 +9,8 @@ the Owner Acquire class. Derived classes will then call QueueURI to register all the URI's they wish to fetch at the initial moment. - Two item classes are provided to provide functionality for downloading - of Index files and downloading of Packages. + Three item classes are provided to provide functionality for + downloading of Index, Translation and Packages files. A Archive class is provided for downloading .deb files. It does Md5 checking and source location as well as a retry algorithm. @@ -31,73 +31,283 @@ #pragma interface "apt-pkg/acquire-item.h" #endif -// Item to acquire +/** \addtogroup acquire + * @{ + * + * \file acquire-item.h + */ + +/** \brief Represents the process by which a pkgAcquire object should + * retrieve a file or a collection of files. + * + * By convention, Item subclasses should insert themselves into the + * acquire queue when they are created by calling QueueURI(), and + * remove themselves by calling Dequeue() when either Done() or + * Failed() is invoked. Item objects are also responsible for + * notifying the download progress indicator (accessible via + * #Owner->Log) of their status. + * + * \see pkgAcquire + */ class pkgAcquire::Item { protected: - // Some private helper methods for registering URIs + /** \brief The acquire object with which this item is associated. */ pkgAcquire *Owner; + + /** \brief Insert this item into its owner's queue. + * + * \param ItemDesc Metadata about this item (its URI and + * description). + */ inline void QueueURI(ItemDesc &Item) {Owner->Enqueue(Item);}; + + /** \brief Remove this item from its owner's queue. */ inline void Dequeue() {Owner->Dequeue(this);}; - // Safe rename function with timestamp preservation + /** \brief Rename a file without modifying its timestamp. + * + * Many item methods call this as their final action. + * + * \param From The file to be renamed. + * + * \param To The new name of #From. If #To exists it will be + * overwritten. + */ void Rename(string From,string To); public: - // State of the item - enum {StatIdle, StatFetching, StatDone, StatError, StatAuthError} Status; + /** \brief The current status of this item. */ + enum ItemState + { + /** \brief The item is waiting to be downloaded. */ + StatIdle, + + /** \brief The item is currently being downloaded. */ + StatFetching, + + /** \brief The item has been successfully downloaded. */ + StatDone, + + /** \brief An error was encountered while downloading this + * item. + */ + StatError, + + /** \brief The item was downloaded but its authenticity could + * not be verified. + */ + StatAuthError + } Status; + + /** \brief Contains a textual description of the error encountered + * if #Status is #StatError or #StatAuthError. + */ string ErrorText; + + /** \brief The size of the object to fetch. */ unsigned long FileSize; - unsigned long PartialSize; + + /** \brief How much of the object was already fetched. */ + unsigned long PartialSize; + + /** \brief If not \b NULL, contains the name of a subprocess that + * is operating on this object (for instance, "gzip" or "gpgv"). + */ const char *Mode; + + /** \brief A client-supplied unique identifier. + * + * This field is initalized to 0; it is meant to be filled in by + * clients that wish to use it to uniquely identify items. + * + * \todo it's unused in apt itself + */ unsigned long ID; + + /** \brief If \b true, the entire object has been successfully fetched. + * + * Subclasses should set this to \b true when appropriate. + */ bool Complete; + + /** \brief If \b true, the URI of this object is "local". + * + * The only effect of this field is to exclude the object from the + * download progress indicator's overall statistics. + */ bool Local; - // Number of queues we are inserted into + /** \brief The number of fetch queues into which this item has been + * inserted. + * + * There is one queue for each source from which an item could be + * downloaded. + * + * \sa pkgAcquire + */ unsigned int QueueCounter; - // File to write the fetch into + /** \brief The name of the file into which the retrieved object + * will be written. + */ string DestFile; - // Action members invoked by the worker + /** \brief Invoked by the acquire worker when the object couldn't + * be fetched. + * + * This is a branch of the continuation of the fetch process. + * + * \param Message An RFC822-formatted message from the acquire + * method describing what went wrong. Use LookupTag() to parse + * it. + * + * \param Cnf The method via which the worker tried to fetch this object. + * + * \sa pkgAcqMethod + */ virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + + /** \brief Invoked by the acquire worker when the object was + * fetched successfully. + * + * Note that the object might \e not have been written to + * DestFile; check for the presence of an Alt-Filename entry in + * Message to find the file to which it was really written. + * + * Done is often used to switch from one stage of the processing + * to the next (e.g. fetching, unpacking, copying). It is one + * branch of the continuation of the fetch process. + * + * \param Message Data from the acquire method. Use LookupTag() + * to parse it. + * \param Size The size of the object that was fetched. + * \param Md5Hash The MD5Sum of the object that was fetched. + * \param Cnf The method via which the object was fetched. + * + * \sa pkgAcqMethod + */ virtual void Done(string Message,unsigned long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); + + /** \brief Invoked when the worker starts to fetch this object. + * + * \param Message RFC822-formatted data from the worker process. + * Use LookupTag() to parse it. + * + * \param Size The size of the object being fetched. + * + * \sa pkgAcqMethod + */ virtual void Start(string Message,unsigned long Size); + + /** \brief Custom headers to be sent to the fetch process. + * + * \return a string containing RFC822-style headers that are to be + * inserted into the 600 URI Acquire message sent to the fetch + * subprocess. The headers are inserted after a newline-less + * line, so they should (if nonempty) have a leading newline and + * no trailing newline. + */ virtual string Custom600Headers() {return string();}; + + /** \brief A "descriptive" URI-like string. + * + * \return a URI that should be used to describe what is being fetched. + */ virtual string DescURI() = 0; + /** \brief Short item description. + * + * \return a brief description of the object being fetched. + */ virtual string ShortDesc() {return DescURI();} + + /** \brief Invoked by the worker when the download is completely done. */ virtual void Finished() {}; - // Inquire functions + /** \brief MD5Sum. + * + * \return the MD5Sum of this object, if applicable; otherwise, an + * empty string. + */ virtual string MD5Sum() {return string();}; + + /** \return the acquire process with which this item is associated. */ pkgAcquire *GetOwner() {return Owner;}; + + /** \return \b true if this object is being fetched from a trusted source. */ virtual bool IsTrusted() {return false;}; - + + /** \brief Initialize an item. + * + * Adds the item to the list of items known to the acquire + * process, but does not place it into any fetch queues (you must + * manually invoke QueueURI() to do so). + * + * Initializes all fields of the item other than Owner to 0, + * false, or the empty string. + * + * \param Owner The new owner of this item. + */ Item(pkgAcquire *Owner); + + /** \brief Remove this item from its owner's queue by invoking + * pkgAcquire::Remove. + */ virtual ~Item(); }; -// item for index diffs - +/** \brief Information about an index patch (aka diff). */ struct DiffInfo { + /** The filename of the diff. */ string file; + + /** The sha1 hash of the diff. */ string sha1; + + /** The size of the diff. */ unsigned long size; }; +/** \brief An item that is responsible for fetching an index file of + * package list diffs and starting the package list's download. + * + * This item downloads the Index file and parses it, then enqueues + * additional downloads of either the individual patches (using + * pkgAcqIndexDiffs) or the entire Packages file (using pkgAcqIndex). + * + * \sa pkgAcqIndexDiffs, pkgAcqIndex + */ class pkgAcqDiffIndex : public pkgAcquire::Item { protected: + /** \brief If \b true, debugging information will be written to std::clog. */ bool Debug; + + /** \brief The item that is currently being downloaded. */ pkgAcquire::ItemDesc Desc; + + /** \brief The URI of the index file to recreate at our end (either + * by downloading it or by applying partial patches). + */ string RealURI; + + /** \brief The MD5Sum that the real index file should have after + * all patches have been applied. + */ string ExpectedMD5; + + /** \brief The index file which will be patched to generate the new + * file. + */ string CurrentPackagesFile; + + /** \brief A description of the Packages file (stored in + * pkgAcquire::ItemDesc::Description). + */ string Description; public: @@ -108,54 +318,199 @@ class pkgAcqDiffIndex : public pkgAcquire::Item virtual string DescURI() {return RealURI + "Index";}; virtual string Custom600Headers(); - // helpers + /** \brief Parse the Index file for a set of Packages diffs. + * + * Parses the Index file and creates additional download items as + * necessary. + * + * \param IndexDiffFile The name of the Index file. + * + * \return \b true if the Index file was successfully parsed, \b + * false otherwise. + */ bool ParseDiffIndex(string IndexDiffFile); + + /** \brief Create a new pkgAcqDiffIndex. + * + * \param Owner The Acquire object that owns this item. + * + * \param URI The URI of the list file to download. + * + * \param URIDesc A long description of the list file to download. + * + * \param ShortDesc A short description of the list file to download. + * + * \param ExpectedMD5 The list file's MD5 signature. + */ pkgAcqDiffIndex(pkgAcquire *Owner,string URI,string URIDesc, - string ShortDesct, string ExpectedMD5); + string ShortDesc, string ExpectedMD5); }; +/** \brief An item that is responsible for fetching all the patches + * that need to be applied to a given package index file. + * + * After downloading and applying a single patch, this item will + * enqueue a new pkgAcqIndexDiffs to download and apply the remaining + * patches. If no patch can be found that applies to an intermediate + * file or if one of the patches cannot be downloaded, falls back to + * downloading the entire package index file using pkgAcqIndex. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ class pkgAcqIndexDiffs : public pkgAcquire::Item { + private: + + /** \brief Queue up the next diff download. + * + * Search for the next available diff that applies to the file + * that currently exists on disk, and enqueue it by calling + * QueueURI(). + * + * \return \b true if an applicable diff was found, \b false + * otherwise. + */ + bool QueueNextDiff(); + + /** \brief Handle tasks that must be performed after the item + * finishes downloading. + * + * Dequeues the item and checks the resulting file's md5sum + * against ExpectedMD5 after the last patch was applied. + * There is no need to check the md5/sha1 after a "normal" + * patch because QueueNextDiff() will check the sha1 later. + * + * \param allDone If \b true, the file was entirely reconstructed, + * and its md5sum is verified. + */ + void Finish(bool allDone=false); + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ bool Debug; + + /** \brief A description of the item that is currently being + * downloaded. + */ pkgAcquire::ItemDesc Desc; + + /** \brief The URI of the package index file that is being + * reconstructed. + */ string RealURI; + + /** \brief The MD5Sum of the package index file that is being + * reconstructed. + */ string ExpectedMD5; - // this is the SHA-1 sum we expect after the patching + /** A description of the file being downloaded. */ string Description; + + /** The patches that remain to be downloaded, including the patch + * being downloaded right now. This list should be ordered so + * that each diff appears before any diff that depends on it. + * + * \todo These are indexed by sha1sum; why not use some sort of + * dictionary instead of relying on ordering and stripping them + * off the front? + */ vector<DiffInfo> available_patches; - enum {StateFetchIndex,StateFetchDiff,StateUnzipDiff,StateApplyDiff} State; + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is in an unknown state. */ + StateFetchUnkown, + + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being uncompressed. */ + StateUnzipDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff + } State; public: - // Specialized action members + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(string Message,unsigned long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string DescURI() {return RealURI + "Index";}; - // various helpers - bool QueueNextDiff(); - bool ApplyDiff(string PatchFile); - void Finish(bool allDone=false); - + /** \brief Create an index diff item. + * + * After filling in its basic fields, this invokes Finish(true) if + * #diffs is empty, or QueueNextDiff() otherwise. + * + * \param Owner The pkgAcquire object that owns this item. + * + * \param URI The URI of the package index file being + * reconstructed. + * + * \param URIDesc A long description of this item. + * + * \param ShortDesc A brief description of this item. + * + * \param ExpectedMD5 The expected md5sum of the completely + * reconstructed package index file; the index file will be tested + * against this value when it is entirely reconstructed. + * + * \param diffs The remaining diffs from the index of diffs. They + * should be ordered so that each diff appears before any diff + * that depends on it. + */ pkgAcqIndexDiffs(pkgAcquire *Owner,string URI,string URIDesc, - string ShortDesct, string ExpectedMD5, + string ShortDesc, string ExpectedMD5, vector<DiffInfo> diffs=vector<DiffInfo>()); }; -// Item class for index files +/** \brief An acquire item that is responsible for fetching an index + * file (e.g., Packages or Sources). + * + * \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans + * + * \todo Why does pkgAcqIndex have protected members? + */ class pkgAcqIndex : public pkgAcquire::Item { protected: - + + /** \brief If \b true, the index file has been decompressed. */ bool Decompression; + + /** \brief If \b true, the partially downloaded file will be + * removed when the download completes. + */ bool Erase; + + /** \brief The download request that is currently being + * processed. + */ pkgAcquire::ItemDesc Desc; + + /** \brief The object that is actually being fetched (minus any + * compression-related extensions). + */ string RealURI; + + /** \brief The expected md5sum of the decompressed index file. */ string ExpectedMD5; + + /** \brief The compression-related file extension that is being + * added to the downloaded file (e.g., ".gz" or ".bz2"). + */ string CompressionExtension; public: @@ -167,26 +522,120 @@ class pkgAcqIndex : public pkgAcquire::Item virtual string Custom600Headers(); virtual string DescURI() {return RealURI + CompressionExtension;}; + /** \brief Create a pkgAcqIndex. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param URI The URI of the index file that is to be downloaded. + * + * \param URIDesc A "URI-style" description of this index file. + * + * \param ShortDesc A brief description of this index file. + * + * \param ExpectedMD5 The expected md5sum of this index file. + * + * \param compressExt The compression-related extension with which + * this index file should be downloaded, or "" to autodetect + * (".bz2" is used if bzip2 is installed, ".gz" otherwise). + */ pkgAcqIndex(pkgAcquire *Owner,string URI,string URIDesc, - string ShortDesct, string ExpectedMD5, string compressExt=""); + string ShortDesc, string ExpectedMD5, string compressExt=""); +}; + +/** \brief An acquire item that is responsible for fetching a + * translated index file. + * + * The only difference from pkgAcqIndex is that transient failures + * are suppressed: no error occurs if the translated index file is + * missing. + */ +class pkgAcqIndexTrans : public pkgAcqIndex +{ + public: + + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + + /** \brief Create a pkgAcqIndexTrans. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param URI The URI of the index file that is to be downloaded. + * + * \param URIDesc A "URI-style" description of this index file. + * + * \param ShortDesc A brief description of this index file. + * + * \param ExpectedMD5 The expected md5sum of this index file. + * + * \param compressExt The compression-related extension with which + * this index file should be downloaded, or "" to autodetect + * (".bz2" is used if bzip2 is installed, ".gz" otherwise). + */ + pkgAcqIndexTrans(pkgAcquire *Owner,string URI,string URIDesc, + string ShortDesc); }; +/** \brief Information about an index file. */ struct IndexTarget { + /** \brief A URI from which the index file can be downloaded. */ string URI; + + /** \brief A description of the index file. */ string Description; + + /** \brief A shorter description of the index file. */ string ShortDesc; + + /** \brief The key by which this index file should be + * looked up within the meta signature file. + */ string MetaKey; }; -// Item class for index signatures +/** \brief An acquire item that downloads the detached signature + * of a meta-index (Release) file, then queues up the release + * file itself. + * + * \todo Why protected members? + * + * \sa pkgAcqMetaIndex + */ class pkgAcqMetaSig : public pkgAcquire::Item { protected: - + /** \brief The fetch request that is currently being processed. */ pkgAcquire::ItemDesc Desc; - string RealURI,MetaIndexURI,MetaIndexURIDesc,MetaIndexShortDesc; + + /** \brief The URI of the signature file. Unlike Desc.URI, this is + * never modified; it is used to determine the file that is being + * downloaded. + */ + string RealURI; + + /** \brief The URI of the meta-index file to be fetched after the signature. */ + string MetaIndexURI; + + /** \brief A "URI-style" description of the meta-index file to be + * fetched after the signature. + */ + string MetaIndexURIDesc; + + /** \brief A brief description of the meta-index file to be fetched + * after the signature. + */ + string MetaIndexShortDesc; + + /** \brief A package-system-specific parser for the meta-index file. */ indexRecords* MetaIndexParser; + + /** \brief The index files which should be looked up in the meta-index + * and then downloaded. + * + * \todo Why a list of pointers instead of a list of structs? + */ const vector<struct IndexTarget*>* IndexTargets; public: @@ -198,29 +647,90 @@ class pkgAcqMetaSig : public pkgAcquire::Item virtual string Custom600Headers(); virtual string DescURI() {return RealURI; }; + /** \brief Create a new pkgAcqMetaSig. */ pkgAcqMetaSig(pkgAcquire *Owner,string URI,string URIDesc, string ShortDesc, string MetaIndexURI, string MetaIndexURIDesc, string MetaIndexShortDesc, const vector<struct IndexTarget*>* IndexTargets, indexRecords* MetaIndexParser); }; -// Item class for index signatures +/** \brief An item that is responsible for downloading the meta-index + * file (i.e., Release) itself and verifying its signature. + * + * Once the download and verification are complete, the downloads of + * the individual index files are queued up using pkgAcqDiffIndex. + * If the meta-index file had a valid signature, the expected md5sums + * of the index files will be the md5sums listed in the meta-index; + * otherwise, the expected md5sums will be "" (causing the + * authentication of the index files to be bypassed). + */ class pkgAcqMetaIndex : public pkgAcquire::Item { protected: - + /** \brief The fetch command that is currently being processed. */ pkgAcquire::ItemDesc Desc; - string RealURI; // FIXME: is this redundant w/ Desc.URI? + + /** \brief The URI that is actually being downloaded; never + * modified by pkgAcqMetaIndex. + */ + string RealURI; + + /** \brief The file in which the signature for this index was stored. + * + * If empty, the signature and the md5sums of the individual + * indices will not be checked. + */ string SigFile; + + /** \brief The index files to download. */ const vector<struct IndexTarget*>* IndexTargets; + + /** \brief The parser for the meta-index file. */ indexRecords* MetaIndexParser; + + /** \brief If \b true, the index's signature is currently being verified. + */ bool AuthPass; // required to deal gracefully with problems caused by incorrect ims hits bool IMSHit; + /** \brief Check that the release file is a release file for the + * correct distribution. + * + * \return \b true if no fatal errors were encountered. + */ bool VerifyVendor(string Message); + + /** \brief Called when a file is finished being retrieved. + * + * If the file was not downloaded to DestFile, a copy process is + * set up to copy it to DestFile; otherwise, Complete is set to \b + * true and the file is moved to its final location. + * + * \param Message The message block received from the fetch + * subprocess. + */ void RetrievalDone(string Message); + + /** \brief Called when authentication succeeded. + * + * Sanity-checks the authenticated file, queues up the individual + * index files for download, and saves the signature in the lists + * directory next to the authenticated list file. + * + * \param Message The message block received from the fetch + * subprocess. + */ void AuthDone(string Message); + + /** \brief Starts downloading the individual index files. + * + * \param verify If \b true, only indices whose expected md5sum + * can be determined from the meta-index will be downloaded, and + * the md5sums of indices will be checked (reporting + * #StatAuthError if there is a mismatch). If verify is \b false, + * no md5sum checking will be performed. + */ void QueueIndexes(bool verify); public: @@ -232,6 +742,7 @@ class pkgAcqMetaIndex : public pkgAcquire::Item virtual string Custom600Headers(); virtual string DescURI() {return RealURI; }; + /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, string URI,string URIDesc, string ShortDesc, string SigFile, @@ -239,28 +750,58 @@ class pkgAcqMetaIndex : public pkgAcquire::Item indexRecords* MetaIndexParser); }; -// Item class for archive files +/** \brief An item that is responsible for fetching a package file. + * + * If the package file already exists in the cache, nothing will be + * done. + */ class pkgAcqArchive : public pkgAcquire::Item { protected: - - // State information for the retry mechanism + /** \brief The package version being fetched. */ pkgCache::VerIterator Version; + + /** \brief The fetch command that is currently being processed. */ pkgAcquire::ItemDesc Desc; + + /** \brief The list of sources from which to pick archives to + * download this package from. + */ pkgSourceList *Sources; + + /** \brief A package records object, used to look up the file + * corresponding to each version of the package. + */ pkgRecords *Recs; + + /** \brief The md5sum of this package. */ string MD5; + + /** \brief A location in which the actual filename of the package + * should be stored. + */ string &StoreFilename; + + /** \brief The next file for this version to try to download. */ pkgCache::VerFileIterator Vf; + + /** \brief How many (more) times to try to find a new source from + * which to download this package version if it fails. + * + * Set from Acquire::Retries. + */ unsigned int Retries; + + /** \brief \b true if this version file is being downloaded from a + * trusted source. + */ bool Trusted; - // Queue the next available file for download. + /** \brief Queue up the next available file for this version. */ bool QueueNext(); public: - // Specialized action members virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); virtual void Done(string Message,unsigned long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); @@ -268,18 +809,49 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string DescURI() {return Desc.URI;}; virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); + virtual bool IsTrusted(); + /** \brief Create a new pkgAcqArchive. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param Sources The sources from which to download version + * files. + * + * \param Recs A package records object, used to look up the file + * corresponding to each version of the package. + * + * \param Version The package version to download. + * + * \param StoreFilename A location in which the actual filename of + * the package should be stored. It will be set to a guessed + * basename in the constructor, and filled in with a fully + * qualified filename once the download finishes. + */ pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, string &StoreFilename); }; -// Fetch a generic file to the current directory +/** \brief Retrieve an arbitrary file to the current directory. + * + * The file is retrieved even if it is accessed via a URL type that + * normally is a NOP, such as "file". If the download fails, the + * partial file is renamed to get a ".FAILED" extension. + */ class pkgAcqFile : public pkgAcquire::Item { + /** \brief The currently active download process. */ pkgAcquire::ItemDesc Desc; + + /** \brief The md5sum of the file to download, if it is known. */ string Md5Hash; + + /** \brief How many times to retry the download, set from + * Acquire::Retries. + */ unsigned int Retries; public: @@ -291,13 +863,40 @@ class pkgAcqFile : public pkgAcquire::Item virtual string MD5Sum() {return Md5Hash;}; virtual string DescURI() {return Desc.URI;}; - // If DestFilename is empty, download to DestDir/<basename> if - // DestDir is non-empty, $CWD/<basename> otherwise. If - // DestFilename is NOT empty, DestDir is ignored and DestFilename - // is the absolute name to which the file should be downloaded. + /** \brief Create a new pkgAcqFile object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * + * \param URI The URI to download. + * + * \param MD5 The md5sum of the file to download, if it is known; + * otherwise "". + * + * \param Size The size of the file to download, if it is known; + * otherwise 0. + * + * \param Desc A description of the file being downloaded. + * + * \param ShortDesc A brief description of the file being + * downloaded. + * + * \param DestDir The directory the file should be downloaded into. + * + * \param DestFilename The filename+path the file is downloaded to. + * + * + * If DestFilename is empty, download to DestDir/<basename> if + * DestDir is non-empty, $CWD/<basename> otherwise. If + * DestFilename is NOT empty, DestDir is ignored and DestFilename + * is the absolute name to which the file should be downloaded. + */ + pkgAcqFile(pkgAcquire *Owner, string URI, string MD5, unsigned long Size, string Desc, string ShortDesc, const string &DestDir="", const string &DestFilename=""); }; +/** @} */ + #endif diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index f46209d12..4f08a43ae 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -10,6 +10,13 @@ ##################################################################### */ /*}}}*/ + +/** \addtogroup acquire + * @{ + * + * \file acquire-method.h + */ + #ifndef PKGLIB_ACQUIRE_METHOD_H #define PKGLIB_ACQUIRE_METHOD_H @@ -86,4 +93,6 @@ class pkgAcqMethod virtual ~pkgAcqMethod() {}; }; +/** @} */ + #endif diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h index 6e1952202..1f6bcc05f 100644 --- a/apt-pkg/acquire-worker.h +++ b/apt-pkg/acquire-worker.h @@ -9,6 +9,13 @@ ##################################################################### */ /*}}}*/ + +/** \addtogroup acquire + * @{ + * + * \file acquire-worker.h + */ + #ifndef PKGLIB_ACQUIRE_WORKER_H #define PKGLIB_ACQUIRE_WORKER_H @@ -18,7 +25,25 @@ #pragma interface "apt-pkg/acquire-worker.h" #endif -// Interfacing to the method process +/** \brief A fetch subprocess. + * + * A worker process is responsible for one stage of the fetch. This + * class encapsulates the communications protocol between the master + * process and the worker, from the master end. + * + * Each worker is intrinsically placed on two linked lists. The + * Queue list (maintained in the #NextQueue variable) is maintained + * by the pkgAcquire::Queue class; it represents the set of workers + * assigned to a particular queue. The Acquire list (maintained in + * the #NextAcquire variable) is maintained by the pkgAcquire class; + * it represents the set of active workers for a particular + * pkgAcquire object. + * + * \todo Like everything else in the Acquire system, this has way too + * many protected items. + * + * \sa pkgAcqMethod, pkgAcquire::Item, pkgAcquire + */ class pkgAcquire::Worker { friend class pkgAcquire; @@ -26,64 +51,274 @@ class pkgAcquire::Worker protected: friend class Queue; - /* Linked list starting at a Queue and a linked list starting - at Acquire */ + /** \brief The next link on the Queue list. + * + * \todo This is always NULL; is it just for future use? + */ Worker *NextQueue; + + /** \brief The next link on the Acquire list. */ Worker *NextAcquire; - // The access association + /** \brief The Queue with which this worker is associated. */ Queue *OwnerQ; + + /** \brief The download progress indicator to which progress + * messages should be sent. + */ pkgAcquireStatus *Log; + + /** \brief The configuration of this method. On startup, the + * target of this pointer is filled in with basic data about the + * method, as reported by the worker. + */ MethodConfig *Config; + + /** \brief The access method to be used by this worker. + * + * \todo Doesn't this duplicate Config->Access? + */ string Access; - // This is the subprocess IPC setup + /** \brief The PID of the subprocess. */ pid_t Process; + + /** \brief A file descriptor connected to the standard output of + * the subprocess. + * + * Used to read messages and data from the subprocess. + */ int InFd; + + /** \brief A file descriptor connected to the standard input of the + * subprocess. + * + * Used to send commands and configuration data to the subprocess. + */ int OutFd; + + /** \brief Set to \b true if the worker is in a state in which it + * might generate data or command responses. + * + * \todo Is this right? It's a guess. + */ bool InReady; + + /** \brief Set to \b true if the worker is in a state in which it + * is legal to send commands to it. + * + * \todo Is this right? + */ bool OutReady; - // Various internal things + /** If \b true, debugging output will be sent to std::clog. */ bool Debug; + + /** \brief The raw text values of messages received from the + * worker, in sequence. + */ vector<string> MessageQueue; + + /** \brief Buffers pending writes to the subprocess. + * + * \todo Wouldn't a std::dequeue be more appropriate? + */ string OutQueue; - // Private constructor helper + /** \brief Common code for the constructor. + * + * Initializes NextQueue and NextAcquire to NULL; Process, InFd, + * and OutFd to -1, OutReady and InReady to \b false, and Debug + * from _config. + */ void Construct(); - // Message handling things + /** \brief Retrieve any available messages from the subprocess. + * + * The messages are retrieved as in ::ReadMessages(), and + * MessageFailure() is invoked if an error occurs; in particular, + * if the pipe to the subprocess dies unexpectedly while a message + * is being read. + * + * \return \b true if the messages were successfully read, \b + * false otherwise. + */ bool ReadMessages(); + + /** \brief Parse and dispatch pending messages. + * + * This dispatches the message in a manner appropriate for its + * type. + * + * \todo Several message types lack separate handlers. + * + * \sa Capabilities(), SendConfiguration(), MediaChange() + */ bool RunMessages(); + + /** \brief Read and dispatch any pending messages from the + * subprocess. + * + * \return \b false if the subprocess died unexpectedly while a + * message was being transmitted. + */ bool InFdReady(); + + /** \brief Send any pending commands to the subprocess. + * + * This method will fail if there is no pending output. + * + * \return \b true if all commands were succeeded, \b false if an + * error occurred (in which case MethodFailure() will be invoked). + */ bool OutFdReady(); - // The message handlers + /** \brief Handle a 100 Capabilities response from the subprocess. + * + * \param Message the raw text of the message from the subprocess. + * + * The message will be parsed and its contents used to fill + * #Config. If #Config is NULL, this routine is a NOP. + * + * \return \b true. + */ bool Capabilities(string Message); + + /** \brief Send a 601 Configuration message (containing the APT + * configuration) to the subprocess. + * + * The APT configuration will be send to the subprocess in a + * message of the following form: + * + * <pre> + * 601 Configuration + * Config-Item: Fully-Qualified-Item=Val + * Config-Item: Fully-Qualified-Item=Val + * ... + * </pre> + * + * \return \b true if the command was successfully sent, \b false + * otherwise. + */ bool SendConfiguration(); + + /** \brief Handle a 403 Media Change message. + * + * \param Message the raw text of the message; the Media field + * indicates what type of media should be changed, and the Drive + * field indicates where the media is located. + * + * Invokes pkgAcquireStatus::MediaChange(Media, Drive) to ask the + * user to swap disks; informs the subprocess of the result (via + * 603 Media Changed, with the Failed field set to \b true if the + * user cancelled the media change). + */ bool MediaChange(string Message); + /** \brief Invoked when the worked process dies unexpectedly. + * + * Waits for the subprocess to terminate and generates an error if + * it terminated abnormally, then closes and blanks out all file + * descriptors. Discards all pending messages from the + * subprocess. + * + * \return \b false. + */ bool MethodFailure(); + + /** \brief Invoked when a fetch job is completed, either + * successfully or unsuccessfully. + * + * Resets the status information for the worker process. + */ void ItemDone(); public: - // The curent method state + /** \brief The queue entry that is currently being downloaded. */ pkgAcquire::Queue::QItem *CurrentItem; + + /** \brief The most recent status string received from the + * subprocess. + */ string Status; + + /** \brief How many bytes of the file have been downloaded. Zero + * if the current progress of the file cannot be determined. + */ unsigned long CurrentSize; + + /** \brief The total number of bytes to be downloaded. Zero if the + * total size of the final is unknown. + */ unsigned long TotalSize; + + /** \brief How much of the file was already downloaded prior to + * starting this worker. + */ unsigned long ResumePoint; - // Load the method and do the startup + /** \brief Tell the subprocess to download the given item. + * + * \param Item the item to queue up. + * \return \b true if the item was successfully enqueued. + * + * Queues up a 600 URI Acquire message for the given item to be + * sent at the next possible moment. Does \e not flush the output + * queue. + */ bool QueueItem(pkgAcquire::Queue::QItem *Item); + + /** \brief Start up the worker and fill in #Config. + * + * Reads the first message from the worker, which is assumed to be + * a 100 Capabilities message. + * + * \return \b true if all operations completed successfully. + */ bool Start(); + + /** \brief Update the worker statistics (CurrentSize, TotalSize, + * etc). + */ void Pulse(); + + /** \return The fetch method configuration. */ inline const MethodConfig *GetConf() const {return Config;}; - + + /** \brief Create a new Worker to download files. + * + * \param OwnerQ The queue into which this worker should be + * placed. + * + * \param Config A location in which to store information about + * the fetch method. + * + * \param Log The download progress indicator that should be used + * to report the progress of this worker. + */ Worker(Queue *OwnerQ,MethodConfig *Config,pkgAcquireStatus *Log); + + /** \brief Create a new Worker that should just retrieve + * information about the fetch method. + * + * Nothing in particular forces you to refrain from actually + * downloading stuff, but the various status callbacks won't be + * invoked. + * + * \param Config A location in which to store information about + * the fetch method. + */ Worker(MethodConfig *Config); + + /** \brief Clean up this worker. + * + * Closes the file descriptors; if MethodConfig::NeedsCleanup is + * \b false, also rudely interrupts the worker with a SIGINT. + */ ~Worker(); }; +/** @} */ + #endif diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 27bb3d363..64dafdc9d 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -29,6 +29,40 @@ ##################################################################### */ /*}}}*/ + +/** \defgroup acquire Acquire system + * + * \brief The Acquire system is responsible for retrieving files from + * local or remote URIs and postprocessing them (for instance, + * verifying their authenticity). The core class in this system is + * pkgAcquire, which is responsible for managing the download queues + * during the download. There is at least one download queue for + * each supported protocol; protocols such as http may provide one + * queue per host. + * + * Each file to download is represented by a subclass of + * pkgAcquire::Item. The files add themselves to the download + * queue(s) by providing their URI information to + * pkgAcquire::Item::QueueURI, which calls pkgAcquire::Enqueue. + * + * Once the system is set up, the Run method will spawn subprocesses + * to handle the enqueued URIs; the scheduler will then take items + * from the queues and feed them into the handlers until the queues + * are empty. + * + * \todo Acquire supports inserting an object into several queues at + * once, but it is not clear what its behavior in this case is, and + * no subclass of pkgAcquire::Item seems to actually use this + * capability. + */ + +/** \addtogroup acquire + * + * @{ + * + * \file acquire.h + */ + #ifndef PKGLIB_ACQUIRE_H #define PKGLIB_ACQUIRE_H @@ -46,6 +80,15 @@ using std::string; #include <unistd.h> class pkgAcquireStatus; + +/** \brief The core download scheduler. + * + * This class represents an ongoing download. It manages the lists + * of active and pending downloads and handles setting up and tearing + * down download-related structures. + * + * \todo Why all the protected data items and methods? + */ class pkgAcquire { public: @@ -60,97 +103,299 @@ class pkgAcquire typedef vector<Item *>::iterator ItemIterator; typedef vector<Item *>::const_iterator ItemCIterator; - + protected: - // List of items to fetch + /** \brief A list of items to download. + * + * This is built monotonically as items are created and only + * emptied when the download shuts down. + */ vector<Item *> Items; - // List of active queues and fetched method configuration parameters + /** \brief The head of the list of active queues. + * + * \todo why a hand-managed list of queues instead of std::list or + * std::set? + */ Queue *Queues; + + /** \brief The head of the list of active workers. + * + * \todo why a hand-managed list of workers instead of std::list + * or std::set? + */ Worker *Workers; + + /** \brief The head of the list of acquire method configurations. + * + * Each protocol (http, ftp, gzip, etc) via which files can be + * fetched can have a representation in this list. The + * configuration data is filled in by parsing the 100 Capabilities + * string output by a method on startup (see + * pkgAcqMethod::pkgAcqMethod and pkgAcquire::GetConfig). + * + * \todo why a hand-managed config dictionary instead of std::map? + */ MethodConfig *Configs; + + /** \brief The progress indicator for this download. */ pkgAcquireStatus *Log; + + /** \brief The total size of the files which are to be fetched. + * + * This is not necessarily the total number of bytes to download + * when, e.g., download resumption and list updates via patches + * are taken into account. + */ unsigned long ToFetch; - // Configurable parameters for the schedular - enum {QueueHost,QueueAccess} QueueMode; + // Configurable parameters for the scheduler + + /** \brief Represents the queuing strategy for remote URIs. */ + enum QueueStrategy { + /** \brief Generate one queue for each protocol/host combination; downloads from + * multiple hosts can proceed in parallel. + */ + QueueHost, + /** \brief Generate a single queue for each protocol; serialize + * downloads from multiple hosts. + */ + QueueAccess} QueueMode; + + /** \brief If \b true, debugging information will be dumped to std::clog. */ bool Debug; + /** \brief If \b true, a download is currently in progress. */ bool Running; - + + /** \brief Add the given item to the list of items. */ void Add(Item *Item); + + /** \brief Remove the given item from the list of items. */ void Remove(Item *Item); + + /** \brief Add the given worker to the list of workers. */ void Add(Worker *Work); + + /** \brief Remove the given worker from the list of workers. */ void Remove(Worker *Work); + /** \brief Insert the given fetch request into the appropriate queue. + * + * \param Item The URI to download and the item to download it + * for. Copied by value into the queue; no reference to Item is + * retained. + */ void Enqueue(ItemDesc &Item); + + /** \brief Remove all fetch requests for this item from all queues. */ void Dequeue(Item *Item); + + /** \brief Determine the fetch method and queue of a URI. + * + * \param URI The URI to fetch. + * + * \param[out] Config A location in which to place the method via + * which the URI is to be fetched. + * + * \return the string-name of the queue in which a fetch request + * for the given URI should be placed. + */ string QueueName(string URI,MethodConfig const *&Config); - // FDSET managers for derived classes + /** \brief Build up the set of file descriptors upon which select() should + * block. + * + * The default implementation inserts the file descriptors + * corresponding to active downloads. + * + * \param[out] Fd The largest file descriptor in the generated sets. + * + * \param[out] RSet The set of file descriptors that should be + * watched for input. + * + * \param[out] WSet The set of file descriptors that should be + * watched for output. + */ virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet); + + /** Handle input from and output to file descriptors which select() + * has determined are ready. The default implementation + * dispatches to all active downloads. + * + * \param RSet The set of file descriptors that are ready for + * input. + * + * \param WSet The set of file descriptors that are ready for + * output. + */ virtual void RunFds(fd_set *RSet,fd_set *WSet); - // A queue calls this when it dequeues an item + /** \brief Check for idle queues with ready-to-fetch items. + * + * Called by pkgAcquire::Queue::Done each time an item is dequeued + * but remains on some queues; i.e., another queue should start + * fetching it. + */ void Bump(); public: + /** \brief Retrieve information about a fetch method by name. + * + * \param Access The name of the method to look up. + * + * \return the method whose name is Access, or \b NULL if no such method exists. + */ MethodConfig *GetConfig(string Access); - enum RunResult {Continue,Failed,Cancelled}; + /** \brief Provides information on how a download terminated. */ + enum RunResult { + /** \brief All files were fetched successfully. */ + Continue, + + /** \brief Some files failed to download. */ + Failed, + + /** \brief The download was cancelled by the user (i.e., #Log's + * pkgAcquireStatus::Pulse() method returned \b false). + */ + Cancelled}; - RunResult Run(int PulseIntervall=500000); + /** \brief Download all the items that have been Add()ed to this + * download process. + * + * This method will block until the download completes, invoking + * methods on #Log to report on the progress of the download. + * + * \param PulseInterval The method pkgAcquireStatus::Pulse will be + * invoked on #Log at intervals of PulseInterval milliseconds. + * + * \return the result of the download. + */ + RunResult Run(int PulseInterval=500000); + + /** \brief Remove all items from this download process, terminate + * all download workers, and empty all queues. + */ void Shutdown(); - // Simple iteration mechanism + /** \brief Get the first #Worker object. + * + * \return the first active worker in this download process. + */ inline Worker *WorkersBegin() {return Workers;}; + + /** \brief Advance to the next #Worker object. + * + * \return the worker immediately following I, or \b NULL if none + * exists. + */ Worker *WorkerStep(Worker *I); + + /** \brief Get the head of the list of items. */ inline ItemIterator ItemsBegin() {return Items.begin();}; + + /** \brief Get the end iterator of the list of items. */ inline ItemIterator ItemsEnd() {return Items.end();}; // Iterate over queued Item URIs class UriIterator; + /** \brief Get the head of the list of enqueued item URIs. + * + * This iterator will step over every element of every active + * queue. + */ UriIterator UriBegin(); + /** \brief Get the end iterator of the list of enqueued item URIs. */ UriIterator UriEnd(); - // Cleans out the download dir + /** Deletes each entry in the given directory that is not being + * downloaded by this object. For instance, when downloading new + * list files, calling Clean() will delete the old ones. + * + * \param Dir The directory to be cleaned out. + * + * \return \b true if the directory exists and is readable. + */ bool Clean(string Dir); - // Returns the size of the total download set + /** \return the total size in bytes of all the items included in + * this download. + */ double TotalNeeded(); + + /** \return the size in bytes of all non-local items included in + * this download. + */ double FetchNeeded(); + + /** \return the amount of data to be fetched that is already + * present on the filesystem. + */ double PartialPresent(); + /** \brief Construct a new pkgAcquire. + * + * \param Log The progress indicator associated with this + * download, or \b NULL for none. This object is not owned by the + * download process and will not be deleted when the pkgAcquire + * object is destroyed. Naturally, it should live for at least as + * long as the pkgAcquire object does. + */ pkgAcquire(pkgAcquireStatus *Log = 0); + + /** \brief Destroy this pkgAcquire object. + * + * Destroys all queue, method, and item objects associated with + * this download. + */ virtual ~pkgAcquire(); }; -// Description of an Item+URI +/** \brief Represents a single download source from which an item + * should be downloaded. + * + * An item may have several assocated ItemDescs over its lifetime. + */ struct pkgAcquire::ItemDesc { + /** \brief The URI from which to download this item. */ string URI; + /** brief A description of this item. */ string Description; + /** brief A shorter description of this item. */ string ShortDesc; + /** brief The underlying item which is to be downloaded. */ Item *Owner; }; -// List of possible items queued for download. +/** \brief A single download queue in a pkgAcquire object. + * + * \todo Why so many protected values? + */ class pkgAcquire::Queue { friend class pkgAcquire; friend class pkgAcquire::UriIterator; friend class pkgAcquire::Worker; + + /** \brief The next queue in the pkgAcquire object's list of queues. */ Queue *Next; protected: - // Queued item + /** \brief A single item placed in this queue. */ struct QItem : pkgAcquire::ItemDesc { - QItem *Next; + /** \brief The next item in the queue. */ + QItem *Next; + /** \brief The worker associated with this item, if any. */ pkgAcquire::Worker *Worker; - + + /** \brief Assign the ItemDesc portion of this QItem from + * another ItemDesc + */ void operator =(pkgAcquire::ItemDesc const &I) { URI = I.URI; @@ -160,45 +405,141 @@ class pkgAcquire::Queue }; }; - // Name of the queue + /** \brief The name of this queue. */ string Name; - // Items queued into this queue + /** \brief The head of the list of items contained in this queue. + * + * \todo why a by-hand list instead of an STL structure? + */ QItem *Items; + + /** \brief The head of the list of workers associated with this queue. + * + * \todo This is plural because support exists in Queue for + * multiple workers. However, it does not appear that there is + * any way to actually associate more than one worker with a + * queue. + * + * \todo Why not just use a std::set? + */ pkgAcquire::Worker *Workers; + + /** \brief the download scheduler with which this queue is associated. */ pkgAcquire *Owner; + + /** \brief The number of entries in this queue that are currently + * being downloaded. + */ signed long PipeDepth; + + /** \brief The maximum number of entries that this queue will + * attempt to download at once. + */ unsigned long MaxPipeDepth; public: - // Put an item into this queue + /** \brief Insert the given fetch request into this queue. */ void Enqueue(ItemDesc &Item); + + /** \brief Remove all fetch requests for the given item from this queue. + * + * \return \b true if at least one request was removed from the queue. + */ bool Dequeue(Item *Owner); - // Find a Queued item + /** \brief Locate an item in this queue. + * + * \param URI A URI to match against. + * \param Owner A pkgAcquire::Worker to match against. + * + * \return the first item in the queue whose URI is #URI and that + * is being downloaded by #Owner. + */ QItem *FindItem(string URI,pkgAcquire::Worker *Owner); + + /** Presumably this should start downloading an item? + * + * \todo Unimplemented. Implement it or remove? + */ bool ItemStart(QItem *Itm,unsigned long Size); + + /** \brief Remove the given item from this queue and set its state + * to pkgAcquire::Item::StatDone. + * + * If this is the only queue containing the item, the item is also + * removed from the main queue by calling pkgAcquire::Dequeue. + * + * \param Itm The item to remove. + * + * \return \b true if no errors are encountered. + */ bool ItemDone(QItem *Itm); + /** \brief Start the worker process associated with this queue. + * + * If a worker process is already associated with this queue, + * this is equivalent to calling Cycle(). + * + * \return \b true if the startup was successful. + */ bool Startup(); + + /** \brief Shut down the worker process associated with this queue. + * + * \param Final If \b true, then the process is stopped unconditionally. + * Otherwise, it is only stopped if it does not need cleanup + * as indicated by the pkgAcqMethod::NeedsCleanup member of + * its configuration. + * + * \return \b true. + */ bool Shutdown(bool Final); + + /** \brief Send idle items to the worker process. + * + * Fills up the pipeline by inserting idle items into the worker's queue. + */ bool Cycle(); + + /** \brief Check for items that could be enqueued. + * + * Call this after an item placed in multiple queues has gone from + * the pkgAcquire::Item::StatFetching state to the + * pkgAcquire::Item::StatIdle state, to possibly refill an empty queue. + * This is an alias for Cycle(). + * + * \todo Why both this and Cycle()? Are they expected to be + * different someday? + */ void Bump(); + /** \brief Create a new Queue. + * + * \param Name The name of the new queue. + * \param Owner The download process that owns the new queue. + */ Queue(string Name,pkgAcquire *Owner); + + /** Shut down all the worker processes associated with this queue + * and empty the queue. + */ ~Queue(); }; +/** \brief Iterates over all the URIs being fetched by a pkgAcquire object. */ class pkgAcquire::UriIterator { + /** The next queue to iterate over. */ pkgAcquire::Queue *CurQ; + /** The item that we currently point at. */ pkgAcquire::Queue::QItem *CurItem; public: - // Advance to the next item inline void operator ++() {operator ++();}; + void operator ++(int) { CurItem = CurItem->Next; @@ -209,11 +550,14 @@ class pkgAcquire::UriIterator } }; - // Accessors inline pkgAcquire::ItemDesc const *operator ->() const {return CurItem;}; inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;}; inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;}; + /** \brief Create a new UriIterator. + * + * \param Q The queue over which this UriIterator should iterate. + */ UriIterator(pkgAcquire::Queue *Q) : CurQ(Q), CurItem(0) { while (CurItem == 0 && CurQ != 0) @@ -224,61 +568,200 @@ class pkgAcquire::UriIterator } }; -// Configuration information from each method +/** \brief Information about the properties of a single acquire method. */ struct pkgAcquire::MethodConfig { + /** \brief The next link on the acquire method list. + * + * \todo Why not an STL container? + */ MethodConfig *Next; + /** \brief The name of this acquire method (e.g., http). */ string Access; + /** \brief The implementation version of this acquire method. */ string Version; + + /** \brief If \b true, only one download queue should be created for this + * method. + */ bool SingleInstance; + + /** \brief If \b true, this method supports pipelined downloading. */ bool Pipeline; + + /** \brief If \b true, the worker process should send the entire + * APT configuration tree to the fetch subprocess when it starts + * up. + */ bool SendConfig; + + /** \brief If \b true, this fetch method does not require network access; + * all files are to be acquired from the local disk. + */ bool LocalOnly; + + /** \brief If \b true, the subprocess has to carry out some cleanup + * actions before shutting down. + * + * For instance, the cdrom method needs to unmount the CD after it + * finishes. + */ bool NeedsCleanup; + + /** \brief If \b true, this fetch method acquires files from removable media. */ bool Removable; + /** \brief Set up the default method parameters. + * + * All fields are initialized to NULL, "", or \b false as + * appropriate. + */ MethodConfig(); }; +/** \brief A monitor object for downloads controlled by the pkgAcquire class. + * + * \todo Why protected members? + * + * \todo Should the double members be uint64_t? + */ class pkgAcquireStatus { protected: + /** \brief The last time at which this monitor object was updated. */ struct timeval Time; + + /** \brief The time at which the download started. */ struct timeval StartTime; + + /** \brief The number of bytes fetched as of the previous call to + * pkgAcquireStatus::Pulse, including local items. + */ double LastBytes; + + /** \brief The current rate of download as of the most recent call + * to pkgAcquireStatus::Pulse, in bytes per second. + */ double CurrentCPS; + + /** \brief The number of bytes fetched as of the most recent call + * to pkgAcquireStatus::Pulse, including local items. + */ double CurrentBytes; + + /** \brief The total number of bytes that need to be fetched. + * + * \warning This member is inaccurate, as new items might be + * enqueued while the download is in progress! + */ double TotalBytes; + + /** \brief The total number of bytes accounted for by items that + * were successfully fetched. + */ double FetchedBytes; + + /** \brief The amount of time that has elapsed since the download + * started. + */ unsigned long ElapsedTime; + + /** \brief The total number of items that need to be fetched. + * + * \warning This member is inaccurate, as new items might be + * enqueued while the download is in progress! + */ unsigned long TotalItems; + + /** \brief The number of items that have been successfully downloaded. */ unsigned long CurrentItems; public: + /** \brief If \b true, the download scheduler should call Pulse() + * at the next available opportunity. + */ bool Update; + + /** \brief If \b true, extra Pulse() invocations will be performed. + * + * With this option set, Pulse() will be called every time that a + * download item starts downloading, finishes downloading, or + * terminates with an error. + */ bool MorePulses; - // Called by items when they have finished a real download + /** \brief Invoked when a local or remote file has been completely fetched. + * + * \param Size The size of the file fetched. + * + * \param ResumePoint How much of the file was already fetched. + */ virtual void Fetched(unsigned long Size,unsigned long ResumePoint); - // Called to change media + /** \brief Invoked when the user should be prompted to change the + * inserted removable media. + * + * This method should not return until the user has confirmed to + * the user interface that the media change is complete. + * + * \param Media The name of the media type that should be changed. + * + * \param Drive The identifying name of the drive whose media + * should be changed. + * + * \return \b true if the user confirms the media change, \b + * false if it is cancelled. + * + * \todo This is a horrible blocking monster; it should be CPSed + * with prejudice. + */ virtual bool MediaChange(string Media,string Drive) = 0; - // Each of these is called by the workers when an event occures + /** \brief Invoked when an item is confirmed to be up-to-date. + + * For instance, when an HTTP download is informed that the file on + * the server was not modified. + */ virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when some of an item's data is fetched. */ virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when an item is successfully and completely fetched. */ virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when the process of fetching an item encounters + * a fatal error. + */ virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {}; - virtual bool Pulse(pkgAcquire *Owner); // returns false on user cancel + + /** \brief Periodically invoked while the Acquire process is underway. + * + * Subclasses should first call pkgAcquireStatus::Pulse(), then + * update their status output. The download process is blocked + * while Pulse() is being called. + * + * \return \b false if the user asked to cancel the whole Acquire process. + * + * \see pkgAcquire::Run + */ + virtual bool Pulse(pkgAcquire *Owner); + + /** \brief Invoked when the Acquire process starts running. */ virtual void Start(); + + /** \brief Invoked when the Acquire process stops running. */ virtual void Stop(); + /** \brief Initialize all counters to 0 and the time to the current time. */ pkgAcquireStatus(); virtual ~pkgAcquireStatus() {}; }; +/** @} */ + #endif diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 2b326bd65..d5a9c7b0d 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -99,7 +99,7 @@ class pkgCache::VerIterator { Version *Ver; pkgCache *Owner; - + void _dummy(); public: @@ -128,6 +128,8 @@ class pkgCache::VerIterator inline const char *Section() const {return Ver->Section == 0?0:Owner->StrP + Ver->Section;}; inline const char *Arch() const {return Ver->Arch == 0?0:Owner->StrP + Ver->Arch;}; inline PkgIterator ParentPkg() const {return PkgIterator(*Owner,Owner->PkgP + Ver->ParentPkg);}; + inline DescIterator DescriptionList() const; + DescIterator TranslatedDescription() const; inline DepIterator DependsList() const; inline PrvIterator ProvidesList() const; inline VerFileIterator FileList() const; @@ -148,6 +150,50 @@ class pkgCache::VerIterator }; }; +// Description Iterator +class pkgCache::DescIterator +{ + Description *Desc; + pkgCache *Owner; + + void _dummy(); + + public: + + // Iteration + void operator ++(int) {if (Desc != Owner->DescP) Desc = Owner->DescP + Desc->NextDesc;}; + inline void operator ++() {operator ++(0);}; + inline bool end() const {return Desc == Owner->DescP?true:false;}; + inline void operator =(const DescIterator &B) {Desc = B.Desc; Owner = B.Owner;}; + + // Comparison + inline bool operator ==(const DescIterator &B) const {return Desc == B.Desc;}; + inline bool operator !=(const DescIterator &B) const {return Desc != B.Desc;}; + int CompareDesc(const DescIterator &B) const; + + // Accessors + inline Description *operator ->() {return Desc;}; + inline Description const *operator ->() const {return Desc;}; + inline Description &operator *() {return *Desc;}; + inline Description const &operator *() const {return *Desc;}; + inline operator Description *() {return Desc == Owner->DescP?0:Desc;}; + inline operator Description const *() const {return Desc == Owner->DescP?0:Desc;}; + inline pkgCache *Cache() {return Owner;}; + + inline const char *LanguageCode() const {return Owner->StrP + Desc->language_code;}; + inline const char *md5() const {return Owner->StrP + Desc->md5sum;}; + inline DescFileIterator FileList() const; + inline unsigned long Index() const {return Desc - Owner->DescP;}; + + inline DescIterator() : Desc(0), Owner(0) {}; + inline DescIterator(pkgCache &Owner,Description *Trg = 0) : Desc(Trg), + Owner(&Owner) + { + if (Desc == 0) + Desc = Owner.DescP; + }; +}; + // Dependency iterator class pkgCache::DepIterator { @@ -338,6 +384,38 @@ class pkgCache::VerFileIterator inline VerFileIterator(pkgCache &Owner,VerFile *Trg) : Owner(&Owner), FileP(Trg) {}; }; +// Description File +class pkgCache::DescFileIterator +{ + pkgCache *Owner; + DescFile *FileP; + + public: + + // Iteration + void operator ++(int) {if (FileP != Owner->DescFileP) FileP = Owner->DescFileP + FileP->NextFile;}; + inline void operator ++() {operator ++(0);}; + inline bool end() const {return FileP == Owner->DescFileP?true:false;}; + + // Comparison + inline bool operator ==(const DescFileIterator &B) const {return FileP == B.FileP;}; + inline bool operator !=(const DescFileIterator &B) const {return FileP != B.FileP;}; + + // Accessors + inline DescFile *operator ->() {return FileP;}; + inline DescFile const *operator ->() const {return FileP;}; + inline DescFile const &operator *() const {return *FileP;}; + inline operator DescFile *() {return FileP == Owner->DescFileP?0:FileP;}; + inline operator DescFile const *() const {return FileP == Owner->DescFileP?0:FileP;}; + inline pkgCache *Cache() {return Owner;}; + + inline PkgFileIterator File() const {return PkgFileIterator(*Owner,FileP->File + Owner->PkgFileP);}; + inline unsigned long Index() const {return FileP - Owner->DescFileP;}; + + inline DescFileIterator() : Owner(0), FileP(0) {}; + inline DescFileIterator(pkgCache &Owner,DescFile *Trg) : Owner(&Owner), FileP(Trg) {}; +}; + // Inlined Begin functions cant be in the class because of order problems inline pkgCache::VerIterator pkgCache::PkgIterator::VersionList() const {return VerIterator(*Owner,Owner->VerP + Pkg->VersionList);}; @@ -347,11 +425,15 @@ inline pkgCache::DepIterator pkgCache::PkgIterator::RevDependsList() const {return DepIterator(*Owner,Owner->DepP + Pkg->RevDepends,Pkg);}; inline pkgCache::PrvIterator pkgCache::PkgIterator::ProvidesList() const {return PrvIterator(*Owner,Owner->ProvideP + Pkg->ProvidesList,Pkg);}; +inline pkgCache::DescIterator pkgCache::VerIterator::DescriptionList() const + {return DescIterator(*Owner,Owner->DescP + Ver->DescriptionList);}; inline pkgCache::PrvIterator pkgCache::VerIterator::ProvidesList() const {return PrvIterator(*Owner,Owner->ProvideP + Ver->ProvidesList,Ver);}; inline pkgCache::DepIterator pkgCache::VerIterator::DependsList() const {return DepIterator(*Owner,Owner->DepP + Ver->DependsList,Ver);}; inline pkgCache::VerFileIterator pkgCache::VerIterator::FileList() const {return VerFileIterator(*Owner,Owner->VerFileP + Ver->FileList);}; +inline pkgCache::DescFileIterator pkgCache::DescIterator::FileList() const + {return DescFileIterator(*Owner,Owner->DescFileP + Desc->FileList);}; #endif diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index ce1beb39b..b42c82dd0 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -30,12 +30,16 @@ using namespace std; search that short circuits when it his a package file in the dir. This speeds it up greatly as the majority of the size is in the binary-* sub dirs. */ -bool pkgCdrom::FindPackages(string CD,vector<string> &List, - vector<string> &SList, vector<string> &SigList, +bool pkgCdrom::FindPackages(string CD, + vector<string> &List, + vector<string> &SList, + vector<string> &SigList, + vector<string> &TransList, string &InfoDir, pkgCdromStatus *log, unsigned int Depth) { static ino_t Inodes[9]; + DIR *D; // if we have a look we "pulse" now if(log) @@ -90,8 +94,28 @@ bool pkgCdrom::FindPackages(string CD,vector<string> &List, if (_config->FindB("APT::CDROM::Thorough",false) == false) return true; } + + // see if we find translatin indexes + if (stat("i18n",&Buf) == 0) + { + D = opendir("i18n"); + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + if(strstr(Dir->d_name,"Translation") != NULL) + { + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "found translations: " << Dir->d_name << "\n"; + string file = Dir->d_name; + if(file.substr(file.size()-3,file.size()) == ".gz") + file = file.substr(0,file.size()-3); + TransList.push_back(CD+"i18n/"+ file); + } + } + closedir(D); + } + - DIR *D = opendir("."); + D = opendir("."); if (D == 0) return _error->Errno("opendir","Unable to read %s",CD.c_str()); @@ -127,7 +151,7 @@ bool pkgCdrom::FindPackages(string CD,vector<string> &List, Inodes[Depth] = Buf.st_ino; // Descend - if (FindPackages(CD + Dir->d_name,List,SList,SigList,InfoDir,log,Depth+1) == false) + if (FindPackages(CD + Dir->d_name,List,SList,SigList,TransList,InfoDir,log,Depth+1) == false) break; if (chdir(CD.c_str()) != 0) @@ -612,9 +636,10 @@ bool pkgCdrom::Add(pkgCdromStatus *log) vector<string> List; vector<string> SourceList; vector<string> SigList; + vector<string> TransList; string StartDir = SafeGetCWD(); string InfoDir; - if (FindPackages(CDROM,List,SourceList, SigList,InfoDir,log) == false) + if (FindPackages(CDROM,List,SourceList, SigList,TransList,InfoDir,log) == false) { log->Update("\n"); return false; @@ -642,11 +667,13 @@ bool pkgCdrom::Add(pkgCdromStatus *log) DropRepeats(List,"Packages"); DropRepeats(SourceList,"Sources"); DropRepeats(SigList,"Release.gpg"); + DropRepeats(TransList,""); if(log) { msg.str(""); - ioprintf(msg, _("Found %i package indexes, %i source indexes and " - "%i signatures\n"), - List.size(), SourceList.size(), SigList.size()); + ioprintf(msg, _("Found %i package indexes, %i source indexes, " + "%i translation indexes and %i signatures\n"), + List.size(), SourceList.size(), TransList.size(), + SigList.size()); log->Update(msg.str(), STEP_SCAN); } @@ -736,8 +763,10 @@ bool pkgCdrom::Add(pkgCdromStatus *log) // Copy the package files to the state directory PackageCopy Copy; SourceCopy SrcCopy; + TranslationsCopy TransCopy; if (Copy.CopyPackages(CDROM,Name,List, log) == false || - SrcCopy.CopyPackages(CDROM,Name,SourceList, log) == false) + SrcCopy.CopyPackages(CDROM,Name,SourceList, log) == false || + TransCopy.CopyTranslations(CDROM,Name,TransList, log) == false) return false; // reduce the List so that it takes less space in sources.list diff --git a/apt-pkg/cdrom.h b/apt-pkg/cdrom.h index 085eb64e2..e18aaff3e 100644 --- a/apt-pkg/cdrom.h +++ b/apt-pkg/cdrom.h @@ -50,8 +50,11 @@ class pkgCdrom }; - bool FindPackages(string CD,vector<string> &List, - vector<string> &SList, vector<string> &SigList, + bool FindPackages(string CD, + vector<string> &List, + vector<string> &SList, + vector<string> &SigList, + vector<string> &TransList, string &InfoDir, pkgCdromStatus *log, unsigned int Depth = 0); bool DropBinaryArch(vector<string> &List); diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index d96155917..37d263794 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -32,12 +32,55 @@ #include <regex.h> #include <errno.h> #include <stdarg.h> +#include <iconv.h> #include "config.h" using namespace std; /*}}}*/ +// UTF8ToCodeset - Convert some UTF-8 string for some codeset /*{{{*/ +// --------------------------------------------------------------------- +/* This is handy to use before display some information for enduser */ +bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest) +{ + iconv_t cd; + const char *inbuf; + char *inptr, *outbuf, *outptr; + size_t insize, outsize; + + cd = iconv_open(codeset, "UTF-8"); + if (cd == (iconv_t)(-1)) { + // Something went wrong + if (errno == EINVAL) + _error->Error("conversion from 'UTF-8' to '%s' not available", + codeset); + else + perror("iconv_open"); + + // Clean the destination string + *dest = ""; + + return false; + } + + insize = outsize = orig.size(); + inbuf = orig.data(); + inptr = (char *)inbuf; + outbuf = new char[insize+1]; + outptr = outbuf; + + iconv(cd, &inptr, &insize, &outptr, &outsize); + *outptr = '\0'; + + *dest = outbuf; + delete[] outbuf; + + iconv_close(cd); + + return true; +} + /*}}}*/ // strstrip - Remove white space from the front and back of a string /*{{{*/ // --------------------------------------------------------------------- /* This is handy to use when parsing a file. It also removes \n's left diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index 6ec2b7811..254087267 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -38,7 +38,8 @@ using std::ostream; #define APT_FORMAT2 #define APT_FORMAT3 #endif - + +bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest); char *_strstrip(char *String); char *_strtabexpand(char *String,size_t Len); bool ParseQuoteWord(const char *&String,string &Res); diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index ff8bce85d..38ecdd16a 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -320,6 +320,170 @@ pkgCache::PkgFileIterator debPackagesIndex::FindInCache(pkgCache &Cache) const } /*}}}*/ +// TranslationsIndex::debTranslationsIndex - Contructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section) : + pkgIndexFile(true), URI(URI), Dist(Dist), Section(Section) +{ +} + /*}}}*/ +// TranslationIndex::Trans* - Return the URI to the translation files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +inline string debTranslationsIndex::IndexFile(const char *Type) const +{ + return _config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); +} +string debTranslationsIndex::IndexURI(const char *Type) const +{ + string Res; + if (Dist[Dist.size() - 1] == '/') + { + if (Dist != "/") + Res = URI + Dist; + else + Res = URI; + } + else + Res = URI + "dists/" + Dist + '/' + Section + + "/i18n/Translation-"; + + Res += Type; + return Res; +} + /*}}}*/ +// TranslationsIndex::GetIndexes - Fetch the index files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debTranslationsIndex::GetIndexes(pkgAcquire *Owner) const +{ + if (TranslationsAvailable()) { + string TranslationFile = "Translation-" + LanguageCode(); + new pkgAcqIndexTrans(Owner, IndexURI(LanguageCode().c_str()), + Info(TranslationFile.c_str()), + TranslationFile); + } + + return true; +} + /*}}}*/ +// TranslationsIndex::Describe - Give a descriptive path to the index /*{{{*/ +// --------------------------------------------------------------------- +/* This should help the user find the index in the sources.list and + in the filesystem for problem solving */ +string debTranslationsIndex::Describe(bool Short) const +{ + char S[300]; + if (Short == true) + snprintf(S,sizeof(S),"%s",Info(TranslationFile().c_str()).c_str()); + else + snprintf(S,sizeof(S),"%s (%s)",Info(TranslationFile().c_str()).c_str(), + IndexFile(LanguageCode().c_str()).c_str()); + return S; +} + /*}}}*/ +// TranslationsIndex::Info - One liner describing the index URI /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string debTranslationsIndex::Info(const char *Type) const +{ + string Info = ::URI::SiteOnly(URI) + ' '; + if (Dist[Dist.size() - 1] == '/') + { + if (Dist != "/") + Info += Dist; + } + else + Info += Dist + '/' + Section; + Info += " "; + Info += Type; + return Info; +} + /*}}}*/ +bool debTranslationsIndex::HasPackages() const +{ + if(!TranslationsAvailable()) + return false; + + return FileExists(IndexFile(LanguageCode().c_str())); +} + +// TranslationsIndex::Exists - Check if the index is available /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debTranslationsIndex::Exists() const +{ + return FileExists(IndexFile(LanguageCode().c_str())); +} + /*}}}*/ +// TranslationsIndex::Size - Return the size of the index /*{{{*/ +// --------------------------------------------------------------------- +/* This is really only used for progress reporting. */ +unsigned long debTranslationsIndex::Size() const +{ + struct stat S; + if (stat(IndexFile(LanguageCode().c_str()).c_str(),&S) != 0) + return 0; + return S.st_size; +} + /*}}}*/ +// TranslationsIndex::Merge - Load the index file into a cache /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const +{ + // Check the translation file, if in use + string TranslationFile = IndexFile(LanguageCode().c_str()); + if (TranslationsAvailable() && FileExists(TranslationFile)) + { + FileFd Trans(TranslationFile,FileFd::ReadOnly); + debListParser TransParser(&Trans); + if (_error->PendingError() == true) + return false; + + Prog.SubProgress(0, Info(TranslationFile.c_str())); + if (Gen.SelectFile(TranslationFile,string(),*this) == false) + return _error->Error("Problem with SelectFile %s",TranslationFile.c_str()); + + // Store the IMS information + pkgCache::PkgFileIterator TransFile = Gen.GetCurFile(); + struct stat TransSt; + if (fstat(Trans.Fd(),&TransSt) != 0) + return _error->Errno("fstat","Failed to stat"); + TransFile->Size = TransSt.st_size; + TransFile->mtime = TransSt.st_mtime; + + if (Gen.MergeList(TransParser) == false) + return _error->Error("Problem with MergeList %s",TranslationFile.c_str()); + } + + return true; +} + /*}}}*/ +// TranslationsIndex::FindInCache - Find this index /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgCache::PkgFileIterator debTranslationsIndex::FindInCache(pkgCache &Cache) const +{ + string FileName = IndexFile(LanguageCode().c_str()); + + pkgCache::PkgFileIterator File = Cache.FileBegin(); + for (; File.end() == false; File++) + { + if (FileName != File.FileName()) + continue; + + struct stat St; + if (stat(File.FileName(),&St) != 0) + return pkgCache::PkgFileIterator(Cache); + if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime) + return pkgCache::PkgFileIterator(Cache); + return File; + } + return File; +} + /*}}}*/ // StatusIndex::debStatusIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -416,6 +580,11 @@ class debIFTypePkg : public pkgIndexFile::Type }; debIFTypePkg() {Label = "Debian Package Index";}; }; +class debIFTypeTrans : public debIFTypePkg +{ + public: + debIFTypeTrans() {Label = "Debian Translation Index";}; +}; class debIFTypeStatus : public pkgIndexFile::Type { public: @@ -428,6 +597,7 @@ class debIFTypeStatus : public pkgIndexFile::Type }; static debIFTypeSrc _apt_Src; static debIFTypePkg _apt_Pkg; +static debIFTypeTrans _apt_Trans; static debIFTypeStatus _apt_Status; const pkgIndexFile::Type *debSourcesIndex::GetType() const @@ -438,6 +608,10 @@ const pkgIndexFile::Type *debPackagesIndex::GetType() const { return &_apt_Pkg; } +const pkgIndexFile::Type *debTranslationsIndex::GetType() const +{ + return &_apt_Trans; +} const pkgIndexFile::Type *debStatusIndex::GetType() const { return &_apt_Status; diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index a1b9583a4..57005222f 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -74,6 +74,36 @@ class debPackagesIndex : public pkgIndexFile debPackagesIndex(string URI,string Dist,string Section,bool Trusted); }; +class debTranslationsIndex : public pkgIndexFile +{ + string URI; + string Dist; + string Section; + + string Info(const char *Type) const; + string IndexFile(const char *Type) const; + string IndexURI(const char *Type) const; + + inline string TranslationFile() const {return "Translation-" + LanguageCode();}; + + public: + + virtual const Type *GetType() const; + + // Interface for acquire + virtual string Describe(bool Short) const; + virtual bool GetIndexes(pkgAcquire *Owner) const; + + // Interface for the Cache Generator + virtual bool Exists() const; + virtual bool HasPackages() const; + virtual unsigned long Size() const; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const; + virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; + + debTranslationsIndex(string URI,string Dist,string Section); +}; + class debSourcesIndex : public pkgIndexFile { string URI; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index d0dc7a260..c2b26b5eb 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -15,6 +15,7 @@ #include <apt-pkg/configuration.h> #include <apt-pkg/strutl.h> #include <apt-pkg/crc-16.h> +#include <apt-pkg/md5.h> #include <ctype.h> @@ -117,6 +118,48 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) return true; } /*}}}*/ +// ListParser::Description - Return the description string /*{{{*/ +// --------------------------------------------------------------------- +/* This is to return the string describing the package in debian + form. If this returns the blank string then the entry is assumed to + only describe package properties */ +string debListParser::Description() +{ + if (DescriptionLanguage().empty()) + return Section.FindS("Description"); + else + return Section.FindS(("Description-" + pkgIndexFile::LanguageCode()).c_str()); +} + /*}}}*/ +// ListParser::DescriptionLanguage - Return the description lang string /*{{{*/ +// --------------------------------------------------------------------- +/* This is to return the string describing the language of + description. If this returns the blank string then the entry is + assumed to describe original description. */ +string debListParser::DescriptionLanguage() +{ + return Section.FindS("Description").empty() ? pkgIndexFile::LanguageCode() : ""; +} + /*}}}*/ +// ListParser::Description - Return the description_md5 MD5SumValue /*{{{*/ +// --------------------------------------------------------------------- +/* This is to return the md5 string to allow the check if it is the right + description. If no Description-md5 is found in the section it will be + calculated. + */ +MD5SumValue debListParser::Description_md5() +{ + string value = Section.FindS("Description-md5"); + + if (value.empty()) + { + MD5Summation md5; + md5.Add((Description() + "\n").c_str()); + return md5.Result(); + } else + return MD5SumValue(value); +} + /*}}}*/ // ListParser::UsePackage - Update a package structure /*{{{*/ // --------------------------------------------------------------------- /* This is called to update the package with any new information diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 3a0e0421b..34bb29c72 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -12,6 +12,7 @@ #define PKGLIB_DEBLISTPARSER_H #include <apt-pkg/pkgcachegen.h> +#include <apt-pkg/indexfile.h> #include <apt-pkg/tagfile.h> class debListParser : public pkgCacheGenerator::ListParser @@ -47,6 +48,9 @@ class debListParser : public pkgCacheGenerator::ListParser virtual string Package(); virtual string Version(); virtual bool NewVersion(pkgCache::VerIterator Ver); + virtual string Description(); + virtual string DescriptionLanguage(); + virtual MD5SumValue Description_md5(); virtual unsigned short VersionHash(); virtual bool UsePackage(pkgCache::PkgIterator Pkg, pkgCache::VerIterator Ver); diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 85e5b16b3..d3b6ed957 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -157,6 +157,14 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool GetAll) const ComputeIndexTargets(), new indexRecords (Dist)); + // Queue the translations + for (vector<const debSectionEntry *>::const_iterator I = SectionEntries.begin(); + I != SectionEntries.end(); I++) { + + debTranslationsIndex i = debTranslationsIndex(URI,Dist,(*I)->Section); + i.GetIndexes(Owner); + } + return true; } @@ -181,11 +189,16 @@ vector <pkgIndexFile *> *debReleaseIndex::GetIndexFiles() Indexes = new vector <pkgIndexFile*>; for (vector<const debSectionEntry *>::const_iterator I = SectionEntries.begin(); - I != SectionEntries.end(); I++) + I != SectionEntries.end(); I++) { if ((*I)->IsSrc) Indexes->push_back(new debSourcesIndex (URI, Dist, (*I)->Section, IsTrusted())); else + { Indexes->push_back(new debPackagesIndex (URI, Dist, (*I)->Section, IsTrusted())); + Indexes->push_back(new debTranslationsIndex(URI, Dist, (*I)->Section)); + } + } + return Indexes; } diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc index 6652a6ad9..518988bb6 100644 --- a/apt-pkg/deb/debrecords.cc +++ b/apt-pkg/deb/debrecords.cc @@ -12,7 +12,9 @@ #pragma implementation "apt-pkg/debrecords.h" #endif #include <apt-pkg/debrecords.h> +#include <apt-pkg/strutl.h> #include <apt-pkg/error.h> +#include <langinfo.h> /*}}}*/ // RecordParser::debRecordParser - Constructor /*{{{*/ @@ -31,6 +33,10 @@ bool debRecordParser::Jump(pkgCache::VerFileIterator const &Ver) { return Tags.Jump(Section,Ver->Offset); } +bool debRecordParser::Jump(pkgCache::DescFileIterator const &Desc) +{ + return Tags.Jump(Section,Desc->Offset); +} /*}}}*/ // RecordParser::FileName - Return the archive filename on the site /*{{{*/ // --------------------------------------------------------------------- @@ -77,7 +83,7 @@ string debRecordParser::Maintainer() /* */ string debRecordParser::ShortDesc() { - string Res = Section.FindS("Description"); + string Res = LongDesc(); string::size_type Pos = Res.find('\n'); if (Pos == string::npos) return Res; @@ -89,7 +95,20 @@ string debRecordParser::ShortDesc() /* */ string debRecordParser::LongDesc() { - return Section.FindS("Description"); + string orig, dest; + char *codeset = nl_langinfo(CODESET); + + if (!Section.FindS("Description").empty()) + orig = Section.FindS("Description").c_str(); + else + orig = Section.FindS(("Description-" + pkgIndexFile::LanguageCode()).c_str()).c_str(); + + if (strcmp(codeset,"UTF-8") != 0) { + UTF8ToCodeset(codeset, orig, &dest); + orig = dest; + } + + return orig; } /*}}}*/ // RecordParser::SourcePkg - Return the source package name if any /*{{{*/ diff --git a/apt-pkg/deb/debrecords.h b/apt-pkg/deb/debrecords.h index efef2e588..24e5aab88 100644 --- a/apt-pkg/deb/debrecords.h +++ b/apt-pkg/deb/debrecords.h @@ -19,6 +19,7 @@ #endif #include <apt-pkg/pkgrecords.h> +#include <apt-pkg/indexfile.h> #include <apt-pkg/tagfile.h> class debRecordParser : public pkgRecords::Parser @@ -30,6 +31,7 @@ class debRecordParser : public pkgRecords::Parser protected: virtual bool Jump(pkgCache::VerFileIterator const &Ver); + virtual bool Jump(pkgCache::DescFileIterator const &Desc); public: diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 1f65062f7..c9dded134 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -32,6 +32,8 @@ using namespace std; + + // IndexCopy::CopyPackages - Copy the package files from the CD /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -512,10 +514,10 @@ bool SourceCopy::RewriteEntry(FILE *Target,string File) fputc('\n',Target); return true; } - - /*}}}*/ - +// SigVerify::Verify - Verify a files md5sum against its metaindex /*{{{*/ +// --------------------------------------------------------------------- +/* */ bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex) { const indexRecords::checkSum *Record = MetaIndex->Lookup(file); @@ -670,3 +672,178 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList, return true; } + + +bool TranslationsCopy::CopyTranslations(string CDROM,string Name,vector<string> &List, + pkgCdromStatus *log) +{ + OpProgress *Progress = NULL; + if (List.size() == 0) + return true; + + if(log) + Progress = log->GetOpProgress(); + + bool Debug = _config->FindB("Debug::aptcdrom",false); + + // Prepare the progress indicator + unsigned long TotalSize = 0; + for (vector<string>::iterator I = List.begin(); I != List.end(); I++) + { + struct stat Buf; + if (stat(string(*I).c_str(),&Buf) != 0 && + stat(string(*I + ".gz").c_str(),&Buf) != 0) + return _error->Errno("stat","Stat failed for %s", + string(*I).c_str()); + TotalSize += Buf.st_size; + } + + unsigned long CurrentSize = 0; + unsigned int NotFound = 0; + unsigned int WrongSize = 0; + unsigned int Packages = 0; + for (vector<string>::iterator I = List.begin(); I != List.end(); I++) + { + string OrigPath = string(*I,CDROM.length()); + unsigned long FileSize = 0; + + // Open the package file + FileFd Pkg; + if (FileExists(*I) == true) + { + Pkg.Open(*I,FileFd::ReadOnly); + FileSize = Pkg.Size(); + } + else + { + FileFd From(*I + ".gz",FileFd::ReadOnly); + if (_error->PendingError() == true) + return false; + FileSize = From.Size(); + + // Get a temp file + FILE *tmp = tmpfile(); + if (tmp == 0) + return _error->Errno("tmpfile","Unable to create a tmp file"); + Pkg.Fd(dup(fileno(tmp))); + fclose(tmp); + + // Fork gzip + pid_t Process = fork(); + if (Process < 0) + return _error->Errno("fork","Couldn't fork gzip"); + + // The child + if (Process == 0) + { + dup2(From.Fd(),STDIN_FILENO); + dup2(Pkg.Fd(),STDOUT_FILENO); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDOUT_FILENO,false); + + const char *Args[3]; + string Tmp = _config->Find("Dir::bin::gzip","gzip"); + Args[0] = Tmp.c_str(); + Args[1] = "-d"; + Args[2] = 0; + execvp(Args[0],(char **)Args); + exit(100); + } + + // Wait for gzip to finish + if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false) + return _error->Error("gzip failed, perhaps the disk is full."); + + Pkg.Seek(0); + } + pkgTagFile Parser(&Pkg); + if (_error->PendingError() == true) + return false; + + // Open the output file + char S[400]; + snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(), + (*I).c_str() + CDROM.length()); + string TargetF = _config->FindDir("Dir::State::lists") + "partial/"; + TargetF += URItoFileName(S); + if (_config->FindB("APT::CDROM::NoAct",false) == true) + TargetF = "/dev/null"; + FileFd Target(TargetF,FileFd::WriteEmpty); + FILE *TargetFl = fdopen(dup(Target.Fd()),"w"); + if (_error->PendingError() == true) + return false; + if (TargetFl == 0) + return _error->Errno("fdopen","Failed to reopen fd"); + + // Setup the progress meter + if(Progress) + Progress->OverallProgress(CurrentSize,TotalSize,FileSize, + string("Reading Translation Indexes")); + + // Parse + if(Progress) + Progress->SubProgress(Pkg.Size()); + pkgTagSection Section; + this->Section = &Section; + string Prefix; + unsigned long Hits = 0; + unsigned long Chop = 0; + while (Parser.Step(Section) == true) + { + if(Progress) + Progress->Progress(Parser.Offset()); + + const char *Start; + const char *Stop; + Section.GetSection(Start,Stop); + fwrite(Start,Stop-Start, 1, TargetFl); + fputc('\n',TargetFl); + + Packages++; + Hits++; + } + fclose(TargetFl); + + if (Debug == true) + cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl; + + if (_config->FindB("APT::CDROM::NoAct",false) == false) + { + // Move out of the partial directory + Target.Close(); + string FinalF = _config->FindDir("Dir::State::lists"); + FinalF += URItoFileName(S); + if (rename(TargetF.c_str(),FinalF.c_str()) != 0) + return _error->Errno("rename","Failed to rename"); + } + + + CurrentSize += FileSize; + } + if(Progress) + Progress->Done(); + + // Some stats + if(log) { + stringstream msg; + if(NotFound == 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records.\n"), Packages); + else if (NotFound != 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records with %i missing files.\n"), + Packages, NotFound); + else if (NotFound == 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), + Packages, WrongSize); + if (NotFound != 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize); + } + + if (Packages == 0) + _error->Warning("No valid records were found."); + + if (NotFound + WrongSize > 10) + _error->Warning("Alot of entries were discarded, something may be wrong.\n"); + + + return true; +} diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h index 4dcb2b46d..7778ae595 100644 --- a/apt-pkg/indexcopy.h +++ b/apt-pkg/indexcopy.h @@ -70,6 +70,17 @@ class SourceCopy : public IndexCopy public: }; +class TranslationsCopy +{ + protected: + pkgTagSection *Section; + + public: + bool CopyTranslations(string CDROM,string Name,vector<string> &List, + pkgCdromStatus *log); +}; + + class SigVerify { bool Verify(string prefix,string file, indexRecords *records); @@ -81,4 +92,6 @@ class SigVerify vector<string> PkgList,vector<string> SrcList); }; + + #endif diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc index 49665161d..496e68b8b 100644 --- a/apt-pkg/indexfile.cc +++ b/apt-pkg/indexfile.cc @@ -12,8 +12,11 @@ #pragma implementation "apt-pkg/indexfile.h" #endif +#include <apt-pkg/configuration.h> #include <apt-pkg/indexfile.h> #include <apt-pkg/error.h> + +#include <clocale> /*}}}*/ // Global list of Item supported @@ -67,3 +70,60 @@ string pkgIndexFile::SourceInfo(pkgSrcRecords::Parser const &Record, return string(); } /*}}}*/ +// IndexFile::TranslationsAvailable - Check if will use Translation /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgIndexFile::TranslationsAvailable() +{ + const string Translation = _config->Find("APT::Acquire::Translation"); + + if (Translation.compare("none") != 0) + return CheckLanguageCode(LanguageCode().c_str()); + else + return false; +} + /*}}}*/ +// IndexFile::CheckLanguageCode - Check the Language Code /*{{{*/ +// --------------------------------------------------------------------- +/* */ +/* common cases: de_DE, de_DE@euro, de_DE.UTF-8, de_DE.UTF-8@euro, + de_DE.ISO8859-1, tig_ER + more in /etc/gdm/locale.conf +*/ + +bool pkgIndexFile::CheckLanguageCode(const char *Lang) +{ + if (strlen(Lang) == 2 || (strlen(Lang) == 5 && Lang[2] == '_')) + return true; + + if (strcmp(Lang,"C") != 0) + _error->Warning("Wrong language code %s", Lang); + + return false; +} + /*}}}*/ +// IndexFile::LanguageCode - Return the Language Code /*{{{*/ +// --------------------------------------------------------------------- +/* return the language code */ +string pkgIndexFile::LanguageCode() +{ + const string Translation = _config->Find("APT::Acquire::Translation"); + + if (Translation.compare("environment") == 0) + { + string lang = std::setlocale(LC_MESSAGES,NULL); + + // FIXME: this needs to be added + // we have a mapping of the language codes that contains all the language + // codes that need the country code as well + // (like pt_BR, pt_PT, sv_SE, zh_*, en_*) + + if(lang.size() > 2) + return lang.substr(0,2); + else + return lang; + } + else + return Translation; +} + /*}}}*/ diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index 61049f4bd..d5d1cf57a 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -5,10 +5,11 @@ Index File - Abstraction for an index of archive/source file. - There are 3 primary sorts of index files, all represented by this + There are 4 primary sorts of index files, all represented by this class: Binary index files + Binary translation files Bianry index files decribing the local system Source index files @@ -80,6 +81,10 @@ class pkgIndexFile virtual bool MergeFileProvides(pkgCacheGenerator &/*Gen*/,OpProgress &/*Prog*/) const {return true;}; virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; + static bool TranslationsAvailable(); + static bool CheckLanguageCode(const char *Lang); + static string LanguageCode(); + bool IsTrusted() const { return Trusted; }; pkgIndexFile(bool Trusted): Trusted(Trusted) {}; diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index b47378d4a..6aa486a7f 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -102,6 +102,9 @@ bool pkgInitConfig(Configuration &Cnf) bindtextdomain(textdomain(0),Cnf.FindDir("Dir::Locale").c_str()); } #endif + + // Translation + Cnf.Set("APT::Acquire::Translation", "environment"); return true; } diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 8255b406a..63547619f 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -17,8 +17,8 @@ #include <apt-pkg/pkgsystem.h> // See the makefile -#define APT_PKG_MAJOR 3 -#define APT_PKG_MINOR 11 +#define APT_PKG_MAJOR 4 +#define APT_PKG_MINOR 0 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 7e5feae53..7887fce92 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=3.11 +MAJOR=4.0 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 9926befe9..162ab4f27 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -26,6 +26,7 @@ #endif #include <apt-pkg/pkgcache.h> +#include <apt-pkg/indexfile.h> #include <apt-pkg/version.h> #include <apt-pkg/error.h> #include <apt-pkg/strutl.h> @@ -43,6 +44,7 @@ using std::string; + // Cache::Header::Header - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Simply initialize the header */ @@ -52,7 +54,7 @@ pkgCache::Header::Header() /* Whenever the structures change the major version should be bumped, whenever the generator changes the minor version should be bumped. */ - MajorVersion = 4; + MajorVersion = 5; MinorVersion = 0; Dirty = false; @@ -60,17 +62,22 @@ pkgCache::Header::Header() PackageSz = sizeof(pkgCache::Package); PackageFileSz = sizeof(pkgCache::PackageFile); VersionSz = sizeof(pkgCache::Version); + DescriptionSz = sizeof(pkgCache::Description); DependencySz = sizeof(pkgCache::Dependency); ProvidesSz = sizeof(pkgCache::Provides); VerFileSz = sizeof(pkgCache::VerFile); + DescFileSz = sizeof(pkgCache::DescFile); PackageCount = 0; VersionCount = 0; + DescriptionCount = 0; DependsCount = 0; PackageFileCount = 0; VerFileCount = 0; + DescFileCount = 0; ProvidesCount = 0; MaxVerFileSize = 0; + MaxDescFileSize = 0; FileList = 0; StringList = 0; @@ -89,8 +96,10 @@ bool pkgCache::Header::CheckSizes(Header &Against) const PackageSz == Against.PackageSz && PackageFileSz == Against.PackageFileSz && VersionSz == Against.VersionSz && + DescriptionSz == Against.DescriptionSz && DependencySz == Against.DependencySz && VerFileSz == Against.VerFileSz && + DescFileSz == Against.DescFileSz && ProvidesSz == Against.ProvidesSz) return true; return false; @@ -115,8 +124,10 @@ bool pkgCache::ReMap() HeaderP = (Header *)Map.Data(); PkgP = (Package *)Map.Data(); VerFileP = (VerFile *)Map.Data(); + DescFileP = (DescFile *)Map.Data(); PkgFileP = (PackageFile *)Map.Data(); VerP = (Version *)Map.Data(); + DescP = (Description *)Map.Data(); ProvideP = (Provides *)Map.Data(); DepP = (Dependency *)Map.Data(); StringItemP = (StringItem *)Map.Data(); @@ -235,11 +246,11 @@ const char *pkgCache::Priority(unsigned char Prio) return 0; } /*}}}*/ - // Bases for iterator classes /*{{{*/ void pkgCache::VerIterator::_dummy() {} void pkgCache::DepIterator::_dummy() {} void pkgCache::PrvIterator::_dummy() {} +void pkgCache::DescIterator::_dummy() {} /*}}}*/ // PkgIterator::operator ++ - Postfix incr /*{{{*/ // --------------------------------------------------------------------- @@ -599,3 +610,20 @@ string pkgCache::PkgFileIterator::RelStr() return Res; } /*}}}*/ +// VerIterator::TranslatedDescription - Return the a DescIter for locale/*{{{*/ +// --------------------------------------------------------------------- +/* return a DescIter for the current locale or the default if none is + * found + */ +pkgCache::DescIterator pkgCache::VerIterator::TranslatedDescription() const +{ + pkgCache::DescIterator DescDefault = DescriptionList(); + pkgCache::DescIterator Desc = DescDefault; + for (; Desc.end() == false; Desc++) + if (pkgIndexFile::LanguageCode() == Desc.LanguageCode()) + break; + if (Desc.end() == true) Desc = DescDefault; + return Desc; +}; + + /*}}}*/ diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 587d97534..c7a3172cc 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -38,24 +38,30 @@ class pkgCache struct Package; struct PackageFile; struct Version; + struct Description; struct Provides; struct Dependency; struct StringItem; struct VerFile; + struct DescFile; // Iterators class PkgIterator; class VerIterator; + class DescIterator; class DepIterator; class PrvIterator; class PkgFileIterator; class VerFileIterator; + class DescFileIterator; friend class PkgIterator; friend class VerIterator; + friend class DescInterator; friend class DepIterator; friend class PrvIterator; friend class PkgFileIterator; friend class VerFileIterator; + friend class DescFileIterator; class Namespace; @@ -98,8 +104,10 @@ class pkgCache Header *HeaderP; Package *PkgP; VerFile *VerFileP; + DescFile *DescFileP; PackageFile *PkgFileP; Version *VerP; + Description *DescP; Provides *ProvideP; Dependency *DepP; StringItem *StringItemP; @@ -151,16 +159,20 @@ struct pkgCache::Header unsigned short PackageSz; unsigned short PackageFileSz; unsigned short VersionSz; + unsigned short DescriptionSz; unsigned short DependencySz; unsigned short ProvidesSz; unsigned short VerFileSz; + unsigned short DescFileSz; // Structure counts unsigned long PackageCount; unsigned long VersionCount; + unsigned long DescriptionCount; unsigned long DependsCount; unsigned long PackageFileCount; unsigned long VerFileCount; + unsigned long DescFileCount; unsigned long ProvidesCount; // Offsets @@ -169,10 +181,11 @@ struct pkgCache::Header map_ptrloc VerSysName; // StringTable map_ptrloc Architecture; // StringTable unsigned long MaxVerFileSize; + unsigned long MaxDescFileSize; /* Allocation pools, there should be one of these for each structure excluding the header */ - DynamicMMap::Pool Pools[7]; + DynamicMMap::Pool Pools[8]; // Rapid package name lookup map_ptrloc HashTable[2*1048]; @@ -193,7 +206,7 @@ struct pkgCache::Package map_ptrloc NextPackage; // Package map_ptrloc RevDepends; // Dependency map_ptrloc ProvidesList; // Provides - + // Install/Remove/Purge etc unsigned char SelectedState; // What unsigned char InstState; // Flags @@ -232,6 +245,14 @@ struct pkgCache::VerFile unsigned short Size; }; +struct pkgCache::DescFile +{ + map_ptrloc File; // PackageFile + map_ptrloc NextFile; // PkgVerFile + map_ptrloc Offset; // File offset + unsigned short Size; +}; + struct pkgCache::Version { map_ptrloc VerStr; // Stringtable @@ -241,6 +262,7 @@ struct pkgCache::Version // Lists map_ptrloc FileList; // VerFile map_ptrloc NextVer; // Version + map_ptrloc DescriptionList; // Description map_ptrloc DependsList; // Dependency map_ptrloc ParentPkg; // Package map_ptrloc ProvidesList; // Provides @@ -252,6 +274,22 @@ struct pkgCache::Version unsigned char Priority; }; +struct pkgCache::Description +{ + // Language Code store the description translation language code. If + // the value has a 0 lenght then this is readed using the Package + // file else the Translation-CODE are used. + map_ptrloc language_code; // StringTable + map_ptrloc md5sum; // StringTable + + // Linked list + map_ptrloc FileList; // DescFile + map_ptrloc NextDesc; // Description + map_ptrloc ParentPkg; // Package + + unsigned short ID; +}; + struct pkgCache::Dependency { map_ptrloc Version; // Stringtable @@ -299,11 +337,13 @@ class pkgCache::Namespace typedef pkgCache::PkgIterator PkgIterator; typedef pkgCache::VerIterator VerIterator; + typedef pkgCache::DescIterator DescIterator; typedef pkgCache::DepIterator DepIterator; typedef pkgCache::PrvIterator PrvIterator; typedef pkgCache::PkgFileIterator PkgFileIterator; typedef pkgCache::VerFileIterator VerFileIterator; typedef pkgCache::Version Version; + typedef pkgCache::Description Description; typedef pkgCache::Package Package; typedef pkgCache::Header Header; typedef pkgCache::Dep Dep; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index de854bee5..1106667d5 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -125,16 +125,46 @@ bool pkgCacheGenerator::MergeList(ListParser &List, string Version = List.Version(); if (Version.empty() == true) { + // we first process the package, then the descriptions + // (this has the bonus that we get MMap error when we run out + // of MMap space) if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false) return _error->Error(_("Error occurred while processing %s (UsePackage1)"), PackageName.c_str()); + + // Find the right version to write the description + MD5SumValue CurMd5 = List.Description_md5(); + pkgCache::VerIterator Ver = Pkg.VersionList(); + map_ptrloc *LastVer = &Pkg->VersionList; + + for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) + { + pkgCache::DescIterator Desc = Ver.DescriptionList(); + map_ptrloc *LastDesc = &Ver->DescriptionList; + + for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++) + { + + if (MD5SumValue(Desc.md5()) == CurMd5) + { + // Add new description + *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); + Desc->ParentPkg = Pkg.Index(); + + if (NewFileDesc(Desc,List) == false) + return _error->Error(_("Error occured while processing %s (NewFileDesc1)"),PackageName.c_str()); + break; + } + } + } + continue; } pkgCache::VerIterator Ver = Pkg.VersionList(); - map_ptrloc *Last = &Pkg->VersionList; + map_ptrloc *LastVer = &Pkg->VersionList; int Res = 1; - for (; Ver.end() == false; Last = &Ver->NextVer, Ver++) + for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) { Res = Cache.VS->CmpVersion(Version,Ver.VerStr()); if (Res >= 0) @@ -168,7 +198,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // Skip to the end of the same version set. if (Res == 0) { - for (; Ver.end() == false; Last = &Ver->NextVer, Ver++) + for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) { Res = Cache.VS->CmpVersion(Version,Ver.VerStr()); if (Res != 0) @@ -177,9 +207,10 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } // Add a new version - *Last = NewVersion(Ver,Version,*Last); + *LastVer = NewVersion(Ver,Version,*LastVer); Ver->ParentPkg = Pkg.Index(); Ver->Hash = Hash; + if (List.NewVersion(Ver) == false) return _error->Error(_("Error occurred while processing %s (NewVersion1)"), PackageName.c_str()); @@ -199,6 +230,21 @@ bool pkgCacheGenerator::MergeList(ListParser &List, FoundFileDeps |= List.HasFileDeps(); return true; } + + /* Record the Description data. Description data always exist in + Packages and Translation-* files. */ + pkgCache::DescIterator Desc = Ver.DescriptionList(); + map_ptrloc *LastDesc = &Ver->DescriptionList; + + // Skip to the end of description set + for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++); + + // Add new description + *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); + Desc->ParentPkg = Pkg.Index(); + + if (NewFileDesc(Desc,List) == false) + return _error->Error(_("Error occured while processing %s (NewFileDesc2)"),PackageName.c_str()); } FoundFileDeps |= List.HasFileDeps(); @@ -209,6 +255,9 @@ bool pkgCacheGenerator::MergeList(ListParser &List, if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1) return _error->Error(_("Wow, you exceeded the number of versions " "this APT is capable of.")); + if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1) + return _error->Error(_("Wow, you exceeded the number of descriptions " + "this APT is capable of.")); if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL) return _error->Error(_("Wow, you exceeded the number of dependencies " "this APT is capable of.")); @@ -271,7 +320,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name Pkg = Cache.FindPkg(Name); if (Pkg.end() == false) return true; - + // Get a structure unsigned long Package = Map.Allocate(sizeof(pkgCache::Package)); if (Package == 0) @@ -349,6 +398,62 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, return Version; } /*}}}*/ +// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, + ListParser &List) +{ + if (CurrentFile == 0) + return true; + + // Get a structure + unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile)); + if (DescFile == 0) + return 0; + + pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile); + DF->File = CurrentFile - Cache.PkgFileP; + + // Link it to the end of the list + map_ptrloc *Last = &Desc->FileList; + for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++) + Last = &D->NextFile; + + DF->NextFile = *Last; + *Last = DF.Index(); + + DF->Offset = List.Offset(); + DF->Size = List.Size(); + if (Cache.HeaderP->MaxDescFileSize < DF->Size) + Cache.HeaderP->MaxDescFileSize = DF->Size; + Cache.HeaderP->DescFileCount++; + + return true; +} + /*}}}*/ +// CacheGenerator::NewDescription - Create a new Description /*{{{*/ +// --------------------------------------------------------------------- +/* This puts a description structure in the linked list */ +map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, + const string &Lang, const MD5SumValue &md5sum, + map_ptrloc Next) +{ + // Get a structure + map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description)); + if (Description == 0) + return 0; + + // Fill it in + Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description); + Desc->NextDesc = Next; + Desc->ID = Cache.HeaderP->DescriptionCount++; + Desc->language_code = Map.WriteString(Lang); + Desc->md5sum = Map.WriteString(md5sum.Value()); + + return Description; +} + /*}}}*/ // ListParser::NewDepends - Create a dependency element /*{{{*/ // --------------------------------------------------------------------- /* This creates a dependency element in the tree. It is linked to the @@ -580,7 +685,7 @@ static bool CheckValidity(const string &CacheFile, FileIterator Start, pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache); if (File.end() == true) return false; - + Visited[File->ID] = true; } diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 9a729eea4..fae1a60a6 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -24,6 +24,7 @@ #endif #include <apt-pkg/pkgcache.h> +#include <apt-pkg/md5.h> class pkgSourceList; class OpProgress; @@ -55,7 +56,9 @@ class pkgCacheGenerator bool NewPackage(pkgCache::PkgIterator &Pkg,const string &Pkg); bool NewFileVer(pkgCache::VerIterator &Ver,ListParser &List); + bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); unsigned long NewVersion(pkgCache::VerIterator &Ver,const string &VerStr,unsigned long Next); + map_ptrloc NewDescription(pkgCache::DescIterator &Desc,const string &Lang,const MD5SumValue &md5sum,map_ptrloc Next); public: @@ -108,6 +111,9 @@ class pkgCacheGenerator::ListParser virtual string Package() = 0; virtual string Version() = 0; virtual bool NewVersion(pkgCache::VerIterator Ver) = 0; + virtual string Description() = 0; + virtual string DescriptionLanguage() = 0; + virtual MD5SumValue Description_md5() = 0; virtual unsigned short VersionHash() = 0; virtual bool UsePackage(pkgCache::PkgIterator Pkg, pkgCache::VerIterator Ver) = 0; diff --git a/apt-pkg/pkgrecords.cc b/apt-pkg/pkgrecords.cc index 9c2655d6a..b22f3e73f 100644 --- a/apt-pkg/pkgrecords.cc +++ b/apt-pkg/pkgrecords.cc @@ -63,3 +63,12 @@ pkgRecords::Parser &pkgRecords::Lookup(pkgCache::VerFileIterator const &Ver) return *Files[Ver.File()->ID]; } /*}}}*/ +// Records::Lookup - Get a parser for the package description file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgRecords::Parser &pkgRecords::Lookup(pkgCache::DescFileIterator const &Desc) +{ + Files[Desc.File()->ID]->Jump(Desc); + return *Files[Desc.File()->ID]; +} + /*}}}*/ diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h index 08f004414..31c444dbf 100644 --- a/apt-pkg/pkgrecords.h +++ b/apt-pkg/pkgrecords.h @@ -38,6 +38,7 @@ class pkgRecords // Lookup function Parser &Lookup(pkgCache::VerFileIterator const &Ver); + Parser &Lookup(pkgCache::DescFileIterator const &Desc); // Construct destruct pkgRecords(pkgCache &Cache); @@ -49,6 +50,7 @@ class pkgRecords::Parser protected: virtual bool Jump(pkgCache::VerFileIterator const &Ver) = 0; + virtual bool Jump(pkgCache::DescFileIterator const &Desc) = 0; public: friend class pkgRecords; diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index fc020436c..2eb269418 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -35,7 +35,7 @@ pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long Size) : Fd(*pFd), Size(Size) { - if (Fd.IsOpen() == false || Fd.Size() == 0) + if (Fd.IsOpen() == false || Fd.Size()) { Buffer = 0; Start = End = Buffer = 0; |