From 66905344357d03c206d99964a0d941b261f7146c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 22 Mar 2010 18:25:37 +0100 Subject: * ftparchive/writer.cc: - write LongDescriptions if they shouldn't be included in Packages file into i18n/Translation-en by default. It is ensured that each package+description is listed only ones in the Translation file even if we generate multiple Packages file in one run. The file is only generated in "generate" - the simple file commands can't create it by now. Also, the LongDescription is currently a global setting, so generating archives with and without LongDescriptions in the Packages file in the same run are currently not possible. --- ftparchive/apt-ftparchive.cc | 47 +++++++++++++++++++++++++++++++++------- ftparchive/writer.cc | 51 ++++++++++++++++++++++++++++++++++++++++++-- ftparchive/writer.h | 19 +++++++++++++++++ 3 files changed, 107 insertions(+), 10 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index f1a182e52..e69c88ddd 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -62,6 +62,9 @@ struct PackageMap string SrcOverride; string SrcExtraOverride; + // Translation master file + TranslationWriter *TransWriter; + // Contents string Contents; string ContentsHead; @@ -100,8 +103,9 @@ struct PackageMap vector::iterator End, unsigned long &Left); - PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false), - PkgDone(false), SrcDone(false), ContentsMTime(0) {}; + PackageMap() : TransWriter(NULL), DeLinkLimit(0), Permissions(1), + ContentsDone(false), PkgDone(false), SrcDone(false), + ContentsMTime(0) {}; }; /*}}}*/ @@ -169,6 +173,8 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) Packages.DirStrip = ArchiveDir; Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix); + Packages.TransWriter = TransWriter; + Packages.Stats.DeLinkBytes = Stats.DeLinkBytes; Packages.DeLinkLimit = DeLinkLimit; @@ -436,6 +442,8 @@ void LoadTree(vector &PkgList,Configuration &Setup) "$(DIST)/$(SECTION)/source/"); string DPkg = Setup.Find("TreeDefault::Packages", "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages"); + string DTrans = Setup.Find("TreeDefault::Translation", + "$(DIST)/$(SECTION)/i18n/Translation-en"); string DIPrfx = Setup.Find("TreeDefault::InternalPrefix", "$(DIST)/$(SECTION)/"); string DContents = Setup.Find("TreeDefault::Contents", @@ -461,15 +469,25 @@ void LoadTree(vector &PkgList,Configuration &Setup) string Section; while (ParseQuoteWord(Sections,Section) == true) { - string Tmp2 = Block.Find("Architectures"); string Arch; + struct SubstVar const Vars[] = {{"$(DIST)",&Dist}, + {"$(SECTION)",&Section}, + {"$(ARCH)",&Arch}, + {}}; + TranslationWriter *TransWriter; + if (DTrans.empty() == false) + { + string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"), + SubstVar(Block.Find("Translation", DTrans.c_str()), Vars)); + TransWriter = new TranslationWriter(TranslationFile); + } + else + TransWriter = NULL; + + string const Tmp2 = Block.Find("Architectures"); const char *Archs = Tmp2.c_str(); while (ParseQuoteWord(Archs,Arch) == true) { - struct SubstVar Vars[] = {{"$(DIST)",&Dist}, - {"$(SECTION)",&Section}, - {"$(ARCH)",&Arch}, - {}}; PackageMap Itm; Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars); @@ -491,6 +509,11 @@ void LoadTree(vector &PkgList,Configuration &Setup) Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars); Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars); Itm.Arch = Arch; + if (TransWriter != NULL) + { + TransWriter->IncreaseRefCounter(); + Itm.TransWriter = TransWriter; + } Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars); Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars); Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars); @@ -500,6 +523,9 @@ void LoadTree(vector &PkgList,Configuration &Setup) Itm.GetGeneral(Setup,Block); PkgList.push_back(Itm); } + // we didn't use this TransWriter, so we can release it + if (TransWriter != NULL && TransWriter->GetRefCounter() == 0) + delete TransWriter; } Top = Top->Next; @@ -788,7 +814,12 @@ bool Generate(CommandLine &CmdL) delete [] List; } - + + // close the Translation master files + for (vector::iterator I = PkgList.begin(); I != PkgList.end(); I++) + if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0) + delete I->TransWriter; + if (_config->FindB("APT::FTPArchive::Contents",true) == false) return true; diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index 9e5b7d4f3..b395903b7 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -300,7 +300,7 @@ bool FTWScanner::Delink(string &FileName,const char *OriginalPath, /* */ PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides, string const &Arch) : - FTWScanner(Arch), Db(DB), Stats(Db.Stats) + FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL) { Output = stdout; SetExts(".deb .udeb"); @@ -317,7 +317,7 @@ PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string c if (Db.Loaded() == false) DoContents = false; - + // Read the override file if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false) return; @@ -448,6 +448,8 @@ bool PackagesWriter::DoPackage(string FileName) descmd5.Add(desc.c_str()); DescriptionMd5 = descmd5.Result().Value(); SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str()); + if (TransWriter != NULL) + TransWriter->DoPackage(Package, desc, DescriptionMd5); } // Rewrite the maintainer field if necessary @@ -494,6 +496,51 @@ bool PackagesWriter::DoPackage(string FileName) } /*}}}*/ +// TranslationWriter::TranslationWriter - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Create a Translation-Master file for this Packages file */ +TranslationWriter::TranslationWriter(string const &File) : Output(NULL), + RefCounter(0) +{ + if (File.empty() == true) + return; + + Output = fopen(File.c_str(), "w"); +} + /*}}}*/ +// TranslationWriter::DoPackage - Process a single package /*{{{*/ +// --------------------------------------------------------------------- +/* Create a Translation-Master file for this Packages file */ +bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc, + string const &MD5) +{ + if (Output == NULL) + return true; + + // Different archs can include different versions and therefore + // different descriptions - so we need to check for both name and md5. + string const Record = Pkg + ":" + MD5; + + if (Included.find(Record) != Included.end()) + return true; + + fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n", + Pkg.c_str(), MD5.c_str(), Desc.c_str()); + + Included.insert(Record); + return true; +} + /*}}}*/ +// TranslationWriter::~TranslationWriter - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +TranslationWriter::~TranslationWriter() +{ + if (Output != NULL) + fclose(Output); +} + /*}}}*/ + // SourcesWriter::SourcesWriter - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/ftparchive/writer.h b/ftparchive/writer.h index af7ba4edd..2afd1af1f 100644 --- a/ftparchive/writer.h +++ b/ftparchive/writer.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "cachedb.h" #include "override.h" @@ -72,6 +73,23 @@ class FTWScanner FTWScanner(string const &Arch = string()); }; +class TranslationWriter +{ + FILE *Output; + std::set Included; + unsigned short RefCounter; + + public: + void IncreaseRefCounter() { ++RefCounter; }; + unsigned short DecreaseRefCounter() { return (RefCounter == 0) ? 0 : --RefCounter; }; + unsigned short GetRefCounter() const { return RefCounter; }; + bool DoPackage(string const &Pkg, string const &Desc, string const &MD5); + + TranslationWriter(string const &File); + TranslationWriter() : Output(NULL), RefCounter(0) {}; + ~TranslationWriter(); +}; + class PackagesWriter : public FTWScanner { Override Over; @@ -93,6 +111,7 @@ class PackagesWriter : public FTWScanner string DirStrip; FILE *Output; struct CacheDB::Stats &Stats; + TranslationWriter *TransWriter; inline bool ReadOverride(string const &File) {return Over.ReadOverride(File);}; inline bool ReadExtraOverride(string const &File) -- cgit v1.2.3 From 4e794c509becfd7e2bddfddc1205dc81397a48bd Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 26 Mar 2010 19:18:21 +0100 Subject: Inclusion of Long Descriptions in the Packages files can be set now also in TreeDefaults and Tree to support generation of archives which should support and which shouldn't support splitted out Translation-en files in the same run. --- ftparchive/apt-ftparchive.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index e69c88ddd..f3e91d90d 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -63,6 +63,7 @@ struct PackageMap string SrcExtraOverride; // Translation master file + bool LongDesc; TranslationWriter *TransWriter; // Contents @@ -103,7 +104,7 @@ struct PackageMap vector::iterator End, unsigned long &Left); - PackageMap() : TransWriter(NULL), DeLinkLimit(0), Permissions(1), + PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1), ContentsDone(false), PkgDone(false), SrcDone(false), ContentsMTime(0) {}; }; @@ -174,6 +175,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix); Packages.TransWriter = TransWriter; + Packages.LongDescription = LongDesc; Packages.Stats.DeLinkBytes = Stats.DeLinkBytes; Packages.DeLinkLimit = DeLinkLimit; @@ -456,6 +458,9 @@ void LoadTree(vector &PkgList,Configuration &Setup) string DFLFile = Setup.Find("TreeDefault::FileList", ""); string DSFLFile = Setup.Find("TreeDefault::SourceFileList", ""); + bool const LongDescription = Setup.FindB("TreeDefault::LongDescription", + _config->FindB("APT::FTPArchive::LongDescription", true)); + // Process 'tree' type sections const Configuration::Item *Top = Setup.Tree("tree"); for (Top = (Top == 0?0:Top->Child); Top != 0;) @@ -474,8 +479,9 @@ void LoadTree(vector &PkgList,Configuration &Setup) {"$(SECTION)",&Section}, {"$(ARCH)",&Arch}, {}}; + bool const LongDesc = Block.FindB("LongDescription", LongDescription); TranslationWriter *TransWriter; - if (DTrans.empty() == false) + if (DTrans.empty() == false && LongDesc == false) { string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"), SubstVar(Block.Find("Translation", DTrans.c_str()), Vars)); @@ -509,6 +515,7 @@ void LoadTree(vector &PkgList,Configuration &Setup) Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars); Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars); Itm.Arch = Arch; + Itm.LongDesc = LongDesc; if (TransWriter != NULL) { TransWriter->IncreaseRefCounter(); -- cgit v1.2.3 From 34f1d96cf5657b5e34cd9880dccfa2028fa16b13 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 26 Mar 2010 22:38:50 +0100 Subject: Switch the TranslationWriter to use MultiCompress to be able to generate the compressed files as we want them and to prevent the file to be replaced without a reason which could save us from steady redownloads of a file with the same content. --- ftparchive/apt-ftparchive.cc | 13 ++++++++----- ftparchive/writer.cc | 12 ++++++++---- ftparchive/writer.h | 6 ++++-- 3 files changed, 20 insertions(+), 11 deletions(-) (limited to 'ftparchive') diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index f3e91d90d..46831b385 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -134,8 +134,6 @@ void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block) PkgExt = Block.Find("Packages::Extensions", Setup.Find("Default::Packages::Extensions",".deb").c_str()); - Permissions = Setup.FindI("Default::FileMode",0644); - if (FLFile.empty() == false) FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile); @@ -458,8 +456,11 @@ void LoadTree(vector &PkgList,Configuration &Setup) string DFLFile = Setup.Find("TreeDefault::FileList", ""); string DSFLFile = Setup.Find("TreeDefault::SourceFileList", ""); - bool const LongDescription = Setup.FindB("TreeDefault::LongDescription", + int const Permissions = Setup.FindI("Default::FileMode",0644); + + bool const LongDescription = Setup.FindB("Default::LongDescription", _config->FindB("APT::FTPArchive::LongDescription", true)); + string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str(); // Process 'tree' type sections const Configuration::Item *Top = Setup.Tree("tree"); @@ -479,13 +480,15 @@ void LoadTree(vector &PkgList,Configuration &Setup) {"$(SECTION)",&Section}, {"$(ARCH)",&Arch}, {}}; + mode_t const Perms = Block.FindI("FileMode", Permissions); bool const LongDesc = Block.FindB("LongDescription", LongDescription); TranslationWriter *TransWriter; if (DTrans.empty() == false && LongDesc == false) { string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"), SubstVar(Block.Find("Translation", DTrans.c_str()), Vars)); - TransWriter = new TranslationWriter(TranslationFile); + string const TransCompress = Block.Find("Translation::Compress", TranslationCompress); + TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms); } else TransWriter = NULL; @@ -495,7 +498,7 @@ void LoadTree(vector &PkgList,Configuration &Setup) while (ParseQuoteWord(Archs,Arch) == true) { PackageMap Itm; - + Itm.Permissions = Perms; Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars); Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars); diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index b395903b7..45a8d212b 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -499,13 +499,15 @@ bool PackagesWriter::DoPackage(string FileName) // TranslationWriter::TranslationWriter - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Create a Translation-Master file for this Packages file */ -TranslationWriter::TranslationWriter(string const &File) : Output(NULL), +TranslationWriter::TranslationWriter(string const &File, string const &TransCompress, + mode_t const &Permissions) : Output(NULL), RefCounter(0) { if (File.empty() == true) return; - Output = fopen(File.c_str(), "w"); + Comp = new MultiCompress(File, TransCompress, Permissions); + Output = Comp->Input; } /*}}}*/ // TranslationWriter::DoPackage - Process a single package /*{{{*/ @@ -536,8 +538,10 @@ bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc, /* */ TranslationWriter::~TranslationWriter() { - if (Output != NULL) - fclose(Output); + if (Comp == NULL) + return; + + delete Comp; } /*}}}*/ diff --git a/ftparchive/writer.h b/ftparchive/writer.h index 2afd1af1f..3123a7f46 100644 --- a/ftparchive/writer.h +++ b/ftparchive/writer.h @@ -22,6 +22,7 @@ #include #include "cachedb.h" +#include "multicompress.h" #include "override.h" #include "apt-ftparchive.h" @@ -75,6 +76,7 @@ class FTWScanner class TranslationWriter { + MultiCompress *Comp; FILE *Output; std::set Included; unsigned short RefCounter; @@ -85,8 +87,8 @@ class TranslationWriter unsigned short GetRefCounter() const { return RefCounter; }; bool DoPackage(string const &Pkg, string const &Desc, string const &MD5); - TranslationWriter(string const &File); - TranslationWriter() : Output(NULL), RefCounter(0) {}; + TranslationWriter(string const &File, string const &TransCompress, mode_t const &Permissions); + TranslationWriter() : Comp(NULL), Output(NULL), RefCounter(0) {}; ~TranslationWriter(); }; -- cgit v1.2.3