From 814ffcfaf34ad1d35e397eeaaafefbf03faed9cf Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 7 Jan 2020 19:21:40 +0100 Subject: hashes: Use Libgcrypt for hashing purposes Switch the code of the Hashes class to use libgcrypt, which allows us to use hardware-accelerated implementations of SHA1 and friends. --- apt-pkg/contrib/hashes.cc | 106 ++++++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 36 deletions(-) (limited to 'apt-pkg/contrib/hashes.cc') diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index 366133b02..d68919778 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -15,18 +15,30 @@ #include #include #include -#include -#include -#include #include #include #include +#include #include #include #include + +#include /*}}}*/ +static const constexpr struct HashAlgo +{ + const char *name; + int gcryAlgo; + Hashes::SupportedHashes ourAlgo; +} Algorithms[] = { + {"MD5Sum", GCRY_MD_MD5, Hashes::MD5SUM}, + {"SHA1", GCRY_MD_SHA1, Hashes::SHA1SUM}, + {"SHA256", GCRY_MD_SHA256, Hashes::SHA256SUM}, + {"SHA512", GCRY_MD_SHA512, Hashes::SHA512SUM}, +}; + const char * HashString::_SupportedHashes[] = { "SHA512", "SHA256", "SHA1", "MD5Sum", "Checksum-FileSize", NULL @@ -286,39 +298,41 @@ bool HashStringList::operator!=(HashStringList const &other) const class PrivateHashes { public: unsigned long long FileSize; - unsigned int CalcHashes; + gcry_md_hd_t hd; + + explicit PrivateHashes(unsigned int const CalcHashes) : FileSize(0) + { + gcry_md_open(&hd, 0, 0); + for (auto & Algo : Algorithms) + { + if ((CalcHashes & Algo.ourAlgo) == Algo.ourAlgo) + gcry_md_enable(hd, Algo.gcryAlgo); + } + } - explicit PrivateHashes(unsigned int const CalcHashes) : FileSize(0), CalcHashes(CalcHashes) {} explicit PrivateHashes(HashStringList const &Hashes) : FileSize(0) { - unsigned int calcHashes = Hashes.usable() ? 0 : ~0; - if (Hashes.find("MD5Sum") != NULL) - calcHashes |= Hashes::MD5SUM; - if (Hashes.find("SHA1") != NULL) - calcHashes |= Hashes::SHA1SUM; - if (Hashes.find("SHA256") != NULL) - calcHashes |= Hashes::SHA256SUM; - if (Hashes.find("SHA512") != NULL) - calcHashes |= Hashes::SHA512SUM; - CalcHashes = calcHashes; + gcry_md_open(&hd, 0, 0); + for (auto & Algo : Algorithms) + { + if (not Hashes.usable() || Hashes.find(Algo.name) != NULL) + gcry_md_enable(hd, Algo.gcryAlgo); + } + } + ~PrivateHashes() + { + gcry_md_close(hd); } }; /*}}}*/ // Hashes::Add* - Add the contents of data or FD /*{{{*/ bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size) { - if (Size == 0) - return true; - bool Res = true; - if ((d->CalcHashes & MD5SUM) == MD5SUM) - Res &= MD5.Add(Data, Size); - if ((d->CalcHashes & SHA1SUM) == SHA1SUM) - Res &= SHA1.Add(Data, Size); - if ((d->CalcHashes & SHA256SUM) == SHA256SUM) - Res &= SHA256.Add(Data, Size); - if ((d->CalcHashes & SHA512SUM) == SHA512SUM) - Res &= SHA512.Add(Data, Size); - d->FileSize += Size; - return Res; + if (Size != 0) + { + gcry_md_write(d->hd, Data, Size); + d->FileSize += Size; + } + return true; } bool Hashes::AddFD(int const Fd,unsigned long long Size) { @@ -364,18 +378,38 @@ bool Hashes::AddFD(FileFd &Fd,unsigned long long Size) return true; } /*}}}*/ + +static APT_PURE std::string HexDigest(gcry_md_hd_t hd, int algo) +{ + char Conv[16] = + {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', + 'c', 'd', 'e', 'f'}; + + auto Size = gcry_md_get_algo_dlen(algo); + char Result[((Size)*2) + 1]; + Result[(Size)*2] = 0; + + auto Sum = gcry_md_read(hd, algo); + + // Convert each char into two letters + size_t J = 0; + size_t I = 0; + for (; I != (Size)*2; J++, I += 2) + { + Result[I] = Conv[Sum[J] >> 4]; + Result[I + 1] = Conv[Sum[J] & 0xF]; + } + return std::string(Result); +}; + HashStringList Hashes::GetHashStringList() { HashStringList hashes; - if ((d->CalcHashes & MD5SUM) == MD5SUM) - hashes.push_back(HashString("MD5Sum", MD5.Result().Value())); - if ((d->CalcHashes & SHA1SUM) == SHA1SUM) - hashes.push_back(HashString("SHA1", SHA1.Result().Value())); - if ((d->CalcHashes & SHA256SUM) == SHA256SUM) - hashes.push_back(HashString("SHA256", SHA256.Result().Value())); - if ((d->CalcHashes & SHA512SUM) == SHA512SUM) - hashes.push_back(HashString("SHA512", SHA512.Result().Value())); + for (auto & Algo : Algorithms) + if (gcry_md_is_enabled(d->hd, Algo.gcryAlgo)) + hashes.push_back(HashString(Algo.name, HexDigest(d->hd, Algo.gcryAlgo))); hashes.FileSize(d->FileSize); + return hashes; } Hashes::Hashes() : d(new PrivateHashes(~0)) { } -- cgit v1.2.3 From b350560e34a369ef7610f9fceeffb00660209390 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 7 Jan 2020 19:55:48 +0100 Subject: Raise buffer size for Hashes::AddFD() from 4 KiB to 64 KiB Move APT_BUFFER_SIZE to macros.h and re-use it in hashes, this also might speed up stuff, the motivation for using 64 KiB buffers in fileutl.cc was precisely that after all. --- apt-pkg/contrib/hashes.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'apt-pkg/contrib/hashes.cc') diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index d68919778..8ddaaf549 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -336,7 +337,7 @@ bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size } bool Hashes::AddFD(int const Fd,unsigned long long Size) { - unsigned char Buf[64*64]; + unsigned char Buf[APT_BUFFER_SIZE]; bool const ToEOF = (Size == UntilEOF); while (Size != 0 || ToEOF) { @@ -355,7 +356,7 @@ bool Hashes::AddFD(int const Fd,unsigned long long Size) } bool Hashes::AddFD(FileFd &Fd,unsigned long long Size) { - unsigned char Buf[64*64]; + unsigned char Buf[APT_BUFFER_SIZE]; bool const ToEOF = (Size == 0); while (Size != 0 || ToEOF) { -- cgit v1.2.3 From 79de3008ebfc6b4a5dd32e9de1d19788da0b885d Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 7 Jan 2020 20:36:53 +0100 Subject: Convert users of {MD5,SHA1,SHA256,SHA512}Summation to use Hashes This makes use of the a function GetHashString() that returns the specific hash string. We also need to implement another overload of Add() for signed chars with sizes, so the existing users do not require reinterpret_cast everywhere. --- apt-pkg/contrib/hashes.cc | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'apt-pkg/contrib/hashes.cc') diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index 8ddaaf549..d506a1361 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -102,27 +102,27 @@ std::string HashString::GetHashForFile(std::string filename) const /*{{{*/ FileFd Fd(filename, FileFd::ReadOnly); if(strcasecmp(Type.c_str(), "MD5Sum") == 0) { - MD5Summation MD5; + Hashes MD5(Hashes::MD5SUM); MD5.AddFD(Fd); - fileHash = (std::string)MD5.Result(); + fileHash = MD5.GetHashString(Hashes::MD5SUM).Hash; } else if (strcasecmp(Type.c_str(), "SHA1") == 0) { - SHA1Summation SHA1; + Hashes SHA1(Hashes::SHA1SUM); SHA1.AddFD(Fd); - fileHash = (std::string)SHA1.Result(); + fileHash = SHA1.GetHashString(Hashes::SHA1SUM).Hash; } else if (strcasecmp(Type.c_str(), "SHA256") == 0) { - SHA256Summation SHA256; + Hashes SHA256(Hashes::SHA256SUM); SHA256.AddFD(Fd); - fileHash = (std::string)SHA256.Result(); + fileHash = SHA256.GetHashString(Hashes::SHA256SUM).Hash; } else if (strcasecmp(Type.c_str(), "SHA512") == 0) { - SHA512Summation SHA512; + Hashes SHA512(Hashes::SHA512SUM); SHA512.AddFD(Fd); - fileHash = (std::string)SHA512.Result(); + fileHash = SHA512.GetHashString(Hashes::SHA512SUM).Hash; } else if (strcasecmp(Type.c_str(), "Checksum-FileSize") == 0) strprintf(fileHash, "%llu", Fd.FileSize()); @@ -413,6 +413,15 @@ HashStringList Hashes::GetHashStringList() return hashes; } + +HashString Hashes::GetHashString(SupportedHashes hash) +{ + for (auto & Algo : Algorithms) + if (hash == Algo.ourAlgo) + return HashString(Algo.name, HexDigest(d->hd, Algo.gcryAlgo)); + + abort(); +} Hashes::Hashes() : d(new PrivateHashes(~0)) { } Hashes::Hashes(unsigned int const Hashes) : d(new PrivateHashes(Hashes)) {} Hashes::Hashes(HashStringList const &Hashes) : d(new PrivateHashes(Hashes)) {} -- cgit v1.2.3