From 2295de2c97e6d1290a86e0b160885212411510a5 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Sep 2018 19:45:42 +0200 Subject: Process status-fd completely before finishing dpkg call Exiting the processing loop as soon as the dpkg process finishes might leave status-fd lines unprocessed which wasn't much of a problem in the past as the progress would just be slightly off, but now that we us the information also for skipping already done tasks and generate warnings if we didn't see all expected messages we should make sure we seem them all. We still need to exit "early" if dpkg exited unsuccessfully/crashed through as the (remaining) status lines we get could be incomplete. --- apt-pkg/deb/dpkgpm.cc | 51 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 15 deletions(-) (limited to 'apt-pkg/deb/dpkgpm.cc') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 790019b98..68674fa02 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -102,8 +102,8 @@ APT_PURE static unsigned int EnvironmentSize() /*{{{*/ class pkgDPkgPMPrivate /*{{{*/ { public: - pkgDPkgPMPrivate() : stdin_is_dev_null(false), dpkgbuf_pos(0), - term_out(NULL), history_out(NULL), + pkgDPkgPMPrivate() : stdin_is_dev_null(false), status_fd_reached_end_of_file(false), + dpkgbuf_pos(0), term_out(NULL), history_out(NULL), progress(NULL), tt_is_valid(false), master(-1), slave(NULL), protect_slave_from_dying(-1), direct_stdin(false) @@ -114,6 +114,7 @@ public: { } bool stdin_is_dev_null; + bool status_fd_reached_end_of_file; // the buffer we use for the dpkg status-fd reading char dpkgbuf[1024]; size_t dpkgbuf_pos; @@ -958,11 +959,19 @@ void pkgDPkgPM::handleCrossUpgradeAction(string const &pkgname) /*{{{*/ // DPkgPM::DoDpkgStatusFd /*{{{*/ void pkgDPkgPM::DoDpkgStatusFd(int statusfd) { - ssize_t const len = read(statusfd, &d->dpkgbuf[d->dpkgbuf_pos], - (sizeof(d->dpkgbuf)/sizeof(d->dpkgbuf[0])) - d->dpkgbuf_pos); - if(len <= 0) - return; - d->dpkgbuf_pos += (len / sizeof(d->dpkgbuf[0])); + auto const remainingBuffer = (sizeof(d->dpkgbuf) / sizeof(d->dpkgbuf[0])) - d->dpkgbuf_pos; + if (likely(remainingBuffer > 0) && d->status_fd_reached_end_of_file == false) + { + auto const len = read(statusfd, &d->dpkgbuf[d->dpkgbuf_pos], remainingBuffer); + if (len < 0) + return; + else if (len == 0 && d->dpkgbuf_pos == 0) + { + d->status_fd_reached_end_of_file = true; + return; + } + d->dpkgbuf_pos += (len / sizeof(d->dpkgbuf[0])); + } // process line by line from the buffer char *p = d->dpkgbuf, *q = nullptr; @@ -2022,6 +2031,7 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) // we read from dpkg here int const _dpkgin = fd[0]; close(fd[1]); // close the write end of the pipe + d->status_fd_reached_end_of_file = false; // apply ionice if (_config->FindB("DPkg::UseIoNice", false) == true) @@ -2041,14 +2051,24 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) int Status = 0; int res; bool waitpid_failure = false; - while ((res=waitpid(Child,&Status, WNOHANG)) != Child) { - if(res < 0) { - // error handling, waitpid returned -1 - if (errno == EINTR) - continue; - waitpid_failure = true; - break; + bool dpkg_finished = false; + do + { + if (dpkg_finished == false) + { + if ((res = waitpid(Child, &Status, WNOHANG)) == Child) + dpkg_finished = true; + else if (res < 0) + { + // error handling, waitpid returned -1 + if (errno == EINTR) + continue; + waitpid_failure = true; + break; + } } + if (dpkg_finished && d->status_fd_reached_end_of_file) + break; // wait for input or output here FD_ZERO(&rfds); @@ -2078,7 +2098,8 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) DoStdin(d->master); if(FD_ISSET(_dpkgin, &rfds)) DoDpkgStatusFd(_dpkgin); - } + + } while (true); close(_dpkgin); // Restore sig int/quit -- cgit v1.2.3 From f484906eff85c8c9eadc13ce817d8857abdd08e8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 10 Sep 2018 18:43:34 +0200 Subject: Don't expect duplicated dpkg status-fd messages The progress reporting relies on parsing the status reports of dpkg which used to repeat being in the same state multiple times in the same run, but by fixing #365921 it will stop doing so. The problem is in theory just with 'config-files' in case we do purge as this (can) do remove + purge in one step, but we remove this also for the unpack + configure combination althrough we handle these currently in two independent dpkg calls. --- apt-pkg/deb/dpkgpm.cc | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) (limited to 'apt-pkg/deb/dpkgpm.cc') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 68674fa02..da2d5e3ae 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1154,16 +1154,14 @@ void pkgDPkgPM::BuildPackagesProgressMap() { // map the dpkg states to the operations that are performed // (this is sorted in the same way as Item::Ops) - static const std::array, 4> DpkgStatesOpMap = {{ + static const std::array, 4> DpkgStatesOpMap = {{ // Install operation {{ {"half-installed", N_("Preparing %s")}, {"unpacked", N_("Unpacking %s") }, - {nullptr, nullptr} }}, // Configure operation {{ - {"unpacked",N_("Preparing to configure %s") }, {"half-configured", N_("Configuring %s") }, { "installed", N_("Installed %s")}, }}, @@ -1171,13 +1169,11 @@ void pkgDPkgPM::BuildPackagesProgressMap() {{ {"half-configured", N_("Preparing for removal of %s")}, {"half-installed", N_("Removing %s")}, - {"config-files", N_("Removed %s")}, }}, // Purge operation {{ {"config-files", N_("Preparing to completely remove %s")}, {"not-installed", N_("Completely removed %s")}, - {nullptr, nullptr} }}, }}; static_assert(Item::Purge == 3, "Enum item has unexpected index for mapping array"); @@ -1193,21 +1189,16 @@ void pkgDPkgPM::BuildPackagesProgressMap() string const name = I.Pkg.FullName(); PackageOpsDone[name] = 0; - auto AddToPackageOps = std::back_inserter(PackageOps[name]); - if (I.Op == Item::Purge && I.Pkg->CurrentVer != 0) - { - // purging a package which is installed first passes through remove states - auto const DpkgOps = DpkgStatesOpMap[Item::Remove]; - std::copy(DpkgOps.begin(), DpkgOps.end(), AddToPackageOps); + auto AddToPackageOps = [&](decltype(I.Op) const Op) { + auto const DpkgOps = DpkgStatesOpMap[Op]; + std::copy(DpkgOps.begin(), DpkgOps.end(), std::back_inserter(PackageOps[name])); PackagesTotal += DpkgOps.size(); - } - auto const DpkgOps = DpkgStatesOpMap[I.Op]; - std::copy_if(DpkgOps.begin(), DpkgOps.end(), AddToPackageOps, [&](DpkgState const &state) { - if (state.state == nullptr) - return false; - ++PackagesTotal; - return true; - }); + }; + // purging a package which is installed first passes through remove states + if (I.Op == Item::Purge && I.Pkg->CurrentVer != 0) + AddToPackageOps(Item::Remove); + AddToPackageOps(I.Op); + if ((I.Op == Item::Remove || I.Op == Item::Purge) && I.Pkg->CurrentVer != 0) { if (I.Pkg->CurrentState == pkgCache::State::UnPacked || @@ -1631,9 +1622,9 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) approvedStates.Remove(*Ver); Purges.erase(Ver); auto && RemOp = PackageOps[C.first->Pkg.FullName()]; - if (RemOp.size() == 5) + if (RemOp.size() == 4) { - RemOp.erase(std::next(RemOp.begin(), 3), RemOp.end()); + RemOp.erase(std::next(RemOp.begin(), 2), RemOp.end()); PackagesTotal -= 2; } else -- cgit v1.2.3 From 329c8d5e149465003ea9007661a7af1099c75c43 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 10 Sep 2018 21:34:11 +0200 Subject: Reorder progress report messages We are seeing 'processing' messages from dpkg first, so it makes sense to translate them to "Preparing" messages instead of using "Installing" and co to override these shortly after with the "Preparing" messages. The difference isn't all to visible as later messages tend to linger far longer in the display than the ealier ones, but at least in a listing it seems more logical. --- apt-pkg/deb/dpkgpm.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'apt-pkg/deb/dpkgpm.cc') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index da2d5e3ae..f83b368cd 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -142,12 +142,12 @@ namespace // Maps the dpkg "processing" info to human readable names. Entry 0 // of each array is the key, entry 1 is the value. const std::pair PackageProcessingOps[] = { - std::make_pair("install", N_("Installing %s")), + std::make_pair("install", N_("Preparing %s")), // we don't care for the difference - std::make_pair("upgrade", N_("Installing %s")), - std::make_pair("configure", N_("Configuring %s")), - std::make_pair("remove", N_("Removing %s")), - std::make_pair("purge", N_("Completely removing %s")), + std::make_pair("upgrade", N_("Preparing %s")), + std::make_pair("configure", N_("Preparing to configure %s")), + std::make_pair("remove", N_("Preparing for removal of %s")), + std::make_pair("purge", N_("Preparing to completely remove %s")), std::make_pair("disappear", N_("Noting disappearance of %s")), std::make_pair("trigproc", N_("Running post-installation trigger %s")) }; @@ -1157,8 +1157,8 @@ void pkgDPkgPM::BuildPackagesProgressMap() static const std::array, 4> DpkgStatesOpMap = {{ // Install operation {{ - {"half-installed", N_("Preparing %s")}, - {"unpacked", N_("Unpacking %s") }, + {"half-installed", N_("Unpacking %s")}, + {"unpacked", N_("Installing %s") }, }}, // Configure operation {{ @@ -1167,12 +1167,12 @@ void pkgDPkgPM::BuildPackagesProgressMap() }}, // Remove operation {{ - {"half-configured", N_("Preparing for removal of %s")}, + {"half-configured", N_("Removing %s")}, {"half-installed", N_("Removing %s")}, }}, // Purge operation {{ - {"config-files", N_("Preparing to completely remove %s")}, + {"config-files", N_("Completely removing %s")}, {"not-installed", N_("Completely removed %s")}, }}, }}; -- cgit v1.2.3