Merge branch 'feature/authconf'

6 files changed, 364 insertions, 32 deletions

diff --git a/test/integration/test-apt-get-update-sourceslist-warning b/test/integration/test-apt-get-update-sourceslist-warning
new file mode 100755
index 000000000..a99356b8b
--- /dev/null
+++ b/test/integration/test-apt-get-update-sourceslist-warning
@@ -0,0 +1,45 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture 'amd64'
+setupaptarchive --no-update
+
+testsuccess apt update
+testsuccess apt update --no-download
+
+echo 'deb ftp://ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list
+cat > rootdir/var/lib/apt/lists/ftp.tlh.debian.org_debian_dists_zurg_Release <<EOF
+Origin: Debian
+Label: Debian
+Suite: unreleased
+Codename: zurg
+Date: Fri, 14 Jul 2017 11:34:35 +0000
+Architectures: amd64
+Components: main
+Description: Debian x.y Zurg - Not Released
+EOF
+chmod 644 rootdir/var/lib/apt/lists/ftp.tlh.debian.org_debian_dists_zurg_Release
+
+testwarningequal "Reading package lists...
+Building dependency tree...
+All packages are up to date.
+W: Debian shuts down public FTP services currently still used in your sources.list(5) as 'ftp://ftp.tlh.debian.org/debian/'.
+   See press release https://debian.org/News/2017/20170425 for details." apt update --no-download
+
+
+echo 'deb http://apt:debian@ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list
+testsuccessequal "Reading package lists...
+Building dependency tree...
+All packages are up to date.
+N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'http://ftp.tlh.debian.org/debian'" apt update --no-download
+
+
+echo 'deb tor+https://apt:debian@ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list
+testsuccessequal "Reading package lists...
+Building dependency tree...
+All packages are up to date.
+N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'tor+https://ftp.tlh.debian.org/debian'" apt update --no-download
diff --git a/test/integration/test-authentication-basic b/test/integration/test-authentication-basic
index 3bfd076ce..011f205af 100755
--- a/test/integration/test-authentication-basic
+++ b/test/integration/test-authentication-basic
@@ -13,6 +13,7 @@ setupaptarchive --no-update
 changetohttpswebserver --authorization="$(printf '%s' 'star@irc:hunter2' | base64 )"
 
 echo 'See, when YOU type hunter2, it shows to us as *******' > aptarchive/bash
+echo 'Debug::Acquire::netrc "true";' > rootdir/etc/apt/apt.conf.d/netrcdebug.conf
 
 testauthfailure() {
 	testfailure apthelper download-file "${1}/bash" ./downloaded/bash
@@ -37,7 +38,11 @@ testauthsuccess() {
 	fi
 
 	rm -rf rootdir/var/lib/apt/lists
-	testsuccess aptget update
+	if expr index "$1" '@' >/dev/null; then
+		testsuccesswithnotice aptget update
+	else
+		testsuccess aptget update
+	fi
 	testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
@@ -94,7 +99,9 @@ rewritesourceslist "http://localhost:${APTHTTPPORT}"
 
 msgmsg 'proxy to server basic auth'
 webserverconfig 'aptwebserver::request::absolute' 'uri'
-export http_proxy="http://localhost:${APTHTTPPORT}"
+# using ip instead of localhost avoids picking up the auth for the repo
+# for the proxy as well as we serve them both over the same server…
+export http_proxy="http://127.0.0.1:${APTHTTPPORT}"
 runtest "http://localhost:${APTHTTPPORT}"
 unset http_proxy
 
diff --git a/test/integration/test-bug-818628-unreadable-source b/test/integration/test-bug-818628-unreadable-source
index cddc79398..59051fb5f 100755
--- a/test/integration/test-bug-818628-unreadable-source
+++ b/test/integration/test-bug-818628-unreadable-source
@@ -16,6 +16,8 @@ insertpackage 'unstable' 'foo' 'amd64' '2'
 
 setupaptarchive --no-update
 
+touch rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list
+touch rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.sources
 touch rootdir/etc/apt/apt.conf.d/unreadable.conf
 touch rootdir/etc/apt/preferences.d/unreadable.pref
 
@@ -31,56 +33,61 @@ foo/unstable 2 amd64 [upgradable from: 1]
 N: There is 1 additional version. Please use the '-a' switch to see it" apt list --upgradable
 
 runthemall() {
-	local ERR1="$1"
-	local ERR2="$1$2"
-	testfailureequal "$ERR1" aptcache policy
-	testfailureequal "$ERR1" aptcache policy foo
-	testfailureequal "$ERR2" aptcache depends foo
-	testfailureequal "$ERR2" aptcache rdepends foo
-	testfailureequal "$ERR2" aptcache search foo
-	testfailureequal "$ERR1" apt policy
-	testfailureequal "$ERR1" apt policy foo
-	testfailureequal "$ERR2" apt depends foo
-	testfailureequal "$ERR2" apt rdepends foo
-	testfailureequal "$ERR2" apt search foo
-	testfailureequal "$ERR2" apt list --upgradable
-	testfailureequal "$ERR2" apt show foo
-	testfailureequal "$ERR2" aptcache show foo --no-all-versions
-	testfailureequal "$ERR2" aptmark auto foo
-	testfailureequal "$ERR2" aptmark manual foo
-	testfailureequal "$ERR2" aptmark auto foo
+	local ERR="$1"
+	local ERRNOTICEVER="$1${2-
+N: There is 1 additional version. Please use the '-a' switch to see it}"
+	local ERRNOTICEREC="$1${2-
+N: There is 1 additional record. Please use the '-a' switch to see it}"
+	testwarningmsg "$ERR" aptcache policy
+	testwarningmsg "$ERR" aptcache policy foo
+	testwarningmsg "$ERR" aptcache depends foo
+	testwarningmsg "$ERR" aptcache rdepends foo
+	testwarningmsg "$ERR" aptcache search foo
+	testwarningmsg "$ERR" apt policy
+	testwarningmsg "$ERR" apt policy foo
+	testwarningmsg "$ERR" apt depends foo
+	testwarningmsg "$ERR" apt rdepends foo
+	testwarningmsg "$ERR" apt search foo
+	testwarningmsg "$ERRNOTICEVER" apt list --upgradable
+	testwarningmsg "$ERRNOTICEREC" apt show foo
+	testwarningmsg "$ERRNOTICEREC" aptcache show foo --no-all-versions
+	testwarningmsg "$ERR" aptmark auto foo
+	testwarningmsg "$ERR" aptmark manual foo
+	testwarningmsg "$ERR" aptmark auto foo
 }
 echo 'Apt::Cmd::Disable-Script-Warning "true";' >> aptconfig.conf
 
-msgmsg 'Unreadable sources file'
+msgmsg 'Unreadable one-line-style sources file'
 chmod -r rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list
-runthemall "E: Opening $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list - ifstream::ifstream (13: Permission denied)
-E: The list of sources could not be read."
+runthemall "W: Unable to read $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list - open (13: Permission denied)"
 chmod +r rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.list
 
+msgmsg 'Unreadable deb822-style sources file'
+chmod -r rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.sources
+runthemall "W: Unable to read $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.sources - open (13: Permission denied)"
+chmod +r rootdir/etc/apt/sources.list.d/apt-test-unstable-deb-src.sources
+
 msgmsg 'Unreadable config file'
 chmod -r rootdir/etc/apt/apt.conf.d/unreadable.conf
-runthemall "E: Opening configuration file ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/unreadable.conf - ifstream::ifstream (13: Permission denied)"
+runthemall "W: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/unreadable.conf - open (13: Permission denied)"
 chmod +r rootdir/etc/apt/apt.conf.d/unreadable.conf
 
 msgmsg 'Unreadable preferences file'
 chmod -r rootdir/etc/apt/preferences.d/unreadable.pref
-runthemall "E: Could not open file ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/preferences.d/unreadable.pref - open (13: Permission denied)"
+runthemall "W: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/preferences.d/unreadable.pref - open (13: Permission denied)"
 chmod +r rootdir/etc/apt/preferences.d/unreadable.pref
 
 msgmsg 'Unreadable sources directory'
 chmod -r rootdir/etc/apt/sources.list.d
-runthemall "E: Unable to read $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/ - opendir (13: Permission denied)" "
-W: You may want to run apt-get update to correct these problems
-E: The package cache file is corrupted"
+runthemall "W: Unable to read $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list.d/ - opendir (13: Permission denied)" ""
 chmod +r rootdir/etc/apt/sources.list.d
 
 msgmsg 'Unreadable config directory'
 chmod -r rootdir/etc/apt/apt.conf.d
-runthemall "E: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/ - opendir (13: Permission denied)"
+runthemall "W: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/ - opendir (13: Permission denied)"
 chmod +r rootdir/etc/apt/apt.conf.d
 
 msgmsg 'Unreadable preferences directory'
 chmod -r rootdir/etc/apt/preferences.d
-runthemall "E: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/preferences.d/ - opendir (13: Permission denied)"
+runthemall "W: Unable to read ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/preferences.d/ - opendir (13: Permission denied)"
 chmod +r rootdir/etc/apt/preferences.d
diff --git a/test/libapt/authconf_test.cc b/test/libapt/authconf_test.cc
new file mode 100644
index 000000000..10d818ec9
--- /dev/null
+++ b/test/libapt/authconf_test.cc
@@ -0,0 +1,223 @@
+#include <config.h>
+
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/netrc.h>
+#include <apt-pkg/strutl.h>
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "file-helpers.h"
+
+TEST(NetRCTest, Parsing)
+{
+   FileFd fd;
+   URI U("http://file.not/open");
+   EXPECT_FALSE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+   EXPECT_EQ("file.not", U.Host);
+   EXPECT_EQ("/open", U.Path);
+
+   createTemporaryFile("doublesignedfile", fd, nullptr, R"apt(
+machine example.netter login bar password foo
+machine example.net login foo password bar
+
+machine example.org:90 login apt password apt
+machine	example.org:8080
+login
+example	password 	 foobar
+
+machine example.org
+login anonymous
+password pass
+
+machine example.com/foo login user1 unknown token password pass1
+machine example.com/bar password pass2 login user2
+		  unknown token
+machine example.com/user login user
+machine example.netter login unused password firstentry
+machine example.last/debian login debian password rules)apt");
+   U = URI("http://example.net/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo", U.User);
+   EXPECT_EQ("bar", U.Password);
+   EXPECT_EQ("example.net", U.Host);
+   EXPECT_EQ("/foo", U.Path);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://user:pass@example.net/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("user", U.User);
+   EXPECT_EQ("pass", U.Password);
+   EXPECT_EQ("example.net", U.Host);
+   EXPECT_EQ("/foo", U.Path);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.org:90/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("apt", U.User);
+   EXPECT_EQ("apt", U.Password);
+   EXPECT_EQ("example.org", U.Host);
+   EXPECT_EQ(90, U.Port);
+   EXPECT_EQ("/foo", U.Path);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.org:8080/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("example", U.User);
+   EXPECT_EQ("foobar", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.net:42/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo", U.User);
+   EXPECT_EQ("bar", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.org/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("anonymous", U.User);
+   EXPECT_EQ("pass", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.com/apt");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.com/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("user1", U.User);
+   EXPECT_EQ("pass1", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.com/fooo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("user1", U.User);
+   EXPECT_EQ("pass1", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.com/fo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.com/bar");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("user2", U.User);
+   EXPECT_EQ("pass2", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.com/user");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("user", U.User);
+   EXPECT_TRUE(U.Password.empty());
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("socks5h://example.last/debian");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("debian", U.User);
+   EXPECT_EQ("rules", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("socks5h://example.debian/");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("socks5h://user:pass@example.debian/");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("user", U.User);
+   EXPECT_EQ("pass", U.Password);
+}
+TEST(NetRCTest, BadFileNoMachine)
+{
+   FileFd fd;
+   createTemporaryFile("doublesignedfile", fd, nullptr, R"apt(
+foo example.org login foo1 password bar
+machin example.org login foo2 password bar
+machine2 example.org login foo3 password bar
+)apt");
+
+   URI U("http://example.org/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+}
+TEST(NetRCTest, BadFileEndsMachine)
+{
+   FileFd fd;
+   createTemporaryFile("doublesignedfile", fd, nullptr, R"apt(
+machine example.org login foo1 password bar
+machine)apt");
+
+   URI U("http://example.org/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo1", U.User);
+   EXPECT_EQ("bar", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.net/foo");
+   EXPECT_FALSE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://foo:bar@example.net/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo", U.User);
+   EXPECT_EQ("bar", U.Password);
+}
+TEST(NetRCTest, BadFileEndsLogin)
+{
+   FileFd fd;
+   createTemporaryFile("doublesignedfile", fd, nullptr, R"apt(
+machine example.org login foo1 password bar
+machine example.net login)apt");
+
+   URI U("http://example.org/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo1", U.User);
+   EXPECT_EQ("bar", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.net/foo");
+   EXPECT_FALSE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://foo:bar@example.net/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo", U.User);
+   EXPECT_EQ("bar", U.Password);
+}
+TEST(NetRCTest, BadFileEndsPassword)
+{
+   FileFd fd;
+   createTemporaryFile("doublesignedfile", fd, nullptr, R"apt(
+machine example.org login foo1 password bar
+machine example.net password)apt");
+
+   URI U("http://example.org/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo1", U.User);
+   EXPECT_EQ("bar", U.Password);
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://example.net/foo");
+   EXPECT_FALSE(MaybeAddAuth(fd, U));
+   EXPECT_TRUE(U.User.empty());
+   EXPECT_TRUE(U.Password.empty());
+
+   EXPECT_TRUE(fd.Seek(0));
+   U = URI("http://foo:bar@example.net/foo");
+   EXPECT_TRUE(MaybeAddAuth(fd, U));
+   EXPECT_EQ("foo", U.User);
+   EXPECT_EQ("bar", U.Password);
+}
diff --git a/test/libapt/configuration_test.cc b/test/libapt/configuration_test.cc
index bdc17cbf4..bfb7144ce 100644
--- a/test/libapt/configuration_test.cc
+++ b/test/libapt/configuration_test.cc
@@ -1,14 +1,14 @@
 #include <config.h>
 
 #include <apt-pkg/configuration.h>
+#include <apt-pkg/fileutl.h>
 
 #include <string>
 #include <vector>
 
 #include <gtest/gtest.h>
 
-//FIXME: Test for configuration file parsing;
-// currently only integration/ tests test them implicitly
+#include "file-helpers.h"
 
 TEST(ConfigurationTest,Lists)
 {
@@ -195,3 +195,40 @@ TEST(ConfigurationTest,Merge)
 	EXPECT_EQ("bar", Cnf.Find("option::foo"));
 	EXPECT_EQ("", Cnf.Find("option::empty"));
 }
+TEST(ConfigurationTest, Parsing)
+{
+   Configuration Cnf;
+   std::string tempfile;
+   FileFd fd;
+   createTemporaryFile("doublesignedfile", fd, &tempfile, R"apt(
+SimpleOption "true";
+/* SimpleOption "false"; */
+Answer::Simple "42";
+# This is a comment
+List::Option { "In"; "One"; "Line"; };
+// this a comment as well
+List::Option2 { "Multi";
+"Line"; # inline comment
+	 "Options";
+}; Trailing "true";
+/* Commented::Out "true"; */
+)apt");
+   EXPECT_TRUE(ReadConfigFile(Cnf, tempfile));
+   if (tempfile.empty() == false)
+      unlink(tempfile.c_str());
+   EXPECT_TRUE(Cnf.FindB("SimpleOption"));
+   EXPECT_EQ(42, Cnf.FindI("Answer::Simple"));
+   EXPECT_TRUE(Cnf.Exists("List::Option"));
+   auto const singleline = Cnf.FindVector("List::Option");
+   EXPECT_EQ(3, singleline.size());
+   EXPECT_EQ("In", singleline[0]);
+   EXPECT_EQ("One", singleline[1]);
+   EXPECT_EQ("Line", singleline[2]);
+   auto const multiline = Cnf.FindVector("List::Option2");
+   EXPECT_EQ(3, multiline.size());
+   EXPECT_EQ("Multi", multiline[0]);
+   EXPECT_EQ("Line", multiline[1]);
+   EXPECT_EQ("Options", multiline[2]);
+   EXPECT_TRUE(Cnf.FindB("Trailing"));
+   EXPECT_FALSE(Cnf.Exists("Commented::Out"));
+}
diff --git a/test/libapt/fileutl_test.cc b/test/libapt/fileutl_test.cc
index 6cc850033..a702c16ec 100644
--- a/test/libapt/fileutl_test.cc
+++ b/test/libapt/fileutl_test.cc
@@ -152,6 +152,19 @@ static void TestFileFd(mode_t const a_umask, mode_t const ExpectedFilePermission
       EXPECT_EQ(strlen(expect), f.Tell());
    }
 #undef APT_INIT_READBACK
+   {
+      test.erase(test.length() - 1);
+      EXPECT_TRUE(f.Seek(0));
+      EXPECT_FALSE(f.Eof());
+      std::string line;
+      EXPECT_TRUE(f.ReadLine(line));
+      EXPECT_FALSE(f.Failed());
+      EXPECT_FALSE(f.Eof());
+      EXPECT_EQ(line.length(), test.length());
+      EXPECT_EQ(line.length() + 1, f.Tell());
+      EXPECT_EQ(f.Size(), f.Tell());
+      EXPECT_EQ(line, test);
+   }
 
    f.Close();
    EXPECT_FALSE(f.IsOpen());