pkg: clean support for repositories

Rework the way the bootstrap fetches pkg, by implementing a full support
for the repositories, the boostrap will now loop over all available repo
and try to fetch the full package from there. It will at the first valid
package found.

Fallback to packagesite (which has been deprecated for a while) if needed, by
transforming it into a repo, if no repo is found.

MFC After:	3 weeks

usr.sbin/pkg/config.c

@@ -59,6 +59,8 @@ struct config_entry {
 	bool main_only;				/* Only set in pkg.conf. */
 };
 
+static struct repositories repositories = STAILQ_HEAD_INITIALIZER(repositories);
+
 static struct config_entry c[] = {
 	[PACKAGESITE] = {
 		PKG_CONFIG_STRING,
@@ -211,7 +213,7 @@ boolstr_to_bool(const char *str)
 }
 
 static void
-config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
+config_parse(const ucl_object_t *obj)
 {
 	FILE *buffp;
 	char *buf = NULL;
@@ -238,29 +240,9 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
 			memset(buf, 0, bufsz);
 		rewind(buffp);
 
-		if (conftype == CONFFILE_PKG) {
 			for (j = 0; j < strlen(key); ++j)
 				fputc(toupper(key[j]), buffp);
 			fflush(buffp);
-		} else if (conftype == CONFFILE_REPO) {
-			if (strcasecmp(key, "url") == 0)
-				fputs("PACKAGESITE", buffp);
-			else if (strcasecmp(key, "mirror_type") == 0)
-				fputs("MIRROR_TYPE", buffp);
-			else if (strcasecmp(key, "signature_type") == 0)
-				fputs("SIGNATURE_TYPE", buffp);
-			else if (strcasecmp(key, "fingerprints") == 0)
-				fputs("FINGERPRINTS", buffp);
-			else if (strcasecmp(key, "pubkey") == 0)
-				fputs("PUBKEY", buffp);
-			else if (strcasecmp(key, "enabled") == 0) {
-				if ((cur->type != UCL_BOOLEAN) ||
-				    !ucl_object_toboolean(cur))
-					goto cleanup;
-			} else
-				continue;
-			fflush(buffp);
-		}
 
 		for (i = 0; i < CONFIG_SIZE; i++) {
 			if (strcmp(buf, c[i].key) == 0)
@@ -324,7 +306,7 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
 		if (c[i].envset)
 			continue;
 		/* Prevent overriding ABI, ASSUME_ALWAYS_YES, etc. */
-		if (conftype != CONFFILE_PKG && c[i].main_only == true)
+		if (c[i].main_only == true)
 			continue;
 		switch (c[i].type) {
 		case PKG_CONFIG_LIST:
@@ -336,12 +318,91 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
 		}
 	}
 
-cleanup:
 	free(temp_config);
 	fclose(buffp);
 	free(buf);
 }
 
+
+static void
+parse_mirror_type(struct repository *r, const char *mt)
+{
+	if (strcasecmp(mt, "srv") == 0)
+		r->mirror_type = MIRROR_SRV;
+	r->mirror_type = MIRROR_NONE;
+}
+
+static bool
+parse_signature_type(struct repository *repo, const char *st)
+{
+	if (strcasecmp(st, "FINGERPRINTS") == 0)
+		repo->signature_type = SIGNATURE_FINGERPRINT;
+	else if (strcasecmp(st, "PUBKEY") == 0)
+		repo->signature_type = SIGNATURE_PUBKEY;
+	else if (strcasecmp(st, "NONE") == 0)
+		repo->signature_type = SIGNATURE_NONE;
+	else {
+		warnx("Signature type %s is not supported for bootstraping,"
+		    " ignoring repository %s", st, repo->name);
+		free(repo->url);
+		free(repo->name);
+		free(repo->fingerprints);
+		free(repo->pubkey);
+		free(repo);
+		return false;
+	}
+	return (true);
+}
+
+static void
+parse_repo(const ucl_object_t *o)
+{
+	const ucl_object_t *cur;
+	const char *key;
+	ucl_object_iter_t it = NULL;
+
+	struct repository *repo = calloc(1, sizeof(struct repository));
+	if (repo == NULL)
+		err(EXIT_FAILURE, "calloc");
+
+	repo->name = strdup(ucl_object_key(o));
+	if (repo->name == NULL)
+		err(EXIT_FAILURE, "strdup");
+	while ((cur = ucl_iterate_object(o, &it, true))) {
+		key = ucl_object_key(cur);
+		if (key == NULL)
+			continue;
+		if (strcasecmp(key, "url") == 0) {
+			repo->url = strdup(ucl_object_tostring(cur));
+			if (repo->url == NULL)
+				err(EXIT_FAILURE, "strdup");
+		} else if (strcasecmp(key, "mirror_type") == 0) {
+			parse_mirror_type(repo, ucl_object_tostring(cur));
+		} else if (strcasecmp(key, "signature_type") == 0) {
+			if (!parse_signature_type(repo, ucl_object_tostring(cur)))
+				return;
+		} else if (strcasecmp(key, "fingerprints") == 0) {
+			repo->fingerprints = strdup(ucl_object_tostring(cur));
+			if (repo->fingerprints == NULL)
+				err(EXIT_FAILURE, "strdup");
+		} else if (strcasecmp(key, "pubkey") == 0) {
+			repo->pubkey = strdup(ucl_object_tostring(cur));
+			if (repo->pubkey == NULL)
+				err(EXIT_FAILURE, "strdup");
+		} else if (strcasecmp(key, "enabled") == 0) {
+			if ((cur->type != UCL_BOOLEAN) ||
+			    !ucl_object_toboolean(cur)) {
+				free(repo->url);
+				free(repo->name);
+				free(repo);
+				return;
+			}
+		}
+	}
+	STAILQ_INSERT_TAIL(&repositories, repo, next);
+	return;
+}
+
 /*-
  * Parse new repo style configs in style:
  * Name:
@@ -367,8 +428,7 @@ parse_repo_file(ucl_object_t *obj, const char *requested_repo)
 
 		if (requested_repo != NULL && strcmp(requested_repo, key) != 0)
 			continue;
-
+		parse_repo(cur);
-		config_parse(cur, CONFFILE_REPO);
 	}
 }
 
@@ -379,8 +439,12 @@ read_conf_file(const char *confpath, const char *requested_repo,
 {
 	struct ucl_parser *p;
 	ucl_object_t *obj = NULL;
+	const char *abi = pkg_get_myabi();
+	if (abi == NULL)
+		errx(EXIT_FAILURE, "Fail do determine ABI");
 
 	p = ucl_parser_new(0);
+	ucl_parser_register_variable(p, "ABI", abi);
 
 	if (!ucl_parser_add_file(p, confpath)) {
 		if (errno != ENOENT)
@@ -397,7 +461,7 @@ read_conf_file(const char *confpath, const char *requested_repo,
 		    "configuration file %s", confpath);
 	else {
 		if (conftype == CONFFILE_PKG)
-			config_parse(obj, conftype);
+			config_parse(obj);
 		else if (conftype == CONFFILE_REPO)
 			parse_repo_file(obj, requested_repo);
 	}
@@ -408,7 +472,7 @@ read_conf_file(const char *confpath, const char *requested_repo,
 	return (0);
 }
 
-static int
+static void
 load_repositories(const char *repodir, const char *requested_repo)
 {
 	struct dirent *ent;
@@ -416,12 +480,9 @@ load_repositories(const char *repodir, const char *requested_repo)
 	char *p;
 	size_t n;
 	char path[MAXPATHLEN];
-	int ret;
-
-	ret = 0;
 
 	if ((d = opendir(repodir)) == NULL)
-		return (1);
+		return;
 
 	while ((ent = readdir(d))) {
 		/* Trim out 'repos'. */
@@ -437,7 +498,6 @@ load_repositories(const char *repodir, const char *requested_repo)
 				continue;
 			if (read_conf_file(path, requested_repo,
 			    CONFFILE_REPO)) {
-				ret = 1;
 				goto cleanup;
 			}
 		}
@@ -445,8 +505,6 @@ load_repositories(const char *repodir, const char *requested_repo)
 
 cleanup:
 	closedir(d);
-
-	return (ret);
 }
 
 int
@@ -508,8 +566,7 @@ config_init(const char *requested_repo)
 	}
 
 	STAILQ_FOREACH(cv, c[REPOS_DIR].list, next)
-		if (load_repositories(cv->value, requested_repo))
+		load_repositories(cv->value, requested_repo);
-			goto finalize;
 
 finalize:
 	if (c[ABI].val == NULL && c[ABI].value == NULL) {
@@ -520,8 +577,6 @@ config_init(const char *requested_repo)
 		c[ABI].val = abi;
 	}
 
-	subst_packagesite(c[ABI].value != NULL ? c[ABI].value : c[ABI].val);
-
 	return (0);
 }
 
@@ -560,6 +615,33 @@ config_bool(pkg_config_key k, bool *val)
 	return (0);
 }
 
+struct repositories *
+config_get_repositories(void)
+{
+	if (STAILQ_EMPTY(&repositories)) {
+		/* Fall back to PACKAGESITE - deprecated - */
+		struct repository *r = calloc(1, sizeof(r));
+		if (r == NULL)
+			err(EXIT_FAILURE, "calloc");
+		r->name = strdup("fallback");
+		if (r->name == NULL)
+			err(EXIT_FAILURE, "strdup");
+		subst_packagesite(c[ABI].value != NULL ? c[ABI].value : c[ABI].val);
+		r->url = c[PACKAGESITE].value;
+		if (c[SIGNATURE_TYPE].value != NULL)
+			if (!parse_signature_type(r, c[SIGNATURE_TYPE].value))
+				exit(EXIT_FAILURE);
+		if (c[MIRROR_TYPE].value != NULL)
+			parse_mirror_type(r, c[MIRROR_TYPE].value);
+		if (c[PUBKEY].value != NULL)
+			r->pubkey = c[PUBKEY].value;
+		if (c[FINGERPRINTS].value != NULL)
+			r->fingerprints = c[FINGERPRINTS].value;
+		STAILQ_INSERT_TAIL(&repositories, r, next);
+	}
+	return (&repositories);
+}
+
 void
 config_finish(void) {
 	int i;

usr.sbin/pkg/config.h

@@ -30,6 +30,7 @@
 #define _PKG_CONFIG_H
 
 #include <paths.h>
+#include <sys/queue.h>
 
 #define URL_SCHEME_PREFIX "pkg+"
 
@@ -58,9 +59,32 @@ typedef enum {
 	CONFFILE_REPO,
 } pkg_conf_file_t;
 
+typedef enum {
+	SIGNATURE_NONE = 0,
+	SIGNATURE_FINGERPRINT,
+	SIGNATURE_PUBKEY,
+} signature_t;
+
+typedef enum {
+	MIRROR_NONE = 0,
+	MIRROR_SRV,
+} mirror_t;
+
+struct repository {
+	char *name;
+	char *url;
+	mirror_t mirror_type;
+	signature_t signature_type;
+	char *fingerprints;
+	char *pubkey;
+	STAILQ_ENTRY(repository) next;
+};
+STAILQ_HEAD(repositories, repository);
+
 int config_init(const char *);
 void config_finish(void);
 int config_string(pkg_config_key, const char **);
 int config_bool(pkg_config_key, bool *);
+struct repositories *config_get_repositories(void);
 
 #endif

usr.sbin/pkg/pkg.c

@@ -255,7 +255,7 @@ install_pkg_static(const char *path, const char *pkgpath, bool force)
 }
 
 static int
-fetch_to_fd(const char *url, char *path, const char *fetchOpts)
+fetch_to_fd(struct repository *repo, const char *url, char *path, const char *fetchOpts)
 {
 	struct url *u;
 	struct dns_srvinfo *mirrors, *current;
@@ -267,18 +267,11 @@ fetch_to_fd(const char *url, char *path, const char *fetchOpts)
 	ssize_t r;
 	char buf[10240];
 	char zone[MAXHOSTNAMELEN + 13];
-	static const char *mirror_type = NULL;
 
 	max_retry = 3;
 	current = mirrors = NULL;
 	remote = NULL;
 
-	if (mirror_type == NULL && config_string(MIRROR_TYPE, &mirror_type)
-	    != 0) {
-		warnx("No MIRROR_TYPE defined");
-		return (-1);
-	}
-
 	if ((fd = mkstemp(path)) == -1) {
 		warn("mkstemp()");
 		return (-1);
@@ -294,7 +287,7 @@ fetch_to_fd(const char *url, char *path, const char *fetchOpts)
 	while (remote == NULL) {
 		if (retry == max_retry) {
 			if (strcmp(u->scheme, "file") != 0 &&
-			    strcasecmp(mirror_type, "srv") == 0) {
+			    repo->mirror_type == MIRROR_SRV) {
 				snprintf(zone, sizeof(zone),
 				    "_%s._tcp.%s", u->scheme, u->host);
 				mirrors = dns_getsrvinfo(zone);
@@ -653,23 +646,31 @@ parse_cert(int fd) {
 }
 
 static bool
-verify_pubsignature(int fd_pkg, int fd_sig)
+verify_pubsignature(int fd_pkg, int fd_sig, struct repository *r)
 {
 	struct pubkey *pk;
-	const char *pubkey;
 	char *data;
 	struct pkgsign_ctx *sctx;
 	size_t datasz;
 	bool ret;
+	const char *pubkey;
 
 	pk = NULL;
-	pubkey = NULL;
 	sctx = NULL;
 	data = NULL;
 	ret = false;
-	if (config_string(PUBKEY, &pubkey) != 0) {
+
-		warnx("No CONFIG_PUBKEY defined");
+	if (r != NULL) {
-		goto cleanup;
+		if (r->pubkey == NULL) {
+			warnx("No CONFIG_PUBKEY defined for %s", r->name);
+			goto cleanup;
+		}
+		pubkey = r->pubkey;
+	} else {
+		if (config_string(PUBKEY, &pubkey) != 0) {
+			warnx("No CONFIG_PUBKEY defined for %s", r->name);
+			goto cleanup;
+		}
 	}
 
 	if ((pk = read_pubkey(fd_sig)) == NULL) {
@@ -703,8 +704,8 @@ verify_pubsignature(int fd_pkg, int fd_sig)
 	}
 
 	/* Verify the signature. */
-	printf("Verifying signature with public key %s... ", pubkey);
+	printf("Verifying signature with public key %s.a.. ", r->pubkey);
-	if (pkgsign_verify_data(sctx, data, datasz, pubkey, NULL, 0, pk->sig,
+	if (pkgsign_verify_data(sctx, data, datasz, r->pubkey, NULL, 0, pk->sig,
 	    pk->siglen) == false) {
 		fprintf(stderr, "Signature is not valid\n");
 		goto cleanup;
@@ -723,7 +724,7 @@ verify_pubsignature(int fd_pkg, int fd_sig)
 }
 
 static bool
-verify_signature(int fd_pkg, int fd_sig)
+verify_signature(int fd_pkg, int fd_sig, struct repository *r)
 {
 	struct fingerprint_list *trusted, *revoked;
 	struct fingerprint *fingerprint;
@@ -742,9 +743,17 @@ verify_signature(int fd_pkg, int fd_sig)
 	ret = false;
 
 	/* Read and parse fingerprints. */
-	if (config_string(FINGERPRINTS, &fingerprints) != 0) {
+	if (r != NULL) {
-		warnx("No CONFIG_FINGERPRINTS defined");
+		if (r->fingerprints == NULL) {
-		goto cleanup;
+			warnx("No FINGERPRINTS defined for %s", r->name);
+			goto cleanup;
+		}
+		fingerprints = r->fingerprints;
+	} else {
+		if (config_string(FINGERPRINTS, &fingerprints) != 0) {
+			warnx("No FINGERPRINTS defined");
+			goto cleanup;
+		}
 	}
 
 	snprintf(path, MAXPATHLEN, "%s/trusted", fingerprints);
@@ -833,7 +842,7 @@ verify_signature(int fd_pkg, int fd_sig)
 }
 
 static int
-bootstrap_pkg(bool force, const char *fetchOpts)
+bootstrap_pkg(bool force, const char *fetchOpts, struct repository *repo)
 {
 	int fd_pkg, fd_sig;
 	int ret;
@@ -841,28 +850,18 @@ bootstrap_pkg(bool force, const char *fetchOpts)
 	char tmppkg[MAXPATHLEN];
 	char tmpsig[MAXPATHLEN];
 	const char *packagesite;
-	const char *signature_type;
 	char pkgstatic[MAXPATHLEN];
 	const char *bootstrap_name;
 
 	fd_sig = -1;
 	ret = -1;
 
-	if (config_string(PACKAGESITE, &packagesite) != 0) {
+	printf("Bootstrapping pkg from %s, please wait...\n", repo->url);
-		warnx("No PACKAGESITE defined");
-		return (-1);
-	}
-
-	if (config_string(SIGNATURE_TYPE, &signature_type) != 0) {
-		warnx("Error looking up SIGNATURE_TYPE");
-		return (-1);
-	}
-
-	printf("Bootstrapping pkg from %s, please wait...\n", packagesite);
 
 	/* Support pkg+http:// for PACKAGESITE which is the new format
 	   in 1.2 to avoid confusion on why http://pkg.FreeBSD.org has
 	   no A record. */
+	packagesite = repo->url;
 	if (strncmp(URL_SCHEME_PREFIX, packagesite,
 	    strlen(URL_SCHEME_PREFIX)) == 0)
 		packagesite += strlen(URL_SCHEME_PREFIX);
@@ -873,53 +872,44 @@ bootstrap_pkg(bool force, const char *fetchOpts)
 		snprintf(tmppkg, MAXPATHLEN, "%s/%s.XXXXXX",
 		    getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
 		    bootstrap_name);
-		if ((fd_pkg = fetch_to_fd(url, tmppkg, fetchOpts)) != -1)
+		if ((fd_pkg = fetch_to_fd(repo, url, tmppkg, fetchOpts)) != -1)
 			break;
 		bootstrap_name = NULL;
 	}
 	if (bootstrap_name == NULL)
 		goto fetchfail;
 
-	if (signature_type != NULL &&
+	if (repo->signature_type == SIGNATURE_FINGERPRINT) {
-	    strcasecmp(signature_type, "NONE") != 0) {
+		snprintf(tmpsig, MAXPATHLEN, "%s/%s.sig.XXXXXX",
-		if (strcasecmp(signature_type, "FINGERPRINTS") == 0) {
+		    getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
+		    bootstrap_name);
+		snprintf(url, MAXPATHLEN, "%s/Latest/%s.sig",
+		    packagesite, bootstrap_name);
 
-			snprintf(tmpsig, MAXPATHLEN, "%s/%s.sig.XXXXXX",
+		if ((fd_sig = fetch_to_fd(repo, url, tmpsig, fetchOpts)) == -1) {
-			    getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
+			fprintf(stderr, "Signature for pkg not "
-			    bootstrap_name);
+			    "available.\n");
-			snprintf(url, MAXPATHLEN, "%s/Latest/%s.sig",
+			goto fetchfail;
-			    packagesite, bootstrap_name);
-
-			if ((fd_sig = fetch_to_fd(url, tmpsig, fetchOpts)) == -1) {
-				fprintf(stderr, "Signature for pkg not "
-				    "available.\n");
-				goto fetchfail;
-			}
-
-			if (verify_signature(fd_pkg, fd_sig) == false)
-				goto cleanup;
-		} else if (strcasecmp(signature_type, "PUBKEY") == 0) {
-
-			snprintf(tmpsig, MAXPATHLEN,
-			    "%s/%s.pubkeysig.XXXXXX",
-			    getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
-			    bootstrap_name);
-			snprintf(url, MAXPATHLEN, "%s/Latest/%s.pubkeysig",
-			    packagesite, bootstrap_name);
-
-			if ((fd_sig = fetch_to_fd(url, tmpsig, fetchOpts)) == -1) {
-				fprintf(stderr, "Signature for pkg not "
-				    "available.\n");
-				goto fetchfail;
-			}
-
-			if (verify_pubsignature(fd_pkg, fd_sig) == false)
-				goto cleanup;
-		} else {
-			warnx("Signature type %s is not supported for "
-			    "bootstrapping.", signature_type);
-			goto cleanup;
 		}
+
+		if (verify_signature(fd_pkg, fd_sig, repo) == false)
+			goto cleanup;
+	} else if (repo->signature_type == SIGNATURE_PUBKEY) {
+		snprintf(tmpsig, MAXPATHLEN,
+		    "%s/%s.pubkeysig.XXXXXX",
+		    getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
+		    bootstrap_name);
+		snprintf(url, MAXPATHLEN, "%s/Latest/%s.pubkeysig",
+		    repo->url, bootstrap_name);
+
+		if ((fd_sig = fetch_to_fd(repo, url, tmpsig, fetchOpts)) == -1) {
+			fprintf(stderr, "Signature for pkg not "
+			    "available.\n");
+			goto fetchfail;
+		}
+
+		if (verify_pubsignature(fd_pkg, fd_sig, repo) == false)
+			goto cleanup;
 	}
 
 	if ((ret = extract_pkg_static(fd_pkg, pkgstatic, MAXPATHLEN)) == 0)
@@ -929,18 +919,15 @@ bootstrap_pkg(bool force, const char *fetchOpts)
 
 fetchfail:
 	for (int j = 0; bootstrap_names[j] != NULL; j++) {
-		warnx("Attempted to fetch %s/Latest/%s", packagesite,
+		warnx("Attempted to fetch %s/Latest/%s", repo->url,
 		    bootstrap_names[j]);
 	}
 	warnx("Error: %s", fetchLastErrString);
 	if (fetchLastErrCode == FETCH_RESOLV) {
 		fprintf(stderr, "Address resolution failed for %s.\n", packagesite);
-		fprintf(stderr, "Consider changing PACKAGESITE.\n");
 	} else {
 		fprintf(stderr, "A pre-built version of pkg could not be found for "
 		    "your system.\n");
-		fprintf(stderr, "Consider changing PACKAGESITE or installing it from "
-		    "ports: 'ports-mgmt/pkg'.\n");
 	}
 
 cleanup:
@@ -1024,7 +1011,7 @@ bootstrap_pkg_local(const char *pkgpath, bool force)
 				goto cleanup;
 			}
 
-			if (verify_signature(fd_pkg, fd_sig) == false)
+			if (verify_signature(fd_pkg, fd_sig, NULL) == false)
 				goto cleanup;
 
 		} else if (strcasecmp(signature_type, "PUBKEY") == 0) {
@@ -1037,7 +1024,7 @@ bootstrap_pkg_local(const char *pkgpath, bool force)
 				goto cleanup;
 			}
 
-			if (verify_pubsignature(fd_pkg, fd_sig) == false)
+			if (verify_pubsignature(fd_pkg, fd_sig, NULL) == false)
 				goto cleanup;
 
 		} else {
@@ -1105,6 +1092,7 @@ main(int argc, char *argv[])
 	signed char ch;
 	const char *fetchOpts;
 	char *command;
+	struct repositories *repositories;
 
 	activation_test = false;
 	add_pkg = false;
@@ -1232,6 +1220,8 @@ main(int argc, char *argv[])
 		fetchDebug = 1;
 
 	if ((bootstrap_only && force) || access(pkgpath, X_OK) == -1) {
+		struct repository *repo;
+		int ret = 0;
 		/*
 		 * To allow 'pkg -N' to be used as a reliable test for whether
 		 * a system is configured to use pkg, don't bootstrap pkg
@@ -1272,7 +1262,12 @@ main(int argc, char *argv[])
 			if (pkg_query_yes_no() == 0)
 				exit(EXIT_FAILURE);
 		}
-		if (bootstrap_pkg(force, fetchOpts) != 0)
+		repositories = config_get_repositories();
+		STAILQ_FOREACH(repo, repositories, next) {
+			if ((ret = bootstrap_pkg(force, fetchOpts, repo)) == 0)
+				break;
+		}
+		if (ret != 0)
 			exit(EXIT_FAILURE);
 		config_finish();