Merge branch 'jk/repack-no-explode-objects-from-old-pack'

Avoid writing out unreachable objects as loose objects when repacking,
if such loose objects will immediately pruned due to its age anyway.

By Jeff King
* jk/repack-no-explode-objects-from-old-pack:
  gc: use argv-array for sub-commands
  argv-array: add a new "pushl" method
  argv-array: refactor empty_argv initialization
  gc: do not explode objects which will be immediately pruned

Documentation/technical/api-argv-array.txt

@@ -37,6 +37,11 @@ Functions
 `argv_array_push`::
 	Push a copy of a string onto the end of the array.
 
+`argv_array_pushl`::
+	Push a list of strings onto the end of the array. The arguments
+	should be a list of `const char *` strings, terminated by a NULL
+	argument.
+
 `argv_array_pushf`::
 	Format a string and push it onto the end of the array. This is a
 	convenience wrapper combining `strbuf_addf` and `argv_array_push`.

argv-array.c

@@ -2,8 +2,7 @@
 #include "argv-array.h"
 #include "strbuf.h"
 
-static const char *empty_argv_storage = NULL;
-const char **empty_argv = &empty_argv_storage;
+const char *empty_argv[] = { NULL };
 
 void argv_array_init(struct argv_array *array)
 {
@@ -39,6 +38,17 @@ void argv_array_pushf(struct argv_array *array, const char *fmt, ...)
 	argv_array_push_nodup(array, strbuf_detach(&v, NULL));
 }
 
+void argv_array_pushl(struct argv_array *array, ...)
+{
+	va_list ap;
+	const char *arg;
+
+	va_start(ap, array);
+	while((arg = va_arg(ap, const char *)))
+		argv_array_push(array, arg);
+	va_end(ap);
+}
+
 void argv_array_clear(struct argv_array *array)
 {
 	if (array->argv != empty_argv) {

argv-array.h

@@ -1,7 +1,7 @@
 #ifndef ARGV_ARRAY_H
 #define ARGV_ARRAY_H
 
-extern const char **empty_argv;
+extern const char *empty_argv[];
 
 struct argv_array {
 	const char **argv;
@@ -15,6 +15,7 @@ void argv_array_init(struct argv_array *);
 void argv_array_push(struct argv_array *, const char *);
 __attribute__((format (printf,2,3)))
 void argv_array_pushf(struct argv_array *, const char *fmt, ...);
+void argv_array_pushl(struct argv_array *, ...);
 void argv_array_clear(struct argv_array *);
 
 #endif /* ARGV_ARRAY_H */

builtin/gc.c

@@ -14,6 +14,7 @@
 #include "cache.h"
 #include "parse-options.h"
 #include "run-command.h"
+#include "argv-array.h"
 
 #define FAILED_RUN "failed to run %s"
 
@@ -28,12 +29,11 @@ static int gc_auto_threshold = 6700;
 static int gc_auto_pack_limit = 50;
 static const char *prune_expire = "2.weeks.ago";
 
-#define MAX_ADD 10
-static const char *argv_pack_refs[] = {"pack-refs", "--all", "--prune", NULL};
-static const char *argv_reflog[] = {"reflog", "expire", "--all", NULL};
-static const char *argv_repack[MAX_ADD] = {"repack", "-d", "-l", NULL};
-static const char *argv_prune[] = {"prune", "--expire", NULL, NULL, NULL};
-static const char *argv_rerere[] = {"rerere", "gc", NULL};
+static struct argv_array pack_refs_cmd = ARGV_ARRAY_INIT;
+static struct argv_array reflog = ARGV_ARRAY_INIT;
+static struct argv_array repack = ARGV_ARRAY_INIT;
+static struct argv_array prune = ARGV_ARRAY_INIT;
+static struct argv_array rerere = ARGV_ARRAY_INIT;
 
 static int gc_config(const char *var, const char *value, void *cb)
 {
@@ -67,19 +67,6 @@ static int gc_config(const char *var, const char *value, void *cb)
 	return git_default_config(var, value, cb);
 }
 
-static void append_option(const char **cmd, const char *opt, int max_length)
-{
-	int i;
-
-	for (i = 0; cmd[i]; i++)
-		;
-
-	if (i + 2 >= max_length)
-		die(_("Too many options specified"));
-	cmd[i++] = opt;
-	cmd[i] = NULL;
-}
-
 static int too_many_loose_objects(void)
 {
 	/*
@@ -144,6 +131,17 @@ static int too_many_packs(void)
 	return gc_auto_pack_limit <= cnt;
 }
 
+static void add_repack_all_option(void)
+{
+	if (prune_expire && !strcmp(prune_expire, "now"))
+		argv_array_push(&repack, "-a");
+	else {
+		argv_array_push(&repack, "-A");
+		if (prune_expire)
+			argv_array_pushf(&repack, "--unpack-unreachable=%s", prune_expire);
+	}
+}
+
 static int need_to_gc(void)
 {
 	/*
@@ -160,10 +158,7 @@ static int need_to_gc(void)
 	 * there is no need.
 	 */
 	if (too_many_packs())
-		append_option(argv_repack,
-			      prune_expire && !strcmp(prune_expire, "now") ?
-			      "-a" : "-A",
-			      MAX_ADD);
+		add_repack_all_option();
 	else if (!too_many_loose_objects())
 		return 0;
 
@@ -177,7 +172,6 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
 	int aggressive = 0;
 	int auto_gc = 0;
 	int quiet = 0;
-	char buf[80];
 
 	struct option builtin_gc_options[] = {
 		OPT__QUIET(&quiet, "suppress progress reporting"),
@@ -192,6 +186,12 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
 	if (argc == 2 && !strcmp(argv[1], "-h"))
 		usage_with_options(builtin_gc_usage, builtin_gc_options);
 
+	argv_array_pushl(&pack_refs_cmd, "pack-refs", "--all", "--prune", NULL);
+	argv_array_pushl(&reflog, "reflog", "expire", "--all", NULL);
+	argv_array_pushl(&repack, "repack", "-d", "-l", NULL);
+	argv_array_pushl(&prune, "prune", "--expire", NULL );
+	argv_array_pushl(&rerere, "rerere", "gc", NULL);
+
 	git_config(gc_config, NULL);
 
 	if (pack_refs < 0)
@@ -203,15 +203,13 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
 		usage_with_options(builtin_gc_usage, builtin_gc_options);
 
 	if (aggressive) {
-		append_option(argv_repack, "-f", MAX_ADD);
-		append_option(argv_repack, "--depth=250", MAX_ADD);
-		if (aggressive_window > 0) {
-			sprintf(buf, "--window=%d", aggressive_window);
-			append_option(argv_repack, buf, MAX_ADD);
-		}
+		argv_array_push(&repack, "-f");
+		argv_array_push(&repack, "--depth=250");
+		if (aggressive_window > 0)
+			argv_array_pushf(&repack, "--window=%d", aggressive_window);
 	}
 	if (quiet)
-		append_option(argv_repack, "-q", MAX_ADD);
+		argv_array_push(&repack, "-q");
 
 	if (auto_gc) {
 		/*
@@ -227,30 +225,27 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
 					"run \"git gc\" manually. See "
 					"\"git help gc\" for more information.\n"));
 	} else
-		append_option(argv_repack,
-			      prune_expire && !strcmp(prune_expire, "now")
-			      ? "-a" : "-A",
-			      MAX_ADD);
+		add_repack_all_option();
 
-	if (pack_refs && run_command_v_opt(argv_pack_refs, RUN_GIT_CMD))
-		return error(FAILED_RUN, argv_pack_refs[0]);
+	if (pack_refs && run_command_v_opt(pack_refs_cmd.argv, RUN_GIT_CMD))
+		return error(FAILED_RUN, pack_refs_cmd.argv[0]);
 
-	if (run_command_v_opt(argv_reflog, RUN_GIT_CMD))
-		return error(FAILED_RUN, argv_reflog[0]);
+	if (run_command_v_opt(reflog.argv, RUN_GIT_CMD))
+		return error(FAILED_RUN, reflog.argv[0]);
 
-	if (run_command_v_opt(argv_repack, RUN_GIT_CMD))
-		return error(FAILED_RUN, argv_repack[0]);
+	if (run_command_v_opt(repack.argv, RUN_GIT_CMD))
+		return error(FAILED_RUN, repack.argv[0]);
 
 	if (prune_expire) {
-		argv_prune[2] = prune_expire;
+		argv_array_push(&prune, prune_expire);
 		if (quiet)
-			argv_prune[3] = "--no-progress";
-		if (run_command_v_opt(argv_prune, RUN_GIT_CMD))
-			return error(FAILED_RUN, argv_prune[0]);
+			argv_array_push(&prune, "--no-progress");
+		if (run_command_v_opt(prune.argv, RUN_GIT_CMD))
+			return error(FAILED_RUN, prune.argv[0]);
 	}
 
-	if (run_command_v_opt(argv_rerere, RUN_GIT_CMD))
-		return error(FAILED_RUN, argv_rerere[0]);
+	if (run_command_v_opt(rerere.argv, RUN_GIT_CMD))
+		return error(FAILED_RUN, rerere.argv[0]);
 
 	if (auto_gc && too_many_loose_objects())
 		warning(_("There are too many unreachable loose objects; "

builtin/pack-objects.c

@@ -63,6 +63,7 @@ static uint32_t nr_objects, nr_alloc, nr_result, nr_written;
 static int non_empty;
 static int reuse_delta = 1, reuse_object = 1;
 static int keep_unreachable, unpack_unreachable, include_tag;
+static unsigned long unpack_unreachable_expiration;
 static int local;
 static int incremental;
 static int ignore_packed_keep;
@@ -2249,6 +2250,10 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
 		if (!p->pack_local || p->pack_keep)
 			continue;
 
+		if (unpack_unreachable_expiration &&
+		    p->mtime < unpack_unreachable_expiration)
+			continue;
+
 		if (open_pack_index(p))
 			die("cannot open pack index");
 
@@ -2315,6 +2320,21 @@ static int option_parse_index_version(const struct option *opt,
 	return 0;
 }
 
+static int option_parse_unpack_unreachable(const struct option *opt,
+					   const char *arg, int unset)
+{
+	if (unset) {
+		unpack_unreachable = 0;
+		unpack_unreachable_expiration = 0;
+	}
+	else {
+		unpack_unreachable = 1;
+		if (arg)
+			unpack_unreachable_expiration = approxidate(arg);
+	}
+	return 0;
+}
+
 static int option_parse_ulong(const struct option *opt,
 			      const char *arg, int unset)
 {
@@ -2392,8 +2412,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 			 "include tag objects that refer to objects to be packed"),
 		OPT_BOOL(0, "keep-unreachable", &keep_unreachable,
 			 "keep unreachable objects"),
-		OPT_BOOL(0, "unpack-unreachable", &unpack_unreachable,
-			 "unpack unreachable objects"),
+		{ OPTION_CALLBACK, 0, "unpack-unreachable", NULL, "time",
+		  "unpack unreachable objects newer than <time>",
+		  PARSE_OPT_OPTARG, option_parse_unpack_unreachable },
 		OPT_BOOL(0, "thin", &thin,
 			 "create thin packs"),
 		OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep,

git-repack.sh

@@ -15,6 +15,7 @@ F               pass --no-reuse-object to git-pack-objects
 n               do not run git-update-server-info
 q,quiet         be quiet
 l               pass --local to git-pack-objects
+unpack-unreachable=  with -A, do not loosen objects older than this
  Packing constraints
 window=         size of the window used for delta compression
 window-memory=  same as the above, but limit memory size instead of entries count
@@ -33,6 +34,8 @@ do
 	-a)	all_into_one=t ;;
 	-A)	all_into_one=t
 		unpack_unreachable=--unpack-unreachable ;;
+	--unpack-unreachable)
+		unpack_unreachable="--unpack-unreachable=$2"; shift ;;
 	-d)	remove_redundant=t ;;
 	-q)	GIT_QUIET=t ;;
 	-f)	no_reuse=--no-reuse-delta ;;
@@ -76,7 +79,12 @@ case ",$all_into_one," in
 		if test -n "$existing" -a -n "$unpack_unreachable" -a \
 			-n "$remove_redundant"
 		then
-			args="$args $unpack_unreachable"
+			# This may have arbitrary user arguments, so we
+			# have to protect it against whitespace splitting
+			# when it gets run as "pack-objects $args" later.
+			# Fortunately, we know it's an approxidate, so we
+			# can just use dots instead.
+			args="$args $(echo "$unpack_unreachable" | tr ' ' .)"
 		fi
 	fi
 	;;

t/t7701-repack-unpack-unreachable.sh

@@ -95,4 +95,18 @@ test_expect_success 'unpacked objects receive timestamp of pack file' '
 	compare_mtimes < mtimes
 '
 
+test_expect_success 'do not bother loosening old objects' '
+	obj1=$(echo one | git hash-object -w --stdin) &&
+	obj2=$(echo two | git hash-object -w --stdin) &&
+	pack1=$(echo $obj1 | git pack-objects .git/objects/pack/pack) &&
+	pack2=$(echo $obj2 | git pack-objects .git/objects/pack/pack) &&
+	git prune-packed &&
+	git cat-file -p $obj1 &&
+	git cat-file -p $obj2 &&
+	test-chmtime =-86400 .git/objects/pack/pack-$pack2.pack &&
+	git repack -A -d --unpack-unreachable=1.hour.ago &&
+	git cat-file -p $obj1 &&
+	test_must_fail git cat-file -p $obj2
+'
+
 test_done