Merge branch 'jl/submodule-fetch-on-demand'

* jl/submodule-fetch-on-demand:
  fetch/pull: Describe --recurse-submodule restrictions in the BUGS section
  submodule update: Don't fetch when the submodule commit is already present
  fetch/pull: Don't recurse into a submodule when commits are already present
  Submodules: Add 'on-demand' value for the 'fetchRecurseSubmodule' option
  config: teach the fetch.recurseSubmodules option the 'on-demand' value
  fetch/pull: Add the 'on-demand' value to the --recurse-submodules option
  fetch/pull: recurse into submodules when necessary

Conflicts:
	builtin/fetch.c
	submodule.c

Documentation/config.txt

@@ -897,9 +897,13 @@ diff.wordRegex::
 	characters are *ignorable* whitespace.
 
 fetch.recurseSubmodules::
-	A boolean value which changes the behavior for fetch and pull, the
-	default is to not recursively fetch populated submodules unless
-	configured otherwise.
+	This option can be either set to a boolean value or to 'on-demand'.
+	Setting it to a boolean changes the behavior of fetch and pull to
+	unconditionally recurse into submodules when set to true or to not
+	recurse at all when set to false. When set to 'on-demand' (the default
+	value), fetch and pull will only recurse into a populated submodule
+	when its superproject retrieves a commit that updates the submodule's
+	reference.
 
 fetch.unpackLimit::
 	If the number of objects fetched over the git native
@@ -1823,7 +1827,7 @@ submodule.<name>.update::
 	linkgit:git-submodule[1] and linkgit:gitmodules[5] for details.
 
 submodule.<name>.fetchRecurseSubmodules::
-	This option can be used to enable/disable recursive fetching of this
+	This option can be used to control recursive fetching of this
 	submodule. It can be overridden by using the --[no-]recurse-submodules
 	command line option to "git fetch" and "git pull".
 	This setting will override that from in the linkgit:gitmodules[5]

Documentation/fetch-options.txt

@@ -65,14 +65,33 @@ ifndef::git-pull[]
 	specified with the remote.<name>.tagopt setting. See
 	linkgit:git-config[1].
 
---[no-]recurse-submodules::
-	This option controls if new commits of all populated submodules should
-	be fetched too (see linkgit:git-config[1] and linkgit:gitmodules[5]).
+--recurse-submodules[=yes|on-demand|no]::
+	This option controls if and under what conditions new commits of
+	populated submodules should be fetched too. It can be used as a
+	boolean option to completely disable recursion when set to 'no' or to
+	unconditionally recurse into all populated submodules when set to
+	'yes', which is the default when this option is used without any
+	value. Use 'on-demand' to only recurse into a populated submodule
+	when the superproject retrieves a commit that updates the submodule's
+	reference to a commit that isn't already in the local submodule
+	clone.
+
+--no-recurse-submodules::
+	Disable recursive fetching of submodules (this has the same effect as
+	using the '--recurse-submodules=no' option).
 
 --submodule-prefix=<path>::
 	Prepend <path> to paths printed in informative messages
 	such as "Fetching submodule foo".  This option is used
 	internally when recursing over submodules.
+
+--recurse-submodules-default=[yes|on-demand]::
+	This option is used internally to temporarily provide a
+	non-negative default value for the --recurse-submodules
+	option.  All other methods of configuring fetch's submodule
+	recursion (such as settings in linkgit:gitmodules[5] and
+	linkgit:git-config[1]) override this option, as does
+	specifying --[no-]recurse-submodules directly.
 endif::git-pull[]
 
 -u::

Documentation/git-fetch.txt

@@ -76,6 +76,15 @@ The `pu` branch will be updated even if it is does not fast-forward,
 because it is prefixed with a plus sign; `tmp` will not be.
 
 
+BUGS
+----
+Using --recurse-submodules can only fetch new commits in already checked
+out submodules right now. When e.g. upstream added a new submodule in the
+just fetched commits of the superproject the submodule itself can not be
+fetched, making it impossible to check out that submodule later without
+having to do a fetch again. This is expected to be fixed in a future git
+version.
+
 SEE ALSO
 --------
 linkgit:git-pull[1]

Documentation/git-pull.txt

@@ -84,7 +84,7 @@ must be given before the options meant for 'git fetch'.
 --verbose::
 	Pass --verbose to git-fetch and git-merge.
 
---[no-]recurse-submodules::
+--[no-]recurse-submodules[=yes|on-demand|no]::
 	This option controls if new commits of all populated submodules should
 	be fetched too (see linkgit:git-config[1] and linkgit:gitmodules[5]).
 	That might be necessary to get the data needed for merging submodule
@@ -220,6 +220,15 @@ If you tried a pull which resulted in a complex conflicts and
 would want to start over, you can recover with 'git reset'.
 
 
+BUGS
+----
+Using --recurse-submodules can only fetch new commits in already checked
+out submodules right now. When e.g. upstream added a new submodule in the
+just fetched commits of the superproject the submodule itself can not be
+fetched, making it impossible to check out that submodule later without
+having to do a fetch again. This is expected to be fixed in a future git
+version.
+
 SEE ALSO
 --------
 linkgit:git-fetch[1], linkgit:git-merge[1], linkgit:git-config[1]

Documentation/gitmodules.txt

@@ -45,12 +45,12 @@ submodule.<name>.update::
 	the '--merge' or '--rebase' options.
 
 submodule.<name>.fetchRecurseSubmodules::
-	This option can be used to enable/disable recursive fetching of this
+	This option can be used to control recursive fetching of this
 	submodule. If this option is also present in the submodules entry in
 	.git/config of the superproject, the setting there will override the
 	one found in .gitmodules.
 	Both settings can be overridden on the command line by using the
-	"--[no-]recurse-submodules" option to "git fetch" and "git pull"..
+	"--[no-]recurse-submodules" option to "git fetch" and "git pull".
 
 submodule.<name>.ignore::
 	Defines under what circumstances "git status" and the diff family show

builtin/fetch.c

@@ -28,12 +28,6 @@ enum {
 	TAGS_SET = 2
 };
 
-enum {
-	RECURSE_SUBMODULES_OFF = 0,
-	RECURSE_SUBMODULES_DEFAULT = 1,
-	RECURSE_SUBMODULES_ON = 2
-};
-
 static int all, append, dry_run, force, keep, multiple, prune, update_head_ok, verbosity;
 static int progress, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT;
@@ -42,6 +36,21 @@ static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;
 static struct transport *transport;
 static const char *submodule_prefix = "";
+static const char *recurse_submodules_default;
+
+static int option_parse_recurse_submodules(const struct option *opt,
+				   const char *arg, int unset)
+{
+	if (unset) {
+		recurse_submodules = RECURSE_SUBMODULES_OFF;
+	} else {
+		if (arg)
+			recurse_submodules = parse_fetch_recurse_submodules_arg(opt->long_name, arg);
+		else
+			recurse_submodules = RECURSE_SUBMODULES_ON;
+	}
+	return 0;
+}
 
 static struct option builtin_fetch_options[] = {
 	OPT__VERBOSITY(&verbosity),
@@ -60,9 +69,9 @@ static struct option builtin_fetch_options[] = {
 		    "do not fetch all tags (--no-tags)", TAGS_UNSET),
 	OPT_BOOLEAN('p', "prune", &prune,
 		    "prune remote-tracking branches no longer on remote"),
-	OPT_SET_INT(0, "recurse-submodules", &recurse_submodules,
+	{ OPTION_CALLBACK, 0, "recurse-submodules", NULL, "on-demand",
 		    "control recursive fetching of submodules",
-		    RECURSE_SUBMODULES_ON),
+		    PARSE_OPT_OPTARG, option_parse_recurse_submodules },
 	OPT_BOOLEAN(0, "dry-run", &dry_run,
 		    "dry run"),
 	OPT_BOOLEAN('k', "keep", &keep, "keep downloaded pack"),
@@ -73,6 +82,9 @@ static struct option builtin_fetch_options[] = {
 		   "deepen history of shallow clone"),
 	{ OPTION_STRING, 0, "submodule-prefix", &submodule_prefix, "dir",
 		   "prepend this to submodule path output", PARSE_OPT_HIDDEN },
+	{ OPTION_STRING, 0, "recurse-submodules-default",
+		   &recurse_submodules_default, NULL,
+		   "default mode for recursion", PARSE_OPT_HIDDEN },
 	OPT_END()
 };
 
@@ -284,6 +296,9 @@ static int update_local_ref(struct ref *ref,
 		else {
 			msg = "storing head";
 			what = _("[new branch]");
+			if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
+			    (recurse_submodules != RECURSE_SUBMODULES_ON))
+				check_for_new_submodule_commits(ref->new_sha1);
 		}
 
 		r = s_update_ref(msg, ref, 0);
@@ -299,6 +314,9 @@ static int update_local_ref(struct ref *ref,
 		strcpy(quickref, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV));
 		strcat(quickref, "..");
 		strcat(quickref, find_unique_abbrev(ref->new_sha1, DEFAULT_ABBREV));
+		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
+		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+			check_for_new_submodule_commits(ref->new_sha1);
 		r = s_update_ref("fast-forward", ref, 1);
 		sprintf(display, "%c %-*s %-*s -> %s%s", r ? '!' : ' ',
 			TRANSPORT_SUMMARY_WIDTH, quickref, REFCOL_WIDTH, remote,
@@ -310,6 +328,9 @@ static int update_local_ref(struct ref *ref,
 		strcpy(quickref, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV));
 		strcat(quickref, "...");
 		strcat(quickref, find_unique_abbrev(ref->new_sha1, DEFAULT_ABBREV));
+		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
+		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+			check_for_new_submodule_commits(ref->new_sha1);
 		r = s_update_ref("forced-update", ref, 1);
 		sprintf(display, "%c %-*s %-*s -> %s  (%s)", r ? '!' : '+',
 			TRANSPORT_SUMMARY_WIDTH, quickref, REFCOL_WIDTH, remote,
@@ -810,6 +831,8 @@ static void add_options_to_argv(int *argc, const char **argv)
 		argv[(*argc)++] = "--keep";
 	if (recurse_submodules == RECURSE_SUBMODULES_ON)
 		argv[(*argc)++] = "--recurse-submodules";
+	else if (recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND)
+		argv[(*argc)++] = "--recurse-submodules=on-demand";
 	if (verbosity >= 2)
 		argv[(*argc)++] = "-v";
 	if (verbosity >= 1)
@@ -951,15 +974,16 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 	if (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) {
 		const char *options[10];
 		int num_options = 0;
-		/* Set recursion as default when we already are recursing */
-		if (submodule_prefix[0])
-			set_config_fetch_recurse_submodules(1);
+		if (recurse_submodules_default) {
+			int arg = parse_fetch_recurse_submodules_arg("--recurse-submodules-default", recurse_submodules_default);
+			set_config_fetch_recurse_submodules(arg);
+		}
 		gitmodules_config();
 		git_config(submodule_config, NULL);
 		add_options_to_argv(&num_options, options);
 		result = fetch_populated_submodules(num_options, options,
 						    submodule_prefix,
-						    recurse_submodules == RECURSE_SUBMODULES_ON,
+						    recurse_submodules,
 						    verbosity < 0);
 	}
 

git-pull.sh

@@ -110,6 +110,9 @@ do
 	--recurse-submodules)
 		recurse_submodules=--recurse-submodules
 		;;
+	--recurse-submodules=*)
+		recurse_submodules="$1"
+		;;
 	--no-recurse-submodules)
 		recurse_submodules=--no-recurse-submodules
 		;;

git-submodule.sh

@@ -488,8 +488,11 @@ cmd_update()
 
 			if test -z "$nofetch"
 			then
+				# Run fetch only if $sha1 isn't present or it
+				# is not reachable from a ref.
 				(clear_local_git_env; cd "$path" &&
-					git-fetch) ||
+					((rev=$(git rev-list -n 1 $sha1 --not --all 2>/dev/null) &&
+					 test -z "$rev") || git-fetch)) ||
 				die "Unable to fetch in submodule path '$path'"
 			fi