Skip to content

Commit

Permalink
fetch_populated_submodules: use new parallel job processing
Browse files Browse the repository at this point in the history 
In a later patch we enable parallel processing of submodules, this
only adds the possibility for it. So this change should not change
any user facing behavior.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
  • Loading branch information
Stefan Beller authored and Junio C Hamano committed Dec 16, 2015
1 parent c553c72 commit fe85ee6
Showing 1 changed file with 98 additions and 44 deletions.
142 changes: 98 additions & 44 deletions submodule.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "sha1-array.h"
#include "argv-array.h"
#include "blob.h"
#include "thread-utils.h"

static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND;
static struct string_list changed_submodule_paths;
Expand Down Expand Up @@ -610,37 +611,28 @@ static void calculate_changed_submodule_paths(void)
initialized_fetch_ref_tips = 0;
}

int fetch_populated_submodules(const struct argv_array *options,
const char *prefix, int command_line_option,
int quiet)
struct submodule_parallel_fetch {
int count;
struct argv_array args;
const char *work_tree;
const char *prefix;
int command_line_option;
int quiet;
int result;
};
#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0}

static int get_next_submodule(struct child_process *cp,
struct strbuf *err, void *data, void **task_cb)
{
int i, result = 0;
struct child_process cp = CHILD_PROCESS_INIT;
struct argv_array argv = ARGV_ARRAY_INIT;
const char *work_tree = get_git_work_tree();
if (!work_tree)
goto out;

if (read_cache() < 0)
die("index file corrupt");

argv_array_push(&argv, "fetch");
for (i = 0; i < options->argc; i++)
argv_array_push(&argv, options->argv[i]);
argv_array_push(&argv, "--recurse-submodules-default");
/* default value, "--submodule-prefix" and its value are added later */

cp.env = local_repo_env;
cp.git_cmd = 1;
cp.no_stdin = 1;

calculate_changed_submodule_paths();
int ret = 0;
struct submodule_parallel_fetch *spf = data;

for (i = 0; i < active_nr; i++) {
for (; spf->count < active_nr; spf->count++) {
struct strbuf submodule_path = STRBUF_INIT;
struct strbuf submodule_git_dir = STRBUF_INIT;
struct strbuf submodule_prefix = STRBUF_INIT;
const struct cache_entry *ce = active_cache[i];
const struct cache_entry *ce = active_cache[spf->count];
const char *git_dir, *default_argv;
const struct submodule *submodule;

Expand All @@ -652,7 +644,7 @@ int fetch_populated_submodules(const struct argv_array *options,
submodule = submodule_from_name(null_sha1, ce->name);

default_argv = "yes";
if (command_line_option == RECURSE_SUBMODULES_DEFAULT) {
if (spf->command_line_option == RECURSE_SUBMODULES_DEFAULT) {
if (submodule &&
submodule->fetch_recurse !=
RECURSE_SUBMODULES_NONE) {
Expand All @@ -675,40 +667,102 @@ int fetch_populated_submodules(const struct argv_array *options,
default_argv = "on-demand";
}
}
} else if (command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {
} else if (spf->command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {
if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
continue;
default_argv = "on-demand";
}

strbuf_addf(&submodule_path, "%s/%s", work_tree, ce->name);
strbuf_addf(&submodule_path, "%s/%s", spf->work_tree, ce->name);
strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
strbuf_addf(&submodule_prefix, "%s%s/", prefix, ce->name);
strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
git_dir = read_gitfile(submodule_git_dir.buf);
if (!git_dir)
git_dir = submodule_git_dir.buf;
if (is_directory(git_dir)) {
if (!quiet)
fprintf(stderr, "Fetching submodule %s%s\n", prefix, ce->name);
cp.dir = submodule_path.buf;
argv_array_push(&argv, default_argv);
argv_array_push(&argv, "--submodule-prefix");
argv_array_push(&argv, submodule_prefix.buf);
cp.argv = argv.argv;
if (run_command(&cp))
result = 1;
argv_array_pop(&argv);
argv_array_pop(&argv);
argv_array_pop(&argv);
child_process_init(cp);
cp->dir = strbuf_detach(&submodule_path, NULL);
cp->env = local_repo_env;
cp->git_cmd = 1;
if (!spf->quiet)
strbuf_addf(err, "Fetching submodule %s%s\n",
spf->prefix, ce->name);
argv_array_init(&cp->args);
argv_array_pushv(&cp->args, spf->args.argv);
argv_array_push(&cp->args, default_argv);
argv_array_push(&cp->args, "--submodule-prefix");
argv_array_push(&cp->args, submodule_prefix.buf);
ret = 1;
}
strbuf_release(&submodule_path);
strbuf_release(&submodule_git_dir);
strbuf_release(&submodule_prefix);
if (ret) {
spf->count++;
return 1;
}
}
argv_array_clear(&argv);
return 0;
}

static int fetch_start_failure(struct child_process *cp,
struct strbuf *err,
void *cb, void *task_cb)
{
struct submodule_parallel_fetch *spf = cb;

spf->result = 1;

return 0;
}

static int fetch_finish(int retvalue, struct child_process *cp,
struct strbuf *err, void *cb, void *task_cb)
{
struct submodule_parallel_fetch *spf = cb;

if (retvalue)
spf->result = 1;

return 0;
}

int fetch_populated_submodules(const struct argv_array *options,
const char *prefix, int command_line_option,
int quiet)
{
int i;
int max_parallel_jobs = 1;
struct submodule_parallel_fetch spf = SPF_INIT;

spf.work_tree = get_git_work_tree();
spf.command_line_option = command_line_option;
spf.quiet = quiet;
spf.prefix = prefix;

if (!spf.work_tree)
goto out;

if (read_cache() < 0)
die("index file corrupt");

argv_array_push(&spf.args, "fetch");
for (i = 0; i < options->argc; i++)
argv_array_push(&spf.args, options->argv[i]);
argv_array_push(&spf.args, "--recurse-submodules-default");
/* default value, "--submodule-prefix" and its value are added later */

calculate_changed_submodule_paths();
run_processes_parallel(max_parallel_jobs,
get_next_submodule,
fetch_start_failure,
fetch_finish,
&spf);

argv_array_clear(&spf.args);
out:
string_list_clear(&changed_submodule_paths, 1);
return result;
return spf.result;
}

unsigned is_submodule_modified(const char *path, int ignore_untracked)
Expand Down

0 comments on commit fe85ee6

Please sign in to comment.