From fcfa32b9e10b3c0373a263a732146850bdf67242 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 2 Nov 2005 23:51:57 +0200 Subject: [PATCH 1/8] Use svn pools to solve the memory leak problem. Signed-off-by: Yaacov Akiba Slama Signed-off-by: Junio C Hamano --- git-svnimport.perl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/git-svnimport.perl b/git-svnimport.perl index 45b6a1986..5bf9ef21e 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -112,7 +112,9 @@ sub file { DIR => File::Spec->tmpdir(), UNLINK => 1); print "... $rev $path ...\n" if $opt_v; - eval { $self->{'svn'}->get_file($path,$rev,$fh); }; + my $pool = SVN::Pool->new(); + eval { $self->{'svn'}->get_file($path,$rev,$fh,$pool); }; + $pool->clear; if($@) { return undef if $@ =~ /Attempted to get checksum/; die $@; @@ -674,7 +676,9 @@ sub commit_all { } while(++$current_rev <= $svn->{'maxrev'}) { - $svn->{'svn'}->get_log("/",$current_rev,$current_rev,$current_rev,1,1,\&_commit_all,""); + my $pool=SVN::Pool->new; + $svn->{'svn'}->get_log("/",$current_rev,$current_rev,1,1,1,\&_commit_all,$pool); + $pool->clear; commit_all(); if($opt_l and not --$opt_l) { print STDERR "Stopping, because there is a memory leak (in the SVN library).\n"; From cbce5d8961f7790a621f7cee6ecb2c25ae372867 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 2 Nov 2005 23:51:57 +0200 Subject: [PATCH 2/8] Add node_kind function to differentiate between file and directory Signed-off-by: Yaacov Akiba Slama Signed-off-by: Junio C Hamano --- git-svnimport.perl | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/git-svnimport.perl b/git-svnimport.perl index 5bf9ef21e..83b70f9bb 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -260,10 +260,17 @@ ($$) open BRANCHES,">>", "$git_dir/svn2git"; -sub get_file($$$) { - my($rev,$branch,$path) = @_; +sub node_kind($$$) { + my ($branch, $path, $revision) = @_; + my $pool=SVN::Pool->new; + my $kind = $svn->{'svn'}->check_path(revert_split_path($branch,$path),$revision,$pool); + $pool->clear; + return $kind; +} + +sub revert_split_path($$) { + my($branch,$path) = @_; - # revert split_path(), below my $svnpath; $path = "" if $path eq "/"; # this should not happen, but ... if($branch eq "/") { @@ -274,6 +281,14 @@ ($$$) $svnpath = "$branch_name/$branch/$path"; } + return $svnpath +} + +sub get_file($$$) { + my($rev,$branch,$path) = @_; + + my $svnpath = revert_split_path($branch,$path); + # now get it my $name; if($opt_d) { From c2c07a5c2adf2aebc19b04a608592489b156a8bb Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 2 Nov 2005 23:51:57 +0200 Subject: [PATCH 3/8] Don't output error on changes in the nodes /, /tags or /branches Signed-off-by: Junio C Hamano --- git-svnimport.perl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git-svnimport.perl b/git-svnimport.perl index 83b70f9bb..ea5bbdb8a 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -336,7 +336,12 @@ ($$) } elsif($path =~ s#^/\Q$branch_name\E/([^/]+)/?##) { $branch = $1; } else { - print STDERR "$rev: Unrecognized path: $path\n"; + my %no_error = ( + "/" => 1, + "/$tag_name" => 1, + "/$branch_name" => 1 + ); + print STDERR "$rev: Unrecognized path: $path\n" unless (defined $no_error{$path}); return () } $path = "/" if $path eq ""; From 8168373fe7e6214205fd23e519f72c3f1bbb7080 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 2 Nov 2005 23:51:57 +0200 Subject: [PATCH 4/8] copy_dir becomes copy_path and handles both files and directories The A (Add) and R (Replace) actions handling are unified. Signed-off-by: Yaacov Akiba Slama Signed-off-by: Junio C Hamano --- git-svnimport.perl | 90 +++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/git-svnimport.perl b/git-svnimport.perl index ea5bbdb8a..9cee629f1 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -348,21 +348,42 @@ ($$) return ($branch,$path); } -sub copy_subdir($$$$$$) { +sub branch_rev($$) { + + my ($srcbranch,$uptorev) = @_; + + my $bbranches = $branches{$srcbranch}; + my @revs = reverse sort { ($a eq 'LAST' ? 0 : $a) <=> ($b eq 'LAST' ? 0 : $b) } keys %$bbranches; + my $therev; + foreach my $arev(@revs) { + next if ($arev eq 'LAST'); + if ($arev <= $uptorev) { + $therev = $arev; + last; + } + } + return $therev; +} + +sub copy_path($$$$$$$) { # Somebody copied a whole subdirectory. # We need to find the index entries from the old version which the # SVN log entry points to, and add them to the new place. - my($newrev,$newbranch,$path,$oldpath,$rev,$new) = @_; - my($branch,$srcpath) = split_path($rev,$oldpath); + my($newrev,$newbranch,$path,$oldpath,$rev,$node_kind,$new) = @_; - my $gitrev = $branches{$branch}{$rev}; + my($srcbranch,$srcpath) = split_path($rev,$oldpath); + my $therev = branch_rev($srcbranch, $rev); + my $gitrev = $branches{$srcbranch}{$therev}; unless($gitrev) { print STDERR "$newrev:$newbranch: could not find $oldpath \@ $rev\n"; return; } - print "$newrev:$newbranch:$path: copying from $branch:$srcpath @ $rev\n" if $opt_v; - $srcpath =~ s#/*$#/#; + print "$newrev:$newbranch:$path: copying from $srcbranch:$srcpath @ $rev\n" if $opt_v; + if ($node_kind eq $SVN::Node::dir) { + $srcpath =~ s#/*$#/#; + } + open my $f,"-|","git-ls-tree","-r","-z",$gitrev,$srcpath; local $/ = "\0"; while(<$f>) { @@ -370,9 +391,12 @@ ($$$$$$) my($m,$p) = split(/\t/,$_,2); my($mode,$type,$sha1) = split(/ /,$m); next if $type ne "blob"; - $p = substr($p,length($srcpath)-1); - print "... found $path$p ...\n" if $opt_v; - push(@$new,[$mode,$sha1,$path.$p]); + if ($node_kind eq $SVN::Node::dir) { + $p = $path . substr($p,length($srcpath)-1); + } else { + $p = $path; + } + push(@$new,[$mode,$sha1,$p]); } close($f) or print STDERR "$newrev:$newbranch: could not list files in $oldpath \@ $rev\n"; @@ -476,39 +500,31 @@ sub commit { foreach my $path(@paths) { my $action = $changed_paths->{$path}; - if ($action->[0] eq "A") { - my $f = get_file($revision,$branch,$path); - if($f) { - push(@new,$f) if $f; - } elsif($action->[1]) { - copy_subdir($revision,$branch,$path,$action->[1],$action->[2],\@new); - } else { - my $opath = $action->[3]; - print STDERR "$revision: $branch: could not fetch '$opath'\n"; + if ($action->[0] eq "R") { + # refer to a file/tree in an earlier commit + push(@old,$path); # remove any old stuff + } + if(($action->[0] eq "A") || ($action->[0] eq "R")) { + my $node_kind = node_kind($branch,$path,$revision); + if($action->[1]) { + copy_path($revision,$branch,$path,$action->[1],$action->[2],$node_kind,\@new); + } elsif ($node_kind eq $SVN::Node::file) { + my $f = get_file($revision,$branch,$path); + if ($f) { + push(@new,$f) if $f; + } else { + my $opath = $action->[3]; + print STDERR "$revision: $branch: could not fetch '$opath'\n"; + } } } elsif ($action->[0] eq "D") { push(@old,$path); } elsif ($action->[0] eq "M") { - my $f = get_file($revision,$branch,$path); - push(@new,$f) if $f; - } elsif ($action->[0] eq "R") { - # refer to a file/tree in an earlier commit - push(@old,$path); # remove any old stuff - - # ... and add any new stuff - my($b,$srcpath) = split_path($revision,$action->[1]); - $srcpath =~ s#/*$#/#; - open my $F,"-|","git-ls-tree","-r","-z", $branches{$b}{$action->[2]}, $srcpath; - local $/ = "\0"; - while(<$F>) { - chomp; - my($m,$p) = split(/\t/,$_,2); - my($mode,$type,$sha1) = split(/ /,$m); - next if $type ne "blob"; - $p = substr($p,length($srcpath)-1); - push(@new,[$mode,$sha1,$path.$p]); + my $node_kind = node_kind($branch,$path,$revision); + if ($node_kind eq $SVN::Node::file) { + my $f = get_file($revision,$branch,$path); + push(@new,$f) if $f; } - close($F); } else { die "$revision: unknown action '".$action->[0]."' for $path\n"; } From 109fc2b97b73090a4a0a6550cdf9b2446fd12389 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 2 Nov 2005 23:51:57 +0200 Subject: [PATCH 5/8] Bundle file copies from multiple branches into a merge. When copying files and/or directories from several branches in one single revision, all these branches are used as parents of the commit. Signed-off-by: Yaacov Akiba Slama Signed-off-by: Junio C Hamano --- git-svnimport.perl | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/git-svnimport.perl b/git-svnimport.perl index 9cee629f1..ae82a7c09 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -365,12 +365,12 @@ ($$) return $therev; } -sub copy_path($$$$$$$) { +sub copy_path($$$$$$$$) { # Somebody copied a whole subdirectory. # We need to find the index entries from the old version which the # SVN log entry points to, and add them to the new place. - my($newrev,$newbranch,$path,$oldpath,$rev,$node_kind,$new) = @_; + my($newrev,$newbranch,$path,$oldpath,$rev,$node_kind,$new,$parents) = @_; my($srcbranch,$srcpath) = split_path($rev,$oldpath); my $therev = branch_rev($srcbranch, $rev); @@ -379,6 +379,9 @@ ($$$$$$$) print STDERR "$newrev:$newbranch: could not find $oldpath \@ $rev\n"; return; } + if ($srcbranch ne $newbranch) { + push(@$parents, $branches{$srcbranch}{'LAST'}); + } print "$newrev:$newbranch:$path: copying from $srcbranch:$srcpath @ $rev\n" if $opt_v; if ($node_kind eq $SVN::Node::dir) { $srcpath =~ s#/*$#/#; @@ -405,7 +408,7 @@ ($$$$$$$) sub commit { my($branch, $changed_paths, $revision, $author, $date, $message) = @_; my($author_name,$author_email,$dest); - my(@old,@new); + my(@old,@new,@parents); if (not defined $author) { $author_name = $author_email = "unknown"; @@ -492,6 +495,8 @@ sub commit { $last_rev = $rev; } + push (@parents, $rev) if defined $rev; + my $cid; if($tag and not %$changed_paths) { $cid = $rev; @@ -507,7 +512,7 @@ sub commit { if(($action->[0] eq "A") || ($action->[0] eq "R")) { my $node_kind = node_kind($branch,$path,$revision); if($action->[1]) { - copy_path($revision,$branch,$path,$action->[1],$action->[2],$node_kind,\@new); + copy_path($revision,$branch,$path,$action->[1],$action->[2],$node_kind,\@new,\@parents); } elsif ($node_kind eq $SVN::Node::file) { my $f = get_file($revision,$branch,$path); if ($f) { @@ -592,7 +597,6 @@ sub commit { $pw->close(); my @par = (); - @par = ("-p",$rev) if defined $rev; # loose detection of merges # based on the commit msg @@ -602,11 +606,17 @@ sub commit { if ($mparent eq 'HEAD') { $mparent = $opt_o }; if ( -e "$git_dir/refs/heads/$mparent") { $mparent = get_headref($mparent, $git_dir); - push @par, '-p', $mparent; + push (@parents, $mparent); print OUT "Merge parent branch: $mparent\n" if $opt_v; } } } + my %seen_parents = (); + my @unique_parents = grep { ! $seen_parents{$_} ++ } @parents; + foreach my $bparent (@unique_parents) { + push @par, '-p', $bparent; + print OUT "Merge parent branch: $bparent\n" if $opt_v; + } exec("env", "GIT_AUTHOR_NAME=$author_name", From a16db4f472887a9050bf528bc7f0b94d6f7e8319 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 2 Nov 2005 23:51:57 +0200 Subject: [PATCH 6/8] Handle a revision that only creates a new tag correctly. Fix an error when a svn revision consists only of the creation of a new tag directory (/tags/this_is_a_tag). Signed-off-by: Yaacov Akiba Slama Signed-off-by: Junio C Hamano --- git-svnimport.perl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/git-svnimport.perl b/git-svnimport.perl index ae82a7c09..e97f47059 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -648,6 +648,10 @@ sub commit { die "Error running git-commit-tree: $?\n" if $?; } + if (not defined $cid) { + $cid = $branches{"/"}{"LAST"}; + } + if(not defined $dest) { print "... no known parent\n" if $opt_v; } elsif(not $tag) { @@ -664,6 +668,7 @@ sub commit { # the tag was 'complex', i.e. did not refer to a "real" revision $dest =~ tr/_/\./ if $opt_u; + $branch = $dest; my $pid = open2($in, $out, 'git-mktag'); print $out ("object $cid\n". From 01d4f0e7758f0fa837eec18a901a668f816cfc5f Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 2 Nov 2005 23:51:58 +0200 Subject: [PATCH 7/8] Lift the default limit of number of revs. Now that the leak is gone, there is by default no limit of revisions to import. No more message about leak when the limit (given by the -l parameter) is reached. Signed-off-by: Yaacov Akiba Slama Signed-off-by: Junio C Hamano --- git-svnimport.perl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/git-svnimport.perl b/git-svnimport.perl index e97f47059..ab690f364 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -53,7 +53,6 @@ END $opt_o ||= "origin"; $opt_s ||= 1; -$opt_l = 100 unless defined $opt_l; my $git_tree = $opt_C; $git_tree ||= "."; @@ -727,15 +726,16 @@ sub commit_all { } while(++$current_rev <= $svn->{'maxrev'}) { + if (defined $opt_l) { + $opt_l--; + if ($opt_l < 0) { + last; + } + } my $pool=SVN::Pool->new; $svn->{'svn'}->get_log("/",$current_rev,$current_rev,1,1,1,\&_commit_all,$pool); $pool->clear; commit_all(); - if($opt_l and not --$opt_l) { - print STDERR "Stopping, because there is a memory leak (in the SVN library).\n"; - print STDERR "Please repeat this command; it will continue safely\n"; - last; - } } From f3ad062560bfd4a2983122de93d08f43ea66e7cf Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 4 Nov 2005 23:30:12 -0800 Subject: [PATCH 8/8] Fix a couple of obvious and insignificant typo. Signed-off-by: Junio C Hamano --- git-svnimport.perl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git-svnimport.perl b/git-svnimport.perl index ab690f364..cb9afb955 100755 --- a/git-svnimport.perl +++ b/git-svnimport.perl @@ -25,7 +25,7 @@ use SVN::Core; use SVN::Ra; -die "Need CVN:Core 1.2.1 or better" if $SVN::Core::VERSION lt "1.2.1"; +die "Need SVN:Core 1.2.1 or better" if $SVN::Core::VERSION lt "1.2.1"; $SIG{'PIPE'}="IGNORE"; $ENV{'TZ'}="UTC"; @@ -34,7 +34,7 @@ sub usage() { print STDERR <