[PATCH] Finish initial cut of git-pack-object/git-unpack-object pair.

This finishes the initial round of git-pack-object /
git-unpack-object pair.  They are now good enough to be used as
a transport medium:

 - Fix delta direction in pack-objects; the original was
   computing delta to create the base object from the object to
   be squashed, which was quite unfriendly for unpacker ;-).

 - Add a script to test the very basics.

 - Implement unpacker for both regular and deltified objects.

Signed-off-by: Junio C Hamano <junkio@cox.net>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

pack-objects.c

@@ -82,7 +82,8 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
 
 	if (!otherbuf)
 		die("unable to read %s", sha1_to_hex(entry->delta->sha1));
-        delta_buf = diff_delta(buf, size, otherbuf, othersize, &delta_size, ~0UL);
+        delta_buf = diff_delta(otherbuf, othersize,
+			       buf, size, &delta_size, ~0UL);
         if (!delta_buf || delta_size != entry->delta_size)
         	die("delta size changed");
         free(buf);
@@ -318,7 +319,8 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
 	max_size = size / 2 - 20;
 	if (cur_entry->delta)
 		max_size = cur_entry->delta_size-1;
-	delta_buf = diff_delta(cur->data, size, old->data, oldsize, &delta_size, max_size);
+	delta_buf = diff_delta(old->data, oldsize,
+			       cur->data, size, &delta_size, max_size);
 	if (!delta_buf)
 		return 0;
 	cur_entry->delta = old_entry;

t/t5300-pack-object.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+
+test_description='git-pack-object
+
+'
+. ./test-lib.sh
+
+TRASH=`pwd`
+
+test_expect_success \
+    'setup' \
+    'rm -f .git/index*
+     for i in a b c
+     do
+	     dd if=/dev/zero bs=4k count=1 | tr "\\0" $i >$i &&
+	     git-update-cache --add $i || exit
+     done &&
+     cat c >d && echo foo >>d && git-update-cache --add d &&
+     tree=`git-write-tree` && {
+	 echo $tree &&
+	 git-ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\)	.*/\\1/"
+     } >obj-list'
+
+test_expect_success \
+    'pack without delta' \
+    'git-pack-objects --window=0 test-1 <obj-list'
+
+rm -fr .git2
+mkdir .git2
+
+test_expect_success \
+    'unpack without delta' \
+    'GIT_OBJECT_DIRECTORY=.git2/objects &&
+     export GIT_OBJECT_DIRECTORY &&
+     git-init-db &&
+     git-unpack-objects test-1'
+
+unset GIT_OBJECT_DIRECTORY
+cd $TRASH/.git2
+
+test_expect_success \
+    'check unpack without delta' \
+    '(cd ../.git && find objects -type f -print) |
+     while read path
+     do
+         cmp $path ../.git/$path || {
+	     echo $path differs.
+	     exit 1
+	 }
+     done'
+cd $TRASH
+
+test_expect_success \
+    'pack with delta' \
+    'pwd &&
+     git-pack-objects test-2 <obj-list'
+
+rm -fr .git2
+mkdir .git2
+
+test_expect_success \
+    'unpack with delta' \
+    'GIT_OBJECT_DIRECTORY=.git2/objects &&
+     export GIT_OBJECT_DIRECTORY &&
+     git-init-db &&
+     git-unpack-objects test-2'
+
+unset GIT_OBJECT_DIRECTORY
+cd $TRASH/.git2
+test_expect_success \
+    'check unpack with delta' \
+    '(cd ../.git && find objects -type f -print) |
+     while read path
+     do
+         cmp $path ../.git/$path || {
+	     echo $path differs.
+	     exit 1
+	 }
+     done'
+cd $TRASH
+
+test_done

unpack-objects.c

@@ -1,18 +1,21 @@
 #include "cache.h"
 #include "object.h"
+#include "delta.h"
 
 static int dry_run;
 static int nr_entries;
 static const char *base_name;
 static const char unpack_usage[] = "git-unpack-objects basename";
 
 struct pack_entry {
-	unsigned int offset;
+	unsigned int offset; /* network byte order */
 	unsigned char sha1[20];
 };
 
 static void *pack_base;
 static unsigned long pack_size;
+static void *index_base;
+static unsigned long index_size;
 
 static struct pack_entry **pack_list;
 
@@ -52,13 +55,13 @@ static int sort_by_offset(const void *_a, const void *_b)
 	return o1 < o2 ? -1 : 1;
 }
 
-static int check_index(void *index, unsigned long idx_size)
+static int check_index(void)
 {
-	unsigned int *array = index;
+	unsigned int *array = index_base;
 	unsigned int nr;
 	int i;
 
-	if (idx_size < 4*256)
+	if (index_size < 4*256)
 		return error("index file too small");
 	nr = 0;
 	for (i = 0; i < 256; i++) {
@@ -67,25 +70,182 @@ static int check_index(void *index, unsigned long idx_size)
 			return error("non-monotonic index");
 		nr = n;
 	}
-	if (idx_size != 4*256 + nr * 24) {
-		printf("idx_size=%lu, expected %u (%u)\n", idx_size, 4*256 + nr * 24, nr);
+	if (index_size != 4*256 + nr * 24) {
+		printf("index_size=%lu, expected %u (%u)\n",
+		       index_size, 4*256 + nr * 24, nr);
 		return error("wrong index file size");
 	}
 
 	nr_entries = nr;
 	pack_list = xmalloc(nr * sizeof(struct pack_entry *));
 	for (i = 0; i < nr; i++)
-		pack_list[i] = index + 4*256 + i*24;
+		pack_list[i] = index_base + 4*256 + i*24;
 
 	qsort(pack_list, nr, sizeof(*pack_list), sort_by_offset);
 
 	printf("%d entries\n", nr);
 	return 0;
 }
 
-static void unpack_entry(struct pack_entry *entry)
+static int unpack_non_delta_entry(struct pack_entry *entry,
+				  unsigned char *pack)
 {
+	int st, kind;
 	unsigned long size;
+	z_stream stream;
+	char *buffer;
+	unsigned char sha1[20];
+	char *type_s;
+	unsigned long offset = ntohl(entry->offset);
+
+	kind = pack[0];
+	size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
+	printf("%s %c %lu\n", sha1_to_hex(entry->sha1), kind, size);
+	pack += 5;
+
+	buffer = xmalloc(size + 1);
+	buffer[size] = 0;
+	memset(&stream, 0, sizeof(stream));
+	stream.next_in = pack;
+	stream.avail_in = pack_size - offset; /* sheesh. */
+	stream.next_out = buffer;
+	stream.avail_out = size;
+
+	inflateInit(&stream);
+	st = inflate(&stream, Z_FINISH);
+	inflateEnd(&stream);
+	if ((st != Z_STREAM_END) || stream.total_out != size)
+		goto err_finish;
+	switch (kind) {
+	case 'C': type_s = "commit"; break;
+	case 'T': type_s = "tree"; break;
+	case 'B': type_s = "blob"; break;
+	default: goto err_finish;
+	}
+	if (write_sha1_file(buffer, size, type_s, sha1) < 0)
+		die("failed to write %s (%s)",
+		    sha1_to_hex(entry->sha1), type_s);
+	printf("%s %s\n", sha1_to_hex(sha1), type_s);
+	if (memcmp(sha1, entry->sha1, 20))
+		die("resulting %s have wrong SHA1", type_s);
+
+ finish:
+	st = 0;
+	free(buffer);
+	return st;
+ err_finish:
+	st = -1;
+	goto finish;
+}
+
+static int find_pack_entry(unsigned char *sha1, struct pack_entry **ent)
+{
+	int *level1_ofs = index_base;
+	int hi = ntohl(level1_ofs[*sha1]);
+	int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
+	void *index = index_base + 4*256;
+
+	do {
+		int mi = (lo + hi) / 2;
+		int cmp = memcmp(index + 24 * mi + 4, sha1, 20);
+		if (!cmp) {
+			*ent = index + 24 * mi;
+			return 1;
+		}
+		if (cmp < 0)
+			hi = mi;
+		else
+			lo = mi;
+	} while (lo < hi);
+	return 0;
+}
+
+/* forward declaration for a mutually recursive function */
+static void unpack_entry(struct pack_entry *);
+
+static int unpack_delta_entry(struct pack_entry *entry, unsigned char *pack)
+{
+	void *delta_data, *result, *base;
+	unsigned long delta_alloc, delta_size, result_size, base_size;
+	z_stream stream;
+	int st;
+	char type[20];
+	unsigned char sha1[20];
+
+	printf("%s D", sha1_to_hex(entry->sha1));
+	printf(" %s\n", sha1_to_hex(pack+1));
+
+	/* pack+1 is the base sha1, unless we have it, we need to
+	 * unpack it first.
+	 */
+	if (!has_sha1_file(pack+1)) {
+		struct pack_entry *base;
+		if (!find_pack_entry(pack+1, &base))
+			die("cannot find delta-pack base object");
+		unpack_entry(base);
+	}
+
+	/* pack+1 thru pack+20 is the base sha1 and
+	 * pack+21 thru unknown number is the delta data.
+	 * we do not even have size of the delta data uncompressed.
+	 * sheesh!
+	 */
+	delta_alloc = 1024;
+	delta_data = xmalloc(delta_alloc);
+
+	memset(&stream, 0, sizeof(stream));
+
+	stream.next_in = pack + 21;
+	stream.avail_in = pack_size - ntohl(entry->offset); /* sheesh. */
+	stream.next_out = delta_data;
+	stream.avail_out = delta_alloc;
+	delta_size = 0;
+
+	inflateInit(&stream);
+	while (1) {
+		st = inflate(&stream, Z_FINISH);
+		if (st == Z_STREAM_END) {
+			delta_size = stream.total_out;
+			break;
+		}
+		if (st < 0)
+			break;
+
+		if (delta_alloc <= stream.total_out) {
+			delta_alloc = (delta_alloc +1024) * 3 / 2;
+			delta_data = xrealloc(delta_data, delta_alloc);
+			stream.next_out = delta_data + stream.total_out;
+			stream.avail_out = delta_alloc - stream.total_out;
+		}
+	}
+	inflateEnd(&stream);
+	if (st != Z_STREAM_END) {
+		free(delta_data);
+		return -1;
+	}
+
+	base = read_sha1_file(pack+1, type, &base_size);
+	if (!base)
+		die("failed to read delta-pack base object");
+	result = patch_delta(base, base_size,
+			     delta_data, delta_size,
+			     &result_size);
+	if (!result)
+		die("failed to apply delta");
+	free(delta_data);
+
+	if (write_sha1_file(result, result_size, type, sha1) < 0)
+		die("failed to write %s (%s)",
+		    sha1_to_hex(entry->sha1), type);
+	free(result);
+	printf("%s %s\n", sha1_to_hex(sha1), type);
+	if (memcmp(sha1, entry->sha1, 20))
+		die("resulting %s have wrong SHA1", type);
+	return 0;
+}
+
+static void unpack_entry(struct pack_entry *entry)
+{
 	unsigned long offset;
 	unsigned char *pack;
 
@@ -100,12 +260,10 @@ static void unpack_entry(struct pack_entry *entry)
 	offset = pack_size - offset;
 	switch (*pack) {
 	case 'C': case 'T': case 'B':
-		size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
-		printf("%s %c %lu\n", sha1_to_hex(entry->sha1), *pack, size);
+		unpack_non_delta_entry(entry, pack);
 		break;
 	case 'D':
-		printf("%s D", sha1_to_hex(entry->sha1));
-		printf(" %s\n", sha1_to_hex(pack+1));
+		unpack_delta_entry(entry, pack);
 		break;
 	default:
 		die("corrupted pack file");
@@ -130,8 +288,6 @@ static void unpack_all(void)
 int main(int argc, char **argv)
 {
 	int i;
-	unsigned long idx_size;
-	void *index;
 
 	for (i = 1 ; i < argc; i++) {
 		const char *arg = argv[i];
@@ -149,9 +305,9 @@ int main(int argc, char **argv)
 	}
 	if (!base_name)
 		usage(unpack_usage);
-	index = map_file("idx", &idx_size);
+	index_base = map_file("idx", &index_size);
 	pack_base = map_file("pack", &pack_size);
-	if (check_index(index, idx_size) < 0)
+	if (check_index() < 0)
 		die("bad index file");
 	unpack_all();
 	return 0;