From e74435a5169b56be901196ad172b4dbda124254d Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 24 Oct 2013 13:59:49 -0400 Subject: [PATCH 1/3] sha1write: make buffer const-correct We are passed a "void *" and write it out without ever touching it; let's indicate that by using "const". Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- csum-file.c | 6 +++--- csum-file.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csum-file.c b/csum-file.c index 53f5375b6..465971c7f 100644 --- a/csum-file.c +++ b/csum-file.c @@ -11,7 +11,7 @@ #include "progress.h" #include "csum-file.h" -static void flush(struct sha1file *f, void *buf, unsigned int count) +static void flush(struct sha1file *f, const void *buf, unsigned int count) { if (0 <= f->check_fd && count) { unsigned char check_buffer[8192]; @@ -86,13 +86,13 @@ int sha1close(struct sha1file *f, unsigned char *result, unsigned int flags) return fd; } -int sha1write(struct sha1file *f, void *buf, unsigned int count) +int sha1write(struct sha1file *f, const void *buf, unsigned int count) { while (count) { unsigned offset = f->offset; unsigned left = sizeof(f->buffer) - offset; unsigned nr = count > left ? left : count; - void *data; + const void *data; if (f->do_crc) f->crc32 = crc32(f->crc32, buf, nr); diff --git a/csum-file.h b/csum-file.h index 3b540bdc2..9dedb038e 100644 --- a/csum-file.h +++ b/csum-file.h @@ -34,7 +34,7 @@ extern struct sha1file *sha1fd(int fd, const char *name); extern struct sha1file *sha1fd_check(const char *name); extern struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp); extern int sha1close(struct sha1file *, unsigned char *, unsigned int); -extern int sha1write(struct sha1file *, void *, unsigned int); +extern int sha1write(struct sha1file *, const void *, unsigned int); extern void sha1flush(struct sha1file *f); extern void crc32_begin(struct sha1file *); extern uint32_t crc32_end(struct sha1file *); From 1190a1acf800acdcfd7569f87ac1560e2d077414 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 5 Dec 2013 15:28:07 -0500 Subject: [PATCH 2/3] pack-objects: name pack files after trailer hash Our current scheme for naming packfiles is to calculate the sha1 hash of the sorted list of objects contained in the packfile. This gives us a unique name, so we are reasonably sure that two packs with the same name will contain the same objects. It does not, however, tell us that two such packs have the exact same bytes. This makes things awkward if we repack the same set of objects. Due to run-to-run variations, the bytes may not be identical (e.g., changed zlib or git versions, different source object reuse due to new packs in the repository, or even different deltas due to races during a multi-threaded delta search). In theory, this could be helpful to a program that cares that the packfile contains a certain set of objects, but does not care about the particular representation. In practice, no part of git makes use of that, and in many cases it is potentially harmful. For example, if a dumb http client fetches the .idx file, it must be sure to get the exact .pack that matches it. Similarly, a partial transfer of a .pack file cannot be safely resumed, as the actual bytes may have changed. This could also affect a local client which opened the .idx and .pack files, closes the .pack file (due to memory or file descriptor limits), and then re-opens a changed packfile. In all of these cases, git can detect the problem, as we have the sha1 of the bytes themselves in the pack trailer (which we verify on transfer), and the .idx file references the trailer from the matching packfile. But it would be simpler and more efficient to actually get the correct bytes, rather than noticing the problem and having to restart the operation. This patch simply uses the pack trailer sha1 as the pack name. It should be similarly unique, but covers the exact representation of the objects. Other parts of git should not care, as the pack name is returned by pack-objects and is essentially opaque. One test needs to be updated, because it actually corrupts a pack and expects that re-packing the corrupted bytes will use the same name. It won't anymore, but we can easily just use the name that pack-objects hands back. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- pack-write.c | 8 +------- pack.h | 2 +- t/t5302-pack-index.sh | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pack-write.c b/pack-write.c index ca9e63be1..ddc174e1a 100644 --- a/pack-write.c +++ b/pack-write.c @@ -44,14 +44,13 @@ static int need_large_offset(off_t offset, const struct pack_idx_option *opts) */ const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, const struct pack_idx_option *opts, - unsigned char *sha1) + const unsigned char *sha1) { struct sha1file *f; struct pack_idx_entry **sorted_by_sha, **list, **last; off_t last_obj_offset = 0; uint32_t array[256]; int i, fd; - git_SHA_CTX ctx; uint32_t index_version; if (nr_objects) { @@ -114,9 +113,6 @@ const char *write_idx_file(const char *index_name, struct pack_idx_entry **objec } sha1write(f, array, 256 * 4); - /* compute the SHA1 hash of sorted object names. */ - git_SHA1_Init(&ctx); - /* * Write the actual SHA1 entries.. */ @@ -128,7 +124,6 @@ const char *write_idx_file(const char *index_name, struct pack_idx_entry **objec sha1write(f, &offset, 4); } sha1write(f, obj->sha1, 20); - git_SHA1_Update(&ctx, obj->sha1, 20); if ((opts->flags & WRITE_IDX_STRICT) && (i && !hashcmp(list[-2]->sha1, obj->sha1))) die("The same object %s appears twice in the pack", @@ -178,7 +173,6 @@ const char *write_idx_file(const char *index_name, struct pack_idx_entry **objec sha1write(f, sha1, 20); sha1close(f, NULL, ((opts->flags & WRITE_IDX_VERIFY) ? CSUM_CLOSE : CSUM_FSYNC)); - git_SHA1_Final(sha1, &ctx); return index_name; } diff --git a/pack.h b/pack.h index aa6ee7d60..12d951659 100644 --- a/pack.h +++ b/pack.h @@ -76,7 +76,7 @@ struct pack_idx_entry { struct progress; typedef int (*verify_fn)(const unsigned char*, enum object_type, unsigned long, void*, int*); -extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, const struct pack_idx_option *, unsigned char *sha1); +extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, const struct pack_idx_option *, const unsigned char *sha1); extern int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, off_t offset, off_t len, unsigned int nr); extern int verify_pack_index(struct packed_git *); extern int verify_pack(struct packed_git *, verify_fn fn, struct progress *, uint32_t); diff --git a/t/t5302-pack-index.sh b/t/t5302-pack-index.sh index fe82025d4..4bbb71875 100755 --- a/t/t5302-pack-index.sh +++ b/t/t5302-pack-index.sh @@ -174,11 +174,11 @@ test_expect_success \ test_expect_success \ '[index v1] 5) pack-objects happily reuses corrupted data' \ 'pack4=$(git pack-objects test-4 Date: Mon, 16 Dec 2013 11:19:33 -0800 Subject: [PATCH 3/3] pack-objects doc: treat output filename as opaque After 1190a1a (pack-objects: name pack files after trailer hash, 2013-12-05), the SHA-1 used to determine the filename is calculated differently. Update the documentation to not guarantee anything more than that the SHA-1 depends on the pack content somehow. Hopefully this will discourage readers from depending on the old or the new calculation. Reported-by: Michael Haggerty Signed-off-by: Jeff King Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- Documentation/git-pack-objects.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Documentation/git-pack-objects.txt b/Documentation/git-pack-objects.txt index d94edcd4b..cdab9ed50 100644 --- a/Documentation/git-pack-objects.txt +++ b/Documentation/git-pack-objects.txt @@ -51,8 +51,7 @@ base-name:: to determine the name of the created file. When this option is used, the two files are written in -.{pack,idx} files. is a hash - of the sorted object names to make the resulting filename - based on the pack content, and written to the standard + based on the pack content and is written to the standard output of the command. --stdout::