From 2162bd8cc461d6c3a12ab81c5db5a44bf5ecabc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Tue, 4 Sep 2012 22:23:38 +0200 Subject: [PATCH 1/4] archive-zip: support UTF-8 paths Set general purpose flag 11 if we encounter a path that contains non-ASCII characters. We assume that all paths are given as UTF-8; no conversion is done. The flag seems to be ignored by unzip unless we also mark the archive entry as coming from a Unix system. This is done by setting the field creator_version ("version made by" in the standard[1]) to 0x03NN. The NN part represents the version of the standard supported by us, and this patch sets it to 3f (for version 6.3) for Unix paths. We keep creator_version set to 0 (FAT filesystem, standard version 0) in the non-special cases, as before. But when we declare a file to have a Unix path, then we have to set the file mode as well, or unzip will extract the files with the permission set 0000, i.e. inaccessible by all. [1] http://www.pkware.com/documents/casestudies/APPNOTE.TXT Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- archive-zip.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/archive-zip.c b/archive-zip.c index f5af81f90..928da1d79 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -4,6 +4,8 @@ #include "cache.h" #include "archive.h" #include "streaming.h" +#include "commit.h" +#include "utf8.h" static int zip_date; static int zip_time; @@ -16,7 +18,8 @@ static unsigned int zip_dir_offset; static unsigned int zip_dir_entries; #define ZIP_DIRECTORY_MIN_SIZE (1024 * 1024) -#define ZIP_STREAM (8) +#define ZIP_STREAM (1 << 3) +#define ZIP_UTF8 (1 << 11) struct zip_local_header { unsigned char magic[4]; @@ -173,7 +176,8 @@ static int write_zip_entry(struct archiver_args *args, { struct zip_local_header header; struct zip_dir_header dirent; - unsigned long attr2; + unsigned int creator_version = 0; + unsigned long attr2 = 0; unsigned long compressed_size; unsigned long crc; unsigned long direntsize; @@ -187,6 +191,13 @@ static int write_zip_entry(struct archiver_args *args, crc = crc32(0, NULL, 0); + if (has_non_ascii(path)) { + if (is_utf8(path)) + flags |= ZIP_UTF8; + else + warning("Path is not valid UTF-8: %s", path); + } + if (pathlen > 0xffff) { return error("path too long (%d chars, SHA1: %s): %s", (int)pathlen, sha1_to_hex(sha1), path); @@ -204,10 +215,15 @@ static int write_zip_entry(struct archiver_args *args, enum object_type type = sha1_object_info(sha1, &size); method = 0; - attr2 = S_ISLNK(mode) ? ((mode | 0777) << 16) : - (mode & 0111) ? ((mode) << 16) : 0; if (S_ISREG(mode) && args->compression_level != 0 && size > 0) method = 8; + if (S_ISLNK(mode) || (mode & 0111) || (flags & ZIP_UTF8)) { + creator_version = 0x033f; + attr2 = mode; + if (S_ISLNK(mode)) + attr2 |= 0777; + attr2 <<= 16; + } compressed_size = size; if (S_ISREG(mode) && type == OBJ_BLOB && !args->convert && @@ -254,8 +270,7 @@ static int write_zip_entry(struct archiver_args *args, } copy_le32(dirent.magic, 0x02014b50); - copy_le16(dirent.creator_version, - S_ISLNK(mode) || (S_ISREG(mode) && (mode & 0111)) ? 0x0317 : 0); + copy_le16(dirent.creator_version, creator_version); copy_le16(dirent.version, 10); copy_le16(dirent.flags, flags); copy_le16(dirent.compression_method, method); From bb52d22ebbc6d8792d3a016e6e89fd6e39c7a39f Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 18 Sep 2012 13:32:39 -0700 Subject: [PATCH 2/4] Revert "archive-zip: support UTF-8 paths" This reverts commit 2162bd8cc461d6c3a12ab81c5db5a44bf5ecabc3; a two-patch series to replace it will follow. --- archive-zip.c | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/archive-zip.c b/archive-zip.c index 928da1d79..f5af81f90 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -4,8 +4,6 @@ #include "cache.h" #include "archive.h" #include "streaming.h" -#include "commit.h" -#include "utf8.h" static int zip_date; static int zip_time; @@ -18,8 +16,7 @@ static unsigned int zip_dir_offset; static unsigned int zip_dir_entries; #define ZIP_DIRECTORY_MIN_SIZE (1024 * 1024) -#define ZIP_STREAM (1 << 3) -#define ZIP_UTF8 (1 << 11) +#define ZIP_STREAM (8) struct zip_local_header { unsigned char magic[4]; @@ -176,8 +173,7 @@ static int write_zip_entry(struct archiver_args *args, { struct zip_local_header header; struct zip_dir_header dirent; - unsigned int creator_version = 0; - unsigned long attr2 = 0; + unsigned long attr2; unsigned long compressed_size; unsigned long crc; unsigned long direntsize; @@ -191,13 +187,6 @@ static int write_zip_entry(struct archiver_args *args, crc = crc32(0, NULL, 0); - if (has_non_ascii(path)) { - if (is_utf8(path)) - flags |= ZIP_UTF8; - else - warning("Path is not valid UTF-8: %s", path); - } - if (pathlen > 0xffff) { return error("path too long (%d chars, SHA1: %s): %s", (int)pathlen, sha1_to_hex(sha1), path); @@ -215,15 +204,10 @@ static int write_zip_entry(struct archiver_args *args, enum object_type type = sha1_object_info(sha1, &size); method = 0; + attr2 = S_ISLNK(mode) ? ((mode | 0777) << 16) : + (mode & 0111) ? ((mode) << 16) : 0; if (S_ISREG(mode) && args->compression_level != 0 && size > 0) method = 8; - if (S_ISLNK(mode) || (mode & 0111) || (flags & ZIP_UTF8)) { - creator_version = 0x033f; - attr2 = mode; - if (S_ISLNK(mode)) - attr2 |= 0777; - attr2 <<= 16; - } compressed_size = size; if (S_ISREG(mode) && type == OBJ_BLOB && !args->convert && @@ -270,7 +254,8 @@ static int write_zip_entry(struct archiver_args *args, } copy_le32(dirent.magic, 0x02014b50); - copy_le16(dirent.creator_version, creator_version); + copy_le16(dirent.creator_version, + S_ISLNK(mode) || (S_ISREG(mode) && (mode & 0111)) ? 0x0317 : 0); copy_le16(dirent.version, 10); copy_le16(dirent.flags, flags); copy_le16(dirent.compression_method, method); From 88182bab001a9b9e9b0acd4b888693fa7c28ff4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Tue, 18 Sep 2012 21:46:56 +0200 Subject: [PATCH 3/4] archive-zip: support UTF-8 paths Set general purpose flag 11 if we encounter a path that contains non-ASCII characters. We assume that all paths are given as UTF-8; no conversion is done. Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- archive-zip.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/archive-zip.c b/archive-zip.c index f5af81f90..0f763e802 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -4,6 +4,7 @@ #include "cache.h" #include "archive.h" #include "streaming.h" +#include "utf8.h" static int zip_date; static int zip_time; @@ -16,7 +17,8 @@ static unsigned int zip_dir_offset; static unsigned int zip_dir_entries; #define ZIP_DIRECTORY_MIN_SIZE (1024 * 1024) -#define ZIP_STREAM (8) +#define ZIP_STREAM (1 << 3) +#define ZIP_UTF8 (1 << 11) struct zip_local_header { unsigned char magic[4]; @@ -164,6 +166,17 @@ static void set_zip_header_data_desc(struct zip_local_header *header, copy_le32(header->size, size); } +static int has_only_ascii(const char *s) +{ + for (;;) { + int c = *s++; + if (c == '\0') + return 1; + if (!isascii(c)) + return 0; + } +} + #define STREAM_BUFFER_SIZE (1024 * 16) static int write_zip_entry(struct archiver_args *args, @@ -187,6 +200,13 @@ static int write_zip_entry(struct archiver_args *args, crc = crc32(0, NULL, 0); + if (!has_only_ascii(path)) { + if (is_utf8(path)) + flags |= ZIP_UTF8; + else + warning("Path is not valid UTF-8: %s", path); + } + if (pathlen > 0xffff) { return error("path too long (%d chars, SHA1: %s): %s", (int)pathlen, sha1_to_hex(sha1), path); From 227bf59806dc408d60f8f80a14d8dfe71b92c6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Mon, 24 Sep 2012 17:56:23 +0200 Subject: [PATCH 4/4] archive-zip: write extended timestamp File modification times in ZIP files are encoded in DOS format: local time with a granularity of two seconds. Add an extra field to all archive entries to also record the mtime in Unix' fashion, as UTC with a granularity of one second. This has the desirable side-effect of convincing Info-ZIP unzip 6.00 to respect general purpose flag 11, which is used to indicate that a file name is encoded in UTF-8. Any extra field would do, actually, but the extended timestamp is a reasonably small one (22 bytes per entry). Archives created by Info-ZIP zip 3.0 contain it, too (but with ctime and atime as well). Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- archive-zip.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/archive-zip.c b/archive-zip.c index 0f763e802..55f66b406 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -76,6 +76,14 @@ struct zip_dir_trailer { unsigned char _end[1]; }; +struct zip_extra_mtime { + unsigned char magic[2]; + unsigned char extra_size[2]; + unsigned char flags[1]; + unsigned char mtime[4]; + unsigned char _end[1]; +}; + /* * On ARM, padding is added at the end of the struct, so a simple * sizeof(struct ...) reports two bytes more than the payload size @@ -85,6 +93,9 @@ struct zip_dir_trailer { #define ZIP_DATA_DESC_SIZE offsetof(struct zip_data_desc, _end) #define ZIP_DIR_HEADER_SIZE offsetof(struct zip_dir_header, _end) #define ZIP_DIR_TRAILER_SIZE offsetof(struct zip_dir_trailer, _end) +#define ZIP_EXTRA_MTIME_SIZE offsetof(struct zip_extra_mtime, _end) +#define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \ + (ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags)) static void copy_le16(unsigned char *dest, unsigned int n) { @@ -186,6 +197,7 @@ static int write_zip_entry(struct archiver_args *args, { struct zip_local_header header; struct zip_dir_header dirent; + struct zip_extra_mtime extra; unsigned long attr2; unsigned long compressed_size; unsigned long crc; @@ -266,8 +278,13 @@ static int write_zip_entry(struct archiver_args *args, } } + copy_le16(extra.magic, 0x5455); + copy_le16(extra.extra_size, ZIP_EXTRA_MTIME_PAYLOAD_SIZE); + extra.flags[0] = 1; /* just mtime */ + copy_le32(extra.mtime, args->time); + /* make sure we have enough free space in the dictionary */ - direntsize = ZIP_DIR_HEADER_SIZE + pathlen; + direntsize = ZIP_DIR_HEADER_SIZE + pathlen + ZIP_EXTRA_MTIME_SIZE; while (zip_dir_size < zip_dir_offset + direntsize) { zip_dir_size += ZIP_DIRECTORY_MIN_SIZE; zip_dir = xrealloc(zip_dir, zip_dir_size); @@ -283,7 +300,7 @@ static int write_zip_entry(struct archiver_args *args, copy_le16(dirent.mdate, zip_date); set_zip_dir_data_desc(&dirent, size, compressed_size, crc); copy_le16(dirent.filename_length, pathlen); - copy_le16(dirent.extra_length, 0); + copy_le16(dirent.extra_length, ZIP_EXTRA_MTIME_SIZE); copy_le16(dirent.comment_length, 0); copy_le16(dirent.disk, 0); copy_le16(dirent.attr1, 0); @@ -301,11 +318,13 @@ static int write_zip_entry(struct archiver_args *args, else set_zip_header_data_desc(&header, size, compressed_size, crc); copy_le16(header.filename_length, pathlen); - copy_le16(header.extra_length, 0); + copy_le16(header.extra_length, ZIP_EXTRA_MTIME_SIZE); write_or_die(1, &header, ZIP_LOCAL_HEADER_SIZE); zip_offset += ZIP_LOCAL_HEADER_SIZE; write_or_die(1, path, pathlen); zip_offset += pathlen; + write_or_die(1, &extra, ZIP_EXTRA_MTIME_SIZE); + zip_offset += ZIP_EXTRA_MTIME_SIZE; if (stream && method == 0) { unsigned char buf[STREAM_BUFFER_SIZE]; ssize_t readlen; @@ -402,6 +421,8 @@ static int write_zip_entry(struct archiver_args *args, zip_dir_offset += ZIP_DIR_HEADER_SIZE; memcpy(zip_dir + zip_dir_offset, path, pathlen); zip_dir_offset += pathlen; + memcpy(zip_dir + zip_dir_offset, &extra, ZIP_EXTRA_MTIME_SIZE); + zip_dir_offset += ZIP_EXTRA_MTIME_SIZE; zip_dir_entries++; return 0;