Skip to content

Commit

Permalink
http*: add helper methods for fetching packs
Browse files Browse the repository at this point in the history 
The code handling the fetching of packs in http-push.c and
http-walker.c have been refactored into new methods and a new struct
(http_pack_request) in http.c. They are not meant to be invoked
elsewhere.

The new methods in http.c are
 - new_http_pack_request
 - finish_http_pack_request
 - release_http_pack_request

and the new struct is http_pack_request.

Add a function, new_http_pack_request(), that deals with the details of
coming up with the filename to store the retrieved packfile, resuming a
previously aborted request, and making a new curl request. Update
http-push.c::start_fetch_packed() and http-walker.c::fetch_pack() to
use this.

Add a function, finish_http_pack_request(), that deals with renaming
the pack, advancing the pack list, and installing the pack. Update
http-push.c::finish_request() and http-walker.c::fetch_pack to use
this.

Update release_request() in http-push.c and http-walker.c to invoke
release_http_pack_request() to clean up pack request helper data.

The local_stream member of the transfer_request struct in http-push.c
has been removed, as the packfile pointer will be managed in the struct
http_pack_request.

Signed-off-by: Tay Ray Chuan <rctay89@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
  • Loading branch information
Tay Ray Chuan authored and Junio C Hamano committed Jun 6, 2009
1 parent 39dc52c commit 2264dfa
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 151 deletions.
110 changes: 26 additions & 84 deletions http-push.c
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#include "cache.h"
#include "commit.h"
#include "pack.h"
#include "tag.h"
#include "blob.h"
#include "http.h"
Expand Down Expand Up @@ -119,7 +118,6 @@ struct transfer_request
char filename[PATH_MAX];
char tmpfile[PATH_MAX];
int local_fileno;
FILE *local_stream;
enum transfer_state state;
CURLcode curl_result;
char errorstr[CURL_ERROR_SIZE];
Expand Down Expand Up @@ -452,16 +450,10 @@ static void start_mkcol(struct transfer_request *request)

static void start_fetch_packed(struct transfer_request *request)
{
char *url;
struct packed_git *target;
FILE *packfile;
char *filename;
long prev_posn = 0;
char range[RANGE_HEADER_SIZE];
struct curl_slist *range_header = NULL;

struct transfer_request *check_request = request_queue_head;
struct active_request_slot *slot;
struct http_pack_request *preq;

target = find_sha1_pack(request->obj->sha1, repo->packs);
if (!target) {
Expand All @@ -474,68 +466,35 @@ static void start_fetch_packed(struct transfer_request *request)
fprintf(stderr, "Fetching pack %s\n", sha1_to_hex(target->sha1));
fprintf(stderr, " which contains %s\n", sha1_to_hex(request->obj->sha1));

filename = sha1_pack_name(target->sha1);
snprintf(request->filename, sizeof(request->filename), "%s", filename);
snprintf(request->tmpfile, sizeof(request->tmpfile),
"%s.temp", filename);

url = xmalloc(strlen(repo->url) + 64);
sprintf(url, "%sobjects/pack/pack-%s.pack",
repo->url, sha1_to_hex(target->sha1));
preq = new_http_pack_request(target, repo->url);
if (preq == NULL) {
release_http_pack_request(preq);
repo->can_update_info_refs = 0;
return;
}
preq->lst = &repo->packs;

/* Make sure there isn't another open request for this pack */
while (check_request) {
if (check_request->state == RUN_FETCH_PACKED &&
!strcmp(check_request->url, url)) {
free(url);
!strcmp(check_request->url, preq->url)) {
release_http_pack_request(preq);
release_request(request);
return;
}
check_request = check_request->next;
}

packfile = fopen(request->tmpfile, "a");
if (!packfile) {
fprintf(stderr, "Unable to open local file %s for pack",
request->tmpfile);
repo->can_update_info_refs = 0;
free(url);
return;
}

slot = get_active_slot();
slot->callback_func = process_response;
slot->callback_data = request;
request->slot = slot;
request->local_stream = packfile;
request->userData = target;

request->url = url;
curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
curl_easy_setopt(slot->curl, CURLOPT_URL, url);
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
slot->local = packfile;

/*
* If there is data present from a previous transfer attempt,
* resume where it left off
*/
prev_posn = ftell(packfile);
if (prev_posn>0) {
if (push_verbosely)
fprintf(stderr,
"Resuming fetch of pack %s at byte %ld\n",
sha1_to_hex(target->sha1), prev_posn);
sprintf(range, "Range: bytes=%ld-", prev_posn);
range_header = curl_slist_append(range_header, range);
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
}
preq->slot->callback_func = process_response;
preq->slot->callback_data = request;
request->slot = preq->slot;
request->userData = preq;

/* Try to get the request started, abort the request on error */
request->state = RUN_FETCH_PACKED;
if (!start_active_slot(slot)) {
if (!start_active_slot(preq->slot)) {
fprintf(stderr, "Unable to start GET request\n");
release_http_pack_request(preq);
repo->can_update_info_refs = 0;
release_request(request);
}
Expand Down Expand Up @@ -718,22 +677,17 @@ static void release_request(struct transfer_request *request)

if (request->local_fileno != -1)
close(request->local_fileno);
if (request->local_stream)
fclose(request->local_stream);
free(request->url);
free(request);
}

static void finish_request(struct transfer_request *request)
{
struct stat st;
struct packed_git *target;
struct packed_git **lst;
struct active_request_slot *slot;
struct http_pack_request *preq;

request->curl_result = request->slot->curl_result;
request->http_code = request->slot->http_code;
slot = request->slot;
request->slot = NULL;

/* Keep locks active */
Expand Down Expand Up @@ -821,31 +775,21 @@ static void finish_request(struct transfer_request *request)
start_fetch_packed(request);

} else if (request->state == RUN_FETCH_PACKED) {
int fail = 1;
if (request->curl_result != CURLE_OK) {
fprintf(stderr, "Unable to get pack file %s\n%s",
request->url, curl_errorstr);
repo->can_update_info_refs = 0;
} else {
off_t pack_size = ftell(request->local_stream);

fclose(request->local_stream);
request->local_stream = NULL;
slot->local = NULL;
if (!move_temp_to_file(request->tmpfile,
request->filename)) {
target = (struct packed_git *)request->userData;
target->pack_size = pack_size;
lst = &repo->packs;
while (*lst != target)
lst = &((*lst)->next);
*lst = (*lst)->next;

if (!verify_pack(target))
install_packed_git(target);
else
repo->can_update_info_refs = 0;
preq = (struct http_pack_request *)request->userData;

if (preq) {
if (finish_http_pack_request(preq) > 0)
fail = 0;
release_http_pack_request(preq);
}
}
if (fail)
repo->can_update_info_refs = 0;
release_request(request);
}
}
Expand Down Expand Up @@ -900,7 +844,6 @@ static void add_fetch_request(struct object *obj)
request->lock = NULL;
request->headers = NULL;
request->local_fileno = -1;
request->local_stream = NULL;
request->state = NEED_FETCH;
request->next = request_queue_head;
request_queue_head = request;
Expand Down Expand Up @@ -940,7 +883,6 @@ static int add_send_request(struct object *obj, struct remote_lock *lock)
request->lock = lock;
request->headers = NULL;
request->local_fileno = -1;
request->local_stream = NULL;
request->state = NEED_PUSH;
request->next = request_queue_head;
request_queue_head = request;
Expand Down
85 changes: 18 additions & 67 deletions http-walker.c
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#include "cache.h"
#include "commit.h"
#include "pack.h"
#include "walker.h"
#include "http.h"

Expand Down Expand Up @@ -590,19 +589,10 @@ static int fetch_indices(struct walker *walker, struct alt_base *repo)

static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
{
char *url;
struct packed_git *target;
struct packed_git **lst;
FILE *packfile;
char *filename;
char tmpfile[PATH_MAX];
int ret;
long prev_posn = 0;
char range[RANGE_HEADER_SIZE];
struct curl_slist *range_header = NULL;

struct active_request_slot *slot;
struct slot_results results;
struct http_pack_request *preq;

if (fetch_indices(walker, repo))
return -1;
Expand All @@ -617,72 +607,33 @@ static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned cha
sha1_to_hex(sha1));
}

url = xmalloc(strlen(repo->base) + 65);
sprintf(url, "%s/objects/pack/pack-%s.pack",
repo->base, sha1_to_hex(target->sha1));
preq = new_http_pack_request(target, repo->base);
if (preq == NULL)
goto abort;
preq->lst = &repo->packs;
preq->slot->results = &results;

filename = sha1_pack_name(target->sha1);
snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
packfile = fopen(tmpfile, "a");
if (!packfile)
return error("Unable to open local file %s for pack",
tmpfile);

slot = get_active_slot();
slot->results = &results;
curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
curl_easy_setopt(slot->curl, CURLOPT_URL, url);
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
slot->local = packfile;

/*
* If there is data present from a previous transfer attempt,
* resume where it left off
*/
prev_posn = ftell(packfile);
if (prev_posn>0) {
if (walker->get_verbosely)
fprintf(stderr,
"Resuming fetch of pack %s at byte %ld\n",
sha1_to_hex(target->sha1), prev_posn);
sprintf(range, "Range: bytes=%ld-", prev_posn);
range_header = curl_slist_append(range_header, range);
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
}

if (start_active_slot(slot)) {
run_active_slot(slot);
if (start_active_slot(preq->slot)) {
run_active_slot(preq->slot);
if (results.curl_result != CURLE_OK) {
fclose(packfile);
slot->local = NULL;
return error("Unable to get pack file %s\n%s", url,
curl_errorstr);
error("Unable to get pack file %s\n%s", preq->url,
curl_errorstr);
goto abort;
}
} else {
fclose(packfile);
slot->local = NULL;
return error("Unable to start request");
error("Unable to start request");
goto abort;
}

target->pack_size = ftell(packfile);
fclose(packfile);
slot->local = NULL;

ret = move_temp_to_file(tmpfile, filename);
ret = finish_http_pack_request(preq);
release_http_pack_request(preq);
if (ret)
return ret;

lst = &repo->packs;
while (*lst != target)
lst = &((*lst)->next);
*lst = (*lst)->next;

if (verify_pack(target))
return -1;
install_packed_git(target);

return 0;

abort:
return -1;
}

static void abort_object_request(struct object_request *obj_req)
Expand Down
Loading

0 comments on commit 2264dfa

Please sign in to comment.