Ything-RepoMirrors/libgit2
2
0
Fork 0
mirror of https://github.com/libgit2/libgit2.git synced 2026-01-25 02:56:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

check

Browse Source
This commit is contained in:
Edward Thomson
2023-06-24 22:24:53 +01:00
parent 5cf77b022e
commit c9604efc4a
10 changed files with 343 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/cli/cmd.h
View File

@@ -29,5 +29,6 @@ extern int cmd_cat_file(int argc, char **argv);
extern int cmd_clone(int argc, char **argv);
extern int cmd_hash_object(int argc, char **argv);
extern int cmd_help(int argc, char **argv);
extern int cmd_index_pack(int argc, char **argv);
#endif /* CLI_cmd_h__ */

119
src/cli/cmd_index_pack.c Normal file
View File

@@ -0,0 +1,119 @@
/*
* Copyright (C) the libgit2 contributors. All rights reserved.
*
* This file is part of libgit2, distributed under the GNU GPL v2 with
* a Linking Exception. For full terms see the included COPYING file.
*/
#include <git2.h>
#include "cli.h"
#include "cmd.h"
#include "progress.h"
#define COMMAND_NAME "index-pack"
#define BUFFER_SIZE (1024 * 1024)
static int show_help, verbose, read_stdin;
static char *filename;
static cli_progress progress = CLI_PROGRESS_INIT;
static const cli_opt_spec opts[] = {
{ CLI_OPT_TYPE_SWITCH, "help", 0, &show_help, 1,
CLI_OPT_USAGE_HIDDEN | CLI_OPT_USAGE_STOP_PARSING, NULL,
"display help about the " COMMAND_NAME " command" },
{ CLI_OPT_TYPE_SWITCH, "verbose", 'v', &verbose, 1,
CLI_OPT_USAGE_DEFAULT, NULL, "display progress output" },
{ CLI_OPT_TYPE_LITERAL },
{ CLI_OPT_TYPE_SWITCH, "stdin", 0, &read_stdin, 1,
CLI_OPT_USAGE_REQUIRED, NULL, "read from stdin" },
{ CLI_OPT_TYPE_ARG, "pack-file", 0, &filename, 0,
CLI_OPT_USAGE_CHOICE, "pack-file", "packfile path" },
{ 0 },
};
static void print_help(void)
{
cli_opt_usage_fprint(stdout, PROGRAM_NAME, COMMAND_NAME, opts);
printf("\n");
printf("Indexes a packfile and writes the index to disk.\n");
printf("\n");
printf("Options:\n");
cli_opt_help_fprint(stdout, opts);
}
int cmd_index_pack(int argc, char **argv)
{
cli_opt invalid_opt;
git_indexer *idx = NULL;
git_indexer_options idx_opts = GIT_INDEXER_OPTIONS_INIT;
git_indexer_progress stats = {0};
char buf[BUFFER_SIZE];
ssize_t read_len;
int fd, ret;
if (cli_opt_parse(&invalid_opt, opts, argv + 1, argc - 1, CLI_OPT_PARSE_GNU))
return cli_opt_usage_error(COMMAND_NAME, opts, &invalid_opt);
if (show_help) {
print_help();
return 0;
}
if (verbose) {
idx_opts.progress_cb = cli_progress_indexer;
idx_opts.progress_cb_payload = &progress;
}
if (read_stdin) {
fd = fileno(stdin);
} else if ((fd = p_open(filename, O_RDONLY)) < 0) {
ret = cli_error_git();
goto done;
}
#ifdef GIT_EXPERIMENTAL_SHA256
ret = git_indexer_new(&idx, ".", GIT_OID_SHA1, &idx_opts);
#else
ret = git_indexer_new(&idx, ".", 0, NULL, &idx_opts);
#endif
if (ret < 0) {
ret = cli_error_git();
goto done;
}
while ((read_len = p_read(fd, buf, sizeof(buf))) > 0) {
if (git_indexer_append(idx, buf, (size_t)read_len, &stats) < 0) {
ret = cli_error_git();
goto done;
}
}
if (read_len < 0) {
ret = cli_error_os();
goto done;
}
if (!read_stdin)
p_close(fd);
if (git_indexer_commit(idx, &stats) < 0) {
ret = cli_error_git();
goto done;
}
cli_progress_finish(&progress);
done:
cli_progress_dispose(&progress);
git_indexer_free(idx);
return ret;
}

1
src/cli/main.c
View File

@@ -32,6 +32,7 @@ const cli_cmd_spec cli_cmds[] = {
{ "clone", cmd_clone, "Clone a repository into a new directory" },
{ "hash-object", cmd_hash_object, "Hash a raw object and product its object ID" },
{ "help", cmd_help, "Display help information" },
{ "index-pack", cmd_index_pack, "Create an index for a packfile" },
{ NULL }
};

53
src/cli/progress.c
View File

@@ -242,7 +242,21 @@ static int fetch_receiving(
done ? ", done." : "");
}
static int fetch_resolving(
static int indexer_indexing(
cli_progress *progress,
const git_indexer_progress *stats)
{
bool done = (stats->received_objects == stats->total_objects);
return progress_printf(progress, false,
"Indexing objects: %3d%% (%d/%d)%s\r",
percent(stats->received_objects, stats->total_objects),
stats->received_objects,
stats->total_objects,
done ? ", done." : "");
}
static int indexer_resolving(
cli_progress *progress,
const git_indexer_progress *stats)
{
@@ -283,7 +297,42 @@ int cli_progress_fetch_transfer(const git_indexer_progress *stats, void *payload
/* fall through */
case CLI_PROGRESS_RESOLVING:
error = fetch_resolving(progress, stats);
error = indexer_resolving(progress, stats);
break;
default:
/* should not be reached */
GIT_ASSERT(!"unexpected progress state");
}
return error;
}
int cli_progress_indexer(
const git_indexer_progress *stats,
void *payload)
{
cli_progress *progress = (cli_progress *)payload;
int error = 0;
switch (progress->action) {
case CLI_PROGRESS_NONE:
progress->action = CLI_PROGRESS_INDEXING;
/* fall through */
case CLI_PROGRESS_INDEXING:
if ((error = indexer_indexing(progress, stats)) < 0)
break;
if (stats->indexed_deltas == stats->total_deltas)
break;
progress_complete(progress);
progress->action = CLI_PROGRESS_RESOLVING;
/* fall through */
case CLI_PROGRESS_RESOLVING:
error = indexer_resolving(progress, stats);
break;
default:

12
src/cli/progress.h
View File

@@ -22,6 +22,7 @@
typedef enum {
CLI_PROGRESS_NONE,
CLI_PROGRESS_RECEIVING,
CLI_PROGRESS_INDEXING,
CLI_PROGRESS_RESOLVING,
CLI_PROGRESS_CHECKING_OUT
} cli_progress_t;
@@ -74,6 +75,17 @@ extern int cli_progress_fetch_transfer(
const git_indexer_progress *stats,
void *payload);
/**
* Prints indexer progress to the console. Suitable for a
* `progress_cb` callback for `git_indexer_options`.
*
* @param stats The indexer stats
* @param payload A pointer to the cli_progress
*/
extern int cli_progress_indexer(
const git_indexer_progress *stats,
void *payload);
/**
* Prints checkout progress to the console. Suitable for a
* `progress_cb` callback for `git_checkout_options`.

218
src/libgit2/indexer.c
View File

@@ -10,9 +10,9 @@
#include "packfile_parser.h"
#include "repository.h"
#include "sizemap.h"
#include "delta.h"
#include "git2_util.h"
#include "posix.h"
#include "git2/indexer.h"
@@ -24,11 +24,15 @@ size_t git_indexer__max_objects = UINT32_MAX;
struct object_entry {
git_object_t type;
git_object_size_t position;
/* TODO: this can be unsigned short */
git_object_size_t header_size;
git_oid id;
};
struct delta_entry {
struct object_entry object;
git_object_t final_type;
unsigned short chain_length;
union {
git_oid ref_id;
git_object_size_t ofs_position;
@@ -60,10 +64,13 @@ struct git_indexer {
/* TODO: pul these directly from the parser instead? */
git_object_size_t current_position;
git_object_t current_type;
git_object_size_t current_header_size;
git_object_size_t current_size;
git_oid current_ref; /* current ref delta base */
git_object_size_t current_offset; /* current ofs delta base */
git_hash_ctx hash_ctx;
git_sizemap *positions; /* map of position to object */
git_vector objects; /* vector of `struct object_entry` */
git_vector deltas; /* vector of `struct delta_entry` */
@@ -121,15 +128,15 @@ static int parse_packfile_header(
static int parse_object_start(
git_object_size_t position,
git_object_t type,
git_object_size_t header_size,
git_object_size_t size,
void *data)
{
git_indexer *indexer = (git_indexer *)data;
printf("--> object start: %d %d %d\n", (int)position, (int)type, (int)size);
indexer->current_position = position;
indexer->current_type = type;
indexer->current_header_size = header_size;
indexer->current_size = size;
return 0;
@@ -143,7 +150,7 @@ static int parse_object_complete(
git_indexer *indexer = (git_indexer *)data;
struct object_entry *entry;
printf("--> object complete: %d %d %d %s\n", (int)indexer->current_size, (int)compressed_size, (int)indexer->current_position, git_oid_tostr_s(oid));
GIT_UNUSED(compressed_size);
/* TODO: pool? */
entry = git__malloc(sizeof(struct object_entry));
@@ -152,6 +159,7 @@ static int parse_object_complete(
entry->type = indexer->current_type;
git_oid_cpy(&entry->id, oid);
entry->position = indexer->current_position;
entry->header_size = indexer->current_header_size;
if (git_sizemap_set(indexer->positions, entry->position, entry) < 0 ||
git_vector_insert(&indexer->objects, entry) < 0)
@@ -163,6 +171,7 @@ static int parse_object_complete(
static int parse_delta_start(
git_object_size_t position,
git_object_t type,
git_object_size_t header_size,
git_object_size_t size,
git_oid *delta_ref,
git_object_size_t delta_offset,
@@ -170,19 +179,13 @@ static int parse_delta_start(
{
git_indexer *indexer = (git_indexer *)data;
printf("--> DELTA start: %d %d %d\n", (int)position, (int)type, (int)size);
if (type == GIT_OBJECT_REF_DELTA)
printf("--> DELTA start: %d %d %d / %s\n", (int)position, (int)type, (int)size, git_oid_tostr_s(delta_ref));
else
printf("--> DELTA start: %d %d %d / %d = %d\n", (int)position, (int)type, (int)size, (int)delta_offset, (int)(position - delta_offset));
GIT_UNUSED(delta_ref);
GIT_UNUSED(delta_offset);
/* TODO: avoid double copy - preallocate the entry */
indexer->current_position = position;
indexer->current_type = type;
indexer->current_header_size = header_size;
indexer->current_size = size;
if (type == GIT_OBJECT_REF_DELTA)
@@ -200,14 +203,17 @@ static int parse_delta_complete(
git_indexer *indexer = (git_indexer *)data;
struct delta_entry *entry;
printf("--> DELTA complete: %d\n", (int)compressed_size);
GIT_UNUSED(compressed_size);
/* TODO: pool? */
entry = git__malloc(sizeof(struct delta_entry));
GIT_ERROR_CHECK_ALLOC(entry);
entry->object.type = indexer->current_type;
entry->object.header_size = indexer->current_header_size;
entry->object.position = indexer->current_position;
entry->final_type = 0;
entry->chain_length = 0;
if (entry->object.type == GIT_OBJECT_REF_DELTA) {
git_oid_cpy(&entry->base.ref_id, &indexer->current_ref);
@@ -221,6 +227,7 @@ static int parse_delta_complete(
}
if (git_sizemap_set(indexer->positions, entry->object.position, entry) < 0 ||
git_vector_insert(&indexer->objects, entry) < 0 ||
git_vector_insert(&indexer->deltas, entry) < 0)
return -1;
@@ -252,6 +259,7 @@ static int indexer_new(
git_indexer_options opts = GIT_INDEXER_OPTIONS_INIT;
git_indexer *indexer;
git_str path = GIT_STR_INIT;
git_hash_algorithm_t hash_type;
int error;
if (in_opts)
@@ -270,7 +278,10 @@ static int indexer_new(
if (git_repository__fsync_gitdir)
indexer->do_fsync = 1;
if ((error = git_packfile_parser_init(&indexer->parser, indexer->oid_type)) < 0 ||
hash_type = git_oid_algorithm(oid_type);
if ((error = git_packfile_parser_init(&indexer->parser, oid_type)) < 0 ||
(error = git_hash_ctx_init(&indexer->hash_ctx, hash_type)) < 0 ||
(error = git_str_joinpath(&path, parent_path, "pack")) < 0 ||
(error = indexer->fd = git_futils_mktmp(&indexer->path, path.ptr, indexer->mode)) < 0)
goto done;
@@ -381,43 +392,13 @@ int git_indexer_append(
return 0;
}
static int load_ofs_base(
const unsigned char **out,
size_t *out_len,
git_indexer *indexer,
git_object_size_t ofs_position);
/* TODO: limit recursion depth -- it looks like git may put a 50 length limit on delta chains? */
static int resolve_delta(git_indexer *indexer, struct delta_entry *delta)
{
const unsigned char *base_data;
size_t base_data_len;
/* TODO: cache lookup here? */
if (delta->object.type == GIT_OBJECT_REF_DELTA) {
printf("--> DELTA removed: ref %s\n", git_oid_tostr_s(&delta->base.ref_id));
return -1;
} else {
if (load_ofs_base(&base_data, &base_data_len, indexer, delta->base.ofs_position) < 0)
return -1;
printf("--> DELTA removed: ofs %d\n", (int)delta->base.ofs_position);
}
printf("i have a delta base: it's %d\n", (int) base_data_len);
return 0;
}
/* TODO: this should live somewhere else -- maybe in packfile parser? */
static int load_object(
const unsigned char **out,
static int unpack_raw_object(
unsigned char **out,
size_t *out_len,
git_indexer *indexer,
git_object_size_t ofs_position)
git_object_size_t raw_position)
{
char c;
git_str data = GIT_STR_INIT;
/* TODO: we know the object size, hint the git_str with it */
@@ -428,40 +409,33 @@ static int load_object(
/* TODO: assert ofs_position <= off_t */
/* TODO: 32 bit vs 64 bit */
if (p_lseek(fd, (off_t)ofs_position, SEEK_SET) < 0) {
if (p_lseek(fd, (off_t)raw_position, SEEK_SET) < 0) {
git_error_set(GIT_ERROR_OS, "could not seek in packfile");
return -1;
}
/* ugh */
/* TODO: send the object header size for each object in the parser callback so that we don't have to do this nonsense. */
do {
if (p_read(fd, &c, 1) != 1) {
git_error_set(GIT_ERROR_OS, "could not read packfile object header");
return -1;
}
} while ((c & 0x80) == 0x80);
if (git_zstream_inflatefile(&data, fd) < 0)
return -1;
/* TODO: validate that data.size == expected size of this object from the positions table */
*out_len = data.size;
*out = (const unsigned char *)git_str_detach(&data);
*out = (unsigned char *)git_str_detach(&data);
return 0;
}
static int load_ofs_base(
const unsigned char **out,
static int unpack_object_at_position(
unsigned char **out,
size_t *out_len,
git_object_t *out_type,
unsigned short *out_chain_length,
git_indexer *indexer,
git_object_size_t ofs_position)
{
struct object_entry *base_object;
printf("resolving ofs delta at position %llu\n", ofs_position);
struct object_entry *object;
git_object_size_t raw_position;
int error;
/*
* TODO: we should cache small delta bases?
@@ -476,26 +450,87 @@ static int load_ofs_base(
* those objects.
*/
if ((base_object = git_sizemap_get(indexer->positions, ofs_position)) == NULL) {
if ((object = git_sizemap_get(indexer->positions, ofs_position)) == NULL) {
git_error_set(GIT_ERROR_INDEXER,
"corrupt packfile - no object at delta offset position %llu",
"corrupt packfile - no object at offset position %llu",
ofs_position);
return -1;
}
if (base_object->type == GIT_OBJECT_REF_DELTA ||
base_object->type == GIT_OBJECT_OFS_DELTA) {
struct delta_entry *delta_entry = (struct delta_entry *)base_object;
const unsigned char *base;
size_t base_len;
if (object->type == GIT_OBJECT_REF_DELTA) {
abort();
} else if (object->type == GIT_OBJECT_OFS_DELTA) {
struct delta_entry *delta_entry = (struct delta_entry *)object;
unsigned char *base, *delta;
size_t base_len, delta_len;
if (load_ofs_base(&base, &base_len, indexer, delta_entry->base.ofs_position) < 0)
/* TODO: overflow check */
raw_position = ofs_position + object->header_size;
if (unpack_object_at_position(&base, &base_len, out_type, out_chain_length, indexer, delta_entry->base.ofs_position) < 0 ||
unpack_raw_object(&delta, &delta_len, indexer, raw_position) < 0)
return -1;
abort();
error = git_delta_apply((void **)out, out_len, base, base_len, delta, delta_len);
(*out_chain_length)++;
git__free(base);
git__free(delta);
return error;
}
return load_object(out, out_len, indexer, ofs_position);
/* TODO: overflow check */
raw_position = ofs_position + object->header_size;
if (unpack_raw_object(out, out_len, indexer, raw_position) < 0)
return -1;
*out_chain_length = 0;
*out_type = object->type;
return 0;
}
/* TODO: limit recursion depth -- it looks like git may put a 50 length limit on delta chains? */
static int resolve_delta(git_indexer *indexer, struct delta_entry *delta)
{
char header[64];
unsigned char *data;
size_t header_len, data_len;
/* TODO: cache lookup here? */
/* TODO: hash ctx per thread */
if (git_hash_init(&indexer->hash_ctx) < 0)
return -1;
if (delta->object.type == GIT_OBJECT_REF_DELTA) {
abort();
return -1;
} else {
if (unpack_object_at_position(&data, &data_len, &delta->final_type, &delta->chain_length, indexer, delta->object.position) < 0)
return -1;
}
/*
* TODO: we don't really need to dedeltafy the whole object just to
* hash it, we could hash in the dedltafication step.
*/
if (git_odb__format_object_header(&header_len, header, sizeof(header), data_len, delta->final_type) < 0 ||
git_hash_update(&indexer->hash_ctx, header, header_len) < 0 ||
git_hash_update(&indexer->hash_ctx, data, data_len) < 0 ||
git_hash_final(delta->object.id.id, &indexer->hash_ctx) < 0)
return -1;
#ifdef GIT_EXPERIMENTAL_SHA256
delta->object.id.type = indexer->oid_type;
#endif
/*printf("resolved: %s\n", git_oid_tostr_s(&delta->object.id));*/
return 0;
}
static int resolve_final_deltas(git_indexer *indexer)
@@ -503,13 +538,29 @@ static int resolve_final_deltas(git_indexer *indexer)
size_t deltas_len = git_vector_length(&indexer->deltas);
struct delta_entry *delta_entry;
size_t i;
int cnt = 0;
while (deltas_len > 0) {
bool progress = false;
printf("loop %d\n", ++cnt);
git_vector_foreach(&indexer->deltas, i, delta_entry) {
if (delta_entry->final_type)
continue;
if (resolve_delta(indexer, delta_entry) < 0)
return -1;
/* TODO */
deltas_len--;
progress = true;
}
if (!progress) {
git_error_set(GIT_ERROR_INDEXER, "could not resolve deltas");
return -1;
}
}
@@ -533,12 +584,26 @@ int git_indexer_commit(git_indexer *indexer, git_indexer_progress *stats)
if (resolve_final_deltas(indexer) < 0)
return -1;
git_vector_sort(&indexer->objects);
git_vector_foreach(&indexer->objects, i, entry) {
printf("--> sorted: %s\n", git_oid_tostr_s(&entry->id));
git_object_t type = git_object__is_delta(entry->type) ?
((struct delta_entry *)entry)->final_type : entry->type;
/*
printf("%s %-6s ... ... %llu",
git_oid_tostr_s(&entry->id),
git_object_type2string(type),
entry->position);
if (git_object__is_delta(entry->type)) {
printf(" %u", ((struct delta_entry *)entry)->chain_length);
}
printf("\n");
*/
}
git_vector_sort(&indexer->objects);
return 0;
}
@@ -547,9 +612,10 @@ void git_indexer_free(git_indexer *indexer)
if (!indexer)
return;
git_hash_ctx_cleanup(&indexer->hash_ctx);
git_sizemap_free(indexer->positions);
git_vector_free(&indexer->deltas);
git_vector_free_deep(&indexer->objects);
git_vector_free_deep(&indexer->deltas);
git_packfile_parser_dispose(&indexer->parser);
git__free(indexer);
}

5
src/libgit2/object.h
View File

@@ -67,6 +67,11 @@ int git_object__write_oid_header(
bool git_object__is_valid(
git_repository *repo, const git_oid *id, git_object_t expected_type);
GIT_INLINE(bool) git_object__is_delta(git_object_t type)
{
return (type == GIT_OBJECT_REF_DELTA || type == GIT_OBJECT_OFS_DELTA);
}
GIT_INLINE(git_object_t) git_object__type_from_filemode(git_filemode_t mode)
{
switch (mode) {

4
src/libgit2/packfile_parser.c
View File

@@ -141,6 +141,7 @@ static int parse_object_header(
int error = parser->object_start(
parser->current_position,
parser->current_type,
parser->current_compressed_size,
parser->current_size,
parser->callback_data);
@@ -259,6 +260,7 @@ static int parse_delta_header(
int error = parser->delta_start(
parser->current_position,
parser->current_type,
parser->current_compressed_size,
parser->current_size,
NULL,
parser->current_offset,
@@ -269,6 +271,7 @@ static int parse_delta_header(
}
parser->state = STATE_DELTA_DATA_START;
break;
}
}
@@ -292,6 +295,7 @@ static int parse_delta_header(
int error = parser->delta_start(
parser->current_position,
parser->current_type,
parser->current_compressed_size,
parser->current_size,
&parser->current_base,
0,

4
src/libgit2/packfile_parser.h
View File

@@ -39,10 +39,10 @@ struct git_packfile_parser {
/* Callbacks */
int (*packfile_header)(uint32_t version, uint32_t entries, void *data);
int (*object_start)(git_object_size_t offset, git_object_t type, git_object_size_t size, void *data);
int (*object_start)(git_object_size_t offset, git_object_t type, git_object_size_t header_size, git_object_size_t size, void *data);
int (*object_data)(void *object_data, size_t len, void *data);
int (*object_complete)(git_object_size_t compressed_size, git_oid *oid, void *data);
int (*delta_start)(git_object_size_t offset, git_object_t type, git_object_size_t size, git_oid *delta_ref, git_object_size_t delta_offset, void *data);
int (*delta_start)(git_object_size_t offset, git_object_t type, git_object_size_t header_size, git_object_size_t size, git_oid *delta_ref, git_object_size_t delta_offset, void *data);
int (*delta_data)(void *delta_data, size_t len, void *data);
int (*delta_complete)(git_object_size_t compressed_size, void *data);
int (*packfile_complete)(const unsigned char *checksum, size_t checksum_len, void *data);

10
tests/libgit2/pack/parser.c
View File

@@ -5,7 +5,8 @@
void test_pack_parser__indexer_single_byte(void)
{
git_indexer *idx;
char buf[1];
char buf[1024];
ssize_t ret;
int fd;
#ifdef GIT_EXPERIMENTAL_SHA256
@@ -14,10 +15,11 @@ void test_pack_parser__indexer_single_byte(void)
cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL));
#endif
cl_assert((fd = p_open("/Users/ethomson/Personal/Projects/libgit2/libgit2-6/tests/resources/testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.pack", O_RDONLY)) >= 0);
//cl_assert((fd = p_open("/Users/ethomson/Personal/Projects/libgit2/libgit2-6/tests/resources/testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.pack", O_RDONLY)) >= 0);
cl_assert((fd = p_open("/tmp/pack-b82c9be473f721eacaac5042d11b837f00e7f31e.pack", O_RDONLY)) >= 0);
while (read(fd, buf, 1) == 1) {
cl_git_pass(git_indexer_append(idx, buf, 1, NULL));
while ((ret = read(fd, buf, sizeof(buf))) > 0) {
cl_git_pass(git_indexer_append(idx, buf, (size_t)ret, NULL));
}
p_close(fd);