Subject: [PATCH v4 0/5] Reftable support git-core

From: "Han-Wen Nienhuys via GitGitGadget" <[email-masked]>
Date: Thu, 06 Feb 2020 22:55:51 +0000

This adds the reftable library, and hooks it up as a ref backend.

At this point, I am mainly interested in feedback on the spots marked with
XXX in the Git source code.

v3

 * passes gitgitgadget CI.

Han-Wen Nienhuys (5):
  refs.h: clarify reflog iteration order
  create .git/refs in files-backend.c
  refs: document how ref_iterator_advance_fn should handle symrefs
  Add reftable library
  Reftable support for git-core

 Makefile                |   24 +-
 builtin/init-db.c       |   49 +-
 cache.h                 |    2 +
 refs.c                  |   20 +-
 refs.h                  |    5 +-
 refs/files-backend.c    |    6 +
 refs/refs-internal.h    |    6 +
 refs/reftable-backend.c |  880 +++++++++++++++++++++++++++++++
 reftable/LICENSE        |   31 ++
 reftable/README.md      |   19 +
 reftable/VERSION        |    5 +
 reftable/basics.c       |  196 +++++++
 reftable/basics.h       |   37 ++
 reftable/block.c        |  401 ++++++++++++++
 reftable/block.h        |   71 +++
 reftable/blocksource.h  |   20 +
 reftable/bytes.c        |    0
 reftable/config.h       |    1 +
 reftable/constants.h    |   27 +
 reftable/dump.c         |   97 ++++
 reftable/file.c         |   97 ++++
 reftable/iter.c         |  229 ++++++++
 reftable/iter.h         |   56 ++
 reftable/merged.c       |  286 ++++++++++
 reftable/merged.h       |   34 ++
 reftable/pq.c           |  114 ++++
 reftable/pq.h           |   34 ++
 reftable/reader.c       |  708 +++++++++++++++++++++++++
 reftable/reader.h       |   52 ++
 reftable/record.c       | 1107 +++++++++++++++++++++++++++++++++++++++
 reftable/record.h       |   79 +++
 reftable/reftable.h     |  399 ++++++++++++++
 reftable/slice.c        |  199 +++++++
 reftable/slice.h        |   39 ++
 reftable/stack.c        |  983 ++++++++++++++++++++++++++++++++++
 reftable/stack.h        |   40 ++
 reftable/system.h       |   55 ++
 reftable/tree.c         |   66 +++
 reftable/tree.h         |   24 +
 reftable/writer.c       |  623 ++++++++++++++++++++++
 reftable/writer.h       |   46 ++
 reftable/zlib-compat.c  |   92 ++++
 repository.c            |    2 +
 repository.h            |    3 +
 setup.c                 |   12 +-
 45 files changed, 7248 insertions(+), 28 deletions(-)
 create mode 100644 refs/reftable-backend.c
 create mode 100644 reftable/LICENSE
 create mode 100644 reftable/README.md
 create mode 100644 reftable/VERSION
 create mode 100644 reftable/basics.c
 create mode 100644 reftable/basics.h
 create mode 100644 reftable/block.c
 create mode 100644 reftable/block.h
 create mode 100644 reftable/blocksource.h
 create mode 100644 reftable/bytes.c
 create mode 100644 reftable/config.h
 create mode 100644 reftable/constants.h
 create mode 100644 reftable/dump.c
 create mode 100644 reftable/file.c
 create mode 100644 reftable/iter.c
 create mode 100644 reftable/iter.h
 create mode 100644 reftable/merged.c
 create mode 100644 reftable/merged.h
 create mode 100644 reftable/pq.c
 create mode 100644 reftable/pq.h
 create mode 100644 reftable/reader.c
 create mode 100644 reftable/reader.h
 create mode 100644 reftable/record.c
 create mode 100644 reftable/record.h
 create mode 100644 reftable/reftable.h
 create mode 100644 reftable/slice.c
 create mode 100644 reftable/slice.h
 create mode 100644 reftable/stack.c
 create mode 100644 reftable/stack.h
 create mode 100644 reftable/system.h
 create mode 100644 reftable/tree.c
 create mode 100644 reftable/tree.h
 create mode 100644 reftable/writer.c
 create mode 100644 reftable/writer.h
 create mode 100644 reftable/zlib-compat.c

base-commit: 5b0ca878e008e82f91300091e793427205ce3544
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v4
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v4
Pull-Request: https://github.com/gitgitgadget/git/pull/539

Range-diff vs v3:

 1:  c00403c94d = 1:  c00403c94d refs.h: clarify reflog iteration order
 2:  57c7342319 < -:  ---------- setup.c: enable repo detection for reftable
 3:  5b7060cb2f ! 2:  4d6da9bc47 create .git/refs in files-backend.c
     @@ -29,9 +29,13 @@
       
      +	files_ref_path(refs, &sb, "refs");
      +	safe_create_dir(sb.buf, 1);
     -+        /* adjust permissions even if directory already exists. */
     ++	/* adjust permissions even if directory already exists. */
      +	adjust_shared_perm(sb.buf);
      +
       	/*
       	 * Create .git/refs/{heads,tags}
       	 */
     ++	strbuf_reset(&sb);
     + 	files_ref_path(refs, &sb, "refs/heads");
     + 	safe_create_dir(sb.buf, 1);
     + 
 4:  1b01c735a9 = 3:  fbdcdccc88 refs: document how ref_iterator_advance_fn should handle symrefs
 5:  eb0df10068 ! 4:  02d2ca8b87 Add reftable library
     @@ -98,11 +98,11 @@
       --- /dev/null
       +++ b/reftable/VERSION
      @@
     -+commit e54326f73d95bfe8b17f264c400f4c365dbd5e5e
     ++commit e7c3fc3099d9999bc8d895f84027b0e36348d5e6
      +Author: Han-Wen Nienhuys <[email-masked]>
     -+Date:   Tue Feb 4 19:48:17 2020 +0100
     ++Date:   Thu Feb 6 20:17:40 2020 +0100
      +
     -+    C: PRI?MAX use; clang-format.
     ++    C: use inttypes.h header definitions
      
       diff --git a/reftable/basics.c b/reftable/basics.c
       new file mode 100644
     @@ -2882,7 +2882,7 @@
      +{
      +	char hex[SHA256_SIZE + 1] = {};
      +
     -+	printf("ref{%s(%" PRIdMAX ") ", ref->ref_name, ref->update_index);
     ++	printf("ref{%s(%" PRIu64 ") ", ref->ref_name, ref->update_index);
      +	if (ref->value != NULL) {
      +		hex_format(hex, ref->value, hash_size);
      +		printf("%s", hex);
     @@ -3274,9 +3274,9 @@
      +{
      +	char hex[SHA256_SIZE + 1] = {};
      +
     -+	printf("log{%s(%" PRIdMAX ") %s <%s> %" PRIuMAX " %04d\n",
     -+	       log->ref_name, log->update_index, log->name, log->email,
     -+	       log->time, log->tz_offset);
     ++	printf("log{%s(%" PRIu64 ") %s <%s> %" PRIu64 " %04d\n", log->ref_name,
     ++	       log->update_index, log->name, log->email, log->time,
     ++	       log->tz_offset);
      +	hex_format(hex, log->old_hash, hash_size);
      +	printf("%s => ", hex);
      +	hex_format(hex, log->new_hash, hash_size);
     @@ -4860,7 +4860,7 @@
      +static void format_name(struct slice *dest, uint64_t min, uint64_t max)
      +{
      +	char buf[100];
     -+	snprintf(buf, sizeof(buf), "%012" PRIxMAX "-%012" PRIxMAX, min, max);
     ++	snprintf(buf, sizeof(buf), "%012" PRIx64 "-%012" PRIx64, min, max);
      +	slice_set_string(dest, buf);
      +}
      +
     @@ -5585,6 +5585,7 @@
      +#include <assert.h>
      +#include <errno.h>
      +#include <fcntl.h>
     ++#include <inttypes.h>
      +#include <stdint.h>
      +#include <stdio.h>
      +#include <stdlib.h>
     @@ -5595,9 +5596,6 @@
      +#include <unistd.h>
      +#include <zlib.h>
      +
     -+#define PRIuMAX "lu"
     -+#define PRIdMAX "ld"
     -+#define PRIxMAX "lx"
      +#define ARRAY_SIZE(a) sizeof((a)) / sizeof((a)[0])
      +#define FREE_AND_NULL(x)    \
      +	do {                \
     @@ -5605,14 +5603,14 @@
      +		(x) = NULL; \
      +	} while (0)
      +#define QSORT(arr, n, cmp) qsort(arr, n, sizeof(arr[0]), cmp)
     -+#define SWAP(a, b) \
     -+  { \
     -+     char tmp[sizeof(a)]; \
     -+     assert(sizeof(a)==sizeof(b)); \
     -+     memcpy(&tmp[0], &a, sizeof(a)); \
     -+     memcpy(&a, &b, sizeof(a)); \
     -+     memcpy(&b, &tmp[0], sizeof(a)); \
     -+  }
     ++#define SWAP(a, b)                              \
     ++	{                                       \
     ++		char tmp[sizeof(a)];            \
     ++		assert(sizeof(a) == sizeof(b)); \
     ++		memcpy(&tmp[0], &a, sizeof(a)); \
     ++		memcpy(&a, &b, sizeof(a));      \
     ++		memcpy(&b, &tmp[0], sizeof(a)); \
     ++	}
      +#endif
      +
      +int uncompress_return_consumed(Bytef *dest, uLongf *destLen,
     @@ -5758,6 +5756,7 @@
      +		return &w->stats.log_stats;
      +	}
      +	assert(false);
     ++	return NULL;
      +}
      +
      +/* write data, queuing the padding for the next write. Returns negative for
     @@ -6299,7 +6298,7 @@
      +	w->stats.blocks++;
      +
      +	if (debug) {
     -+		fprintf(stderr, "block %c off %" PRIuMAX " sz %d (%d)\n", typ,
     ++		fprintf(stderr, "block %c off %" PRIu64 " sz %d (%d)\n", typ,
      +			w->next, raw_bytes,
      +			get_u24(w->block + w->block_writer->header_off + 1));
      +	}
 6:  0b2a1a81d6 ! 5:  2786a6bf61 Reftable support for git-core
     @@ -39,6 +39,7 @@
            reftable.LogRecord{RefName:"HEAD", UpdateIndex:0x2, New:[]uint8{0x37, 0x3d, 0x96, 0x97, 0x2f, 0xca, 0x9b, 0x63, 0x59, 0x57, 0x40, 0xbb, 0xa3, 0x89, 0x8a, 0x76, 0x27, 0x78, 0xba, 0x20}, Old:[]uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, Name:"Han-Wen Nienhuys", Email:"[email-masked]", Time:0x5e29ef27, TZOffset:100, Message:"commit (initial): x\n"}
      
          Signed-off-by: Han-Wen Nienhuys <[email-masked]>
     +    Co-authored-by: Jeff King <[email-masked]>
      
       diff --git a/Makefile b/Makefile
       --- a/Makefile
     @@ -120,6 +121,31 @@ $^
       {
       	struct stat st1;
       	struct strbuf buf = STRBUF_INIT;
     +@@
     + 	is_bare_repository_cfg = init_is_bare_repository;
     + 	if (init_shared_repository != -1)
     + 		set_shared_repository(init_shared_repository);
     ++	if (flags & INIT_DB_REFTABLE)
     ++		the_repository->ref_storage_format = xstrdup("reftable");
     + 
     + 	/*
     + 	 * We would have created the above under user's umask -- under
     +@@
     + 		adjust_shared_perm(get_git_dir());
     + 	}
     + 
     ++	/*
     ++	 * Check to see if .git/HEAD exists; this must happen before
     ++	 * initializing the ref db, because we want to see if there is an
     ++	 * existing HEAD.
     ++	 */
     ++	path = git_path_buf(&buf, "HEAD");
     ++	reinit = (!access(path, R_OK) ||
     ++		  readlink(path, junk, sizeof(junk) - 1) != -1);
     ++
     + 	/*
     + 	 * We need to create a "refs" dir in any case so that older
     + 	 * versions of git can tell that this is a repository.
      @@
       	 * Create the default symlink from ".git/HEAD" to the "master"
       	 * branch, if it does not exist yet.
     @@ -127,18 +153,14 @@ $^
      -	path = git_path_buf(&buf, "HEAD");
      -	reinit = (!access(path, R_OK)
      -		  || readlink(path, junk, sizeof(junk)-1) != -1);
     -+	if (flags & INIT_DB_REFTABLE) {
     -+		reinit = 0; /* XXX - how do we recognize a reinit,
     -+			     * and what should we do? */
     -+	} else {
     -+		path = git_path_buf(&buf, "HEAD");
     -+		reinit = (!access(path, R_OK) ||
     -+			  readlink(path, junk, sizeof(junk) - 1) != -1);
     -+	}
     -+
       	if (!reinit) {
       		if (create_symref("HEAD", "refs/heads/master", NULL) < 0)
       			exit(1);
     ++	} else {
     ++		/*
     ++		 * XXX should check whether our ref backend matches the
     ++		 * original one; if not, either die() or convert
     ++		 */
       	}
       
       	/* This forces creation of new config file */
     @@ -251,10 +273,8 @@ $^
       
      -	r->refs = ref_store_init(r->gitdir, REF_STORE_ALL_CAPS);
      +	r->refs = ref_store_init(r->gitdir,
     -+				 /* XXX r->ref_storage_format == NULL. Where
     -+				  * should the config file be parsed out? */
      +				 r->ref_storage_format ? r->ref_storage_format :
     -+							 "reftable",
     ++							 "files",
      +				 REF_STORE_ALL_CAPS);
       	return r->refs;
       }
     @@ -410,13 +430,14 @@ $^
      +{
      +	struct reftable_ref_store *refs =
      +		(struct reftable_ref_store *)ref_store;
     -+
     ++	FILE *file;
      +	safe_create_dir(refs->reftable_dir, 1);
     -+	FILE *f = fopen(refs->table_list_file, "a");
     -+	if (f == NULL) {
     ++
     ++	file = fopen(refs->table_list_file, "a");
     ++	if (file == NULL) {
      +		return -1;
      +	}
     -+	fclose(f);
     ++	fclose(file);
      +	return 0;
      +}
      +
     @@ -777,7 +798,6 @@ $^
      +			log.old_hash = old_oid.hash;
      +		}
      +
     -+		/* XXX should the resolution be done relative or absolute? */
      +		if (refs_resolve_ref_unsafe((struct ref_store *)create->refs,
      +					    create->target, RESOLVE_REF_READING,
      +					    &new_oid, NULL) != NULL) {
     @@ -1194,9 +1214,7 @@ $^
       	if (worktree)
       		repo_set_worktree(repo, worktree);
       
     -+	repo->ref_storage_format = format.ref_storage != NULL ?
     -+					   xstrdup(format.ref_storage) :
     -+					   "files"; /* XXX */
     ++	repo->ref_storage_format = xstrdup_or_null(format.ref_storage);
      +
       	clear_repository_format(&format);
       	return 0;
     @@ -1227,7 +1245,7 @@ $^
      +		} else if (!strcmp(ext, "worktreeconfig")) {
       			data->worktree_config = git_config_bool(var, value);
      -		else
     -+		} else if (!strcmp(ext, "refStorage")) {
     ++		} else if (!strcmp(ext, "refstorage")) {
      +			data->ref_storage = xstrdup(value);
      +		} else
       			string_list_append(&data->unknown_extensions, ext);
     @@ -1241,3 +1259,16 @@ $^
       	init_repository_format(format);
       }
       
     +@@
     + 				gitdir = DEFAULT_GIT_DIR_ENVIRONMENT;
     + 			setup_git_env(gitdir);
     + 		}
     +-		if (startup_info->have_repository)
     ++		if (startup_info->have_repository) {
     + 			repo_set_hash_algo(the_repository, repo_fmt.hash_algo);
     ++			the_repository->ref_storage_format =
     ++				xstrdup_or_null(repo_fmt.ref_storage);
     ++		}
     + 	}
     + 
     + 	strbuf_release(&dir);

-- 
gitgitgadget