Subject: [PATCH v6 00/21] Integrate commit-graph into 'fsck' and 'gc'

From: Derrick Stolee <[email-masked]>
Date: Fri, 8 Jun 2018 13:56:12 +0000

Sorry for the fast rerolls, but Ævar found some runtime issues with the
string-list use and '\*' as multiplication on some platforms. Thus, v4
and v5 are broken. I did get a repro of those issues using VSTS Linux
build servers and checked that they are fixed in this version.

This version depends on 'next' and sb/object-store-alloc. To accommodate
the repository arguments added by sb/object-store-alloc, the following
diff occurs from the previous patch:

diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c
index 76423b3fa5..c7d0db5ab4 100644
--- a/builtin/commit-graph.c
+++ b/builtin/commit-graph.c
@@ -3,6 +3,7 @@
 #include "dir.h"
 #include "lockfile.h"
 #include "parse-options.h"
+#include "repository.h"
 #include "commit-graph.h"

 static char const * const builtin_commit_graph_usage[] = {
@@ -63,7 +64,7 @@ static int graph_verify(int argc, const char **argv)
        if (!graph)
                return 0;

-       return verify_commit_graph(graph);
+       return verify_commit_graph(the_repository, graph);
 }

 static int graph_read(int argc, const char **argv)
@@ -152,9 +153,9 @@ static int graph_write(int argc, const char **argv)
                return 0;
        }

+       string_list_init(&lines, 0);
        if (opts.stdin_packs || opts.stdin_commits) {
                struct strbuf buf = STRBUF_INIT;
-               string_list_init(&lines, 0);

                while (strbuf_getline(&buf, stdin) != EOF)
                        string_list_append(&lines, strbuf_detach(&buf, NULL));
diff --git a/commit-graph.c b/commit-graph.c
index 0d5adc8035..adf54e3fe7 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -12,6 +12,7 @@
 #include "sha1-lookup.h"
 #include "commit-graph.h"
 #include "object-store.h"
+#include "alloc.h"

 #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
 #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
@@ -866,7 +867,7 @@ static void graph_report(const char *fmt, ...)
 #define GENERATION_ZERO_EXISTS 1
 #define GENERATION_NUMBER_EXISTS 2

-int verify_commit_graph(struct commit_graph *g)
+int verify_commit_graph(struct repository *r, struct commit_graph *g)
 {
        uint32_t i, cur_fanout_pos = 0;
        struct object_id prev_oid, cur_oid, checksum;
@@ -948,7 +949,7 @@ int verify_commit_graph(struct commit_graph *g)
                hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);

                graph_commit = lookup_commit(&cur_oid);
-               odb_commit = (struct commit *)create_object(cur_oid.hash, alloc_commit_node());
+               odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
                if (parse_commit_internal(odb_commit, 0, 0)) {
                        graph_report("failed to parse %s from object database",
                                     oid_to_hex(&cur_oid));
diff --git a/commit-graph.h b/commit-graph.h
index ee20f5e280..506cb45fb1 100644
--- a/commit-graph.h
+++ b/commit-graph.h
@@ -2,6 +2,7 @@
 #define COMMIT_GRAPH_H

 #include "git-compat-util.h"
+#include "repository.h"
 #include "string-list.h"

 char *get_commit_graph_filename(const char *obj_dir);
@@ -53,6 +54,6 @@ void write_commit_graph(const char *obj_dir,
                        struct string_list *commit_hex,
                        int append);

-int verify_commit_graph(struct commit_graph *g);
+int verify_commit_graph(struct repository *r, struct commit_graph *g);

 #endif
diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh
index b24e8b6689..9a0661983c 100755
--- a/t/t5318-commit-graph.sh
+++ b/t/t5318-commit-graph.sh
@@ -33,8 +33,8 @@ test_expect_success 'create commits and repack' '
 '

 graph_git_two_modes() {
-       git -c core.commitGraph=true $1 >output
-       git -c core.commitGraph=false $1 >expect
+       git -c core.graph=true $1 >output
+       git -c core.graph=false $1 >expect
        test_cmp output expect
 }

@@ -282,17 +282,17 @@ GRAPH_CHUNK_LOOKUP_WIDTH=12
 GRAPH_CHUNK_LOOKUP_ROWS=5
 GRAPH_BYTE_OID_FANOUT_ID=$GRAPH_CHUNK_LOOKUP_OFFSET
 GRAPH_BYTE_OID_LOOKUP_ID=$(($GRAPH_CHUNK_LOOKUP_OFFSET + \
-                           1 \* $GRAPH_CHUNK_LOOKUP_WIDTH))
+                           1 * $GRAPH_CHUNK_LOOKUP_WIDTH))
 GRAPH_BYTE_COMMIT_DATA_ID=$(($GRAPH_CHUNK_LOOKUP_OFFSET + \
-                            2 \* $GRAPH_CHUNK_LOOKUP_WIDTH))
+                            2 * $GRAPH_CHUNK_LOOKUP_WIDTH))
 GRAPH_FANOUT_OFFSET=$(($GRAPH_CHUNK_LOOKUP_OFFSET + \
-                      $GRAPH_CHUNK_LOOKUP_WIDTH \* $GRAPH_CHUNK_LOOKUP_ROWS))
-GRAPH_BYTE_FANOUT1=$(($GRAPH_FANOUT_OFFSET + 4 \* 4))
-GRAPH_BYTE_FANOUT2=$(($GRAPH_FANOUT_OFFSET + 4 \* 255))
-GRAPH_OID_LOOKUP_OFFSET=$(($GRAPH_FANOUT_OFFSET + 4 \* 256))
-GRAPH_BYTE_OID_LOOKUP_ORDER=$(($GRAPH_OID_LOOKUP_OFFSET + $HASH_LEN \* 8))
-GRAPH_BYTE_OID_LOOKUP_MISSING=$(($GRAPH_OID_LOOKUP_OFFSET + $HASH_LEN \* 4 + 10))
-GRAPH_COMMIT_DATA_OFFSET=$(($GRAPH_OID_LOOKUP_OFFSET + $HASH_LEN \* $NUM_COMMITS))
+                      $GRAPH_CHUNK_LOOKUP_WIDTH * $GRAPH_CHUNK_LOOKUP_ROWS))
+GRAPH_BYTE_FANOUT1=$(($GRAPH_FANOUT_OFFSET + 4 * 4))
+GRAPH_BYTE_FANOUT2=$(($GRAPH_FANOUT_OFFSET + 4 * 255))
+GRAPH_OID_LOOKUP_OFFSET=$(($GRAPH_FANOUT_OFFSET + 4 * 256))
+GRAPH_BYTE_OID_LOOKUP_ORDER=$(($GRAPH_OID_LOOKUP_OFFSET + $HASH_LEN * 8))
+GRAPH_BYTE_OID_LOOKUP_MISSING=$(($GRAPH_OID_LOOKUP_OFFSET + $HASH_LEN * 4 + 10))
+GRAPH_COMMIT_DATA_OFFSET=$(($GRAPH_OID_LOOKUP_OFFSET + $HASH_LEN * $NUM_COMMITS))
 GRAPH_BYTE_COMMIT_TREE=$GRAPH_COMMIT_DATA_OFFSET
 GRAPH_BYTE_COMMIT_PARENT=$(($GRAPH_COMMIT_DATA_OFFSET + $HASH_LEN))
 GRAPH_BYTE_COMMIT_EXTRA_PARENT=$(($GRAPH_COMMIT_DATA_OFFSET + $HASH_LEN + 4))
@@ -301,9 +301,9 @@ GRAPH_BYTE_COMMIT_GENERATION=$(($GRAPH_COMMIT_DATA_OFFSET + $HASH_LEN + 11))
 GRAPH_BYTE_COMMIT_DATE=$(($GRAPH_COMMIT_DATA_OFFSET + $HASH_LEN + 12))
 GRAPH_COMMIT_DATA_WIDTH=$(($HASH_LEN + 16))
 GRAPH_OCTOPUS_DATA_OFFSET=$(($GRAPH_COMMIT_DATA_OFFSET + \
-                            $GRAPH_COMMIT_DATA_WIDTH \* $NUM_COMMITS))
+                            $GRAPH_COMMIT_DATA_WIDTH * $NUM_COMMITS))
 GRAPH_BYTE_OCTOPUS=$(($GRAPH_OCTOPUS_DATA_OFFSET + 4))
-GRAPH_BYTE_FOOTER=$(($GRAPH_OCTOPUS_DATA_OFFSET + 4 \* $NUM_OCTOPUS_EDGES))
+GRAPH_BYTE_FOOTER=$(($GRAPH_OCTOPUS_DATA_OFFSET + 4 * $NUM_OCTOPUS_EDGES))

 # usage: corrupt_graph_and_verify <position> <data> <string>
 # Manipulates the commit-graph file at the position

---

Derrick Stolee (21):
  commit-graph: UNLEAK before die()
  commit-graph: fix GRAPH_MIN_SIZE
  commit-graph: parse commit from chosen graph
  commit: force commit to parse from object database
  commit-graph: load a root tree from specific graph
  commit-graph: add 'verify' subcommand
  commit-graph: verify catches corrupt signature
  commit-graph: verify required chunks are present
  commit-graph: verify corrupt OID fanout and lookup
  commit-graph: verify objects exist
  commit-graph: verify root tree OIDs
  commit-graph: verify parent list
  commit-graph: verify generation number
  commit-graph: verify commit date
  commit-graph: test for corrupted octopus edge
  commit-graph: verify contents match checksum
  fsck: verify commit-graph
  commit-graph: use string-list API for input
  commit-graph: add '--reachable' option
  gc: automatically write commit-graph files
  commit-graph: update design document

 Documentation/config.txt                 |  10 +-
 Documentation/git-commit-graph.txt       |  14 +-
 Documentation/git-fsck.txt               |   3 +
 Documentation/git-gc.txt                 |   4 +
 Documentation/technical/commit-graph.txt |  22 --
 builtin/commit-graph.c                   |  99 ++++++---
 builtin/fsck.c                           |  21 ++
 builtin/gc.c                             |   6 +
 commit-graph.c                           | 249 +++++++++++++++++++++--
 commit-graph.h                           |  11 +-
 commit.c                                 |   9 +-
 commit.h                                 |   1 +
 t/t5318-commit-graph.sh                  | 201 ++++++++++++++++++
 13 files changed, 572 insertions(+), 78 deletions(-)

-- 
2.18.0.rc1