Skip to content

Commit

Permalink
remote: introduce config to set prefetch refs
Browse files Browse the repository at this point in the history
This commit introduces a new configuration option, remote.<name>.prefetchref,
which allows users to specify specific ref patterns to be prefetched during a
git fetch --prefetch operation.

The new option accepts a space-separated list of ref patterns. When the
--prefetch option is used with git fetch, only the refs matching these
patterns will be prefetched, instead of the default behavior of prefetching
all fetchable refs.

Example usage in .git/config:
[remote "origin"]
    prefetchref = "refs/heads/main refs/heads/feature/*"

This change allows users to optimize their prefetch operations, potentially
reducing network traffic and improving performance for large repositories
with many refs.

Signed-off-by: Shubham Kanodia <[email protected]>
  • Loading branch information
pastelsky committed Sep 7, 2024
1 parent 2e7b89e commit 86a34d1
Show file tree
Hide file tree
Showing 5 changed files with 125 additions and 2 deletions.
6 changes: 6 additions & 0 deletions Documentation/config/remote.txt
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ remote.<name>.fetch::
The default set of "refspec" for linkgit:git-fetch[1]. See
linkgit:git-fetch[1].

remote.<name>.prefetchref::
Specify the refs to be prefetched when fetching from this remote.
The value is a space-separated list of ref patterns (e.g., "refs/heads/master refs/heads/develop*").
These patterns are used as the source part of the refspecs for prefetching.
This can be used to optimize fetch operations by specifying exactly which refs should be prefetched.

remote.<name>.push::
The default set of "refspec" for linkgit:git-push[1]. See
linkgit:git-push[1].
Expand Down
40 changes: 38 additions & 2 deletions builtin/fetch.c
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,40 @@ static void find_non_local_tags(const struct ref *refs,
oidset_clear(&fetch_oids);
}

static struct refspec get_prefetch_refspec(struct remote *remote) {
if (remote->prefetch_refs && remote->prefetch_refs.nr > 0) {

Check failure on line 438 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / linux32 (daald/ubuntu32:xenial)

builtin/fetch.c:438:6: used struct type value where scalar is required

Check failure on line 438 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / linux-musl (alpine)

builtin/fetch.c:438:13: used struct type value where scalar is required

Check failure on line 438 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / pedantic (fedora)

builtin/fetch.c:438:13: used struct type value where scalar is required

Check failure on line 438 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / win build

builtin/fetch.c:438:13: used struct type value where scalar is required
struct refspec rs = REFSPEC_INIT_FETCH;
int i;
// Clear existing refspecs
for (i = 0; i < rs.nr; i++) {
free(rs.items[i].src);
free(rs.items[i].dst);
}
rs.nr = 0;

// Add prefetch refs
for (i = 0; i < remote->prefetch_refs.nr; i++) {
const char *src = remote->prefetch_refs.items[i].string;
struct strbuf dst = STRBUF_INIT;

strbuf_addf(&dst, "refs/remotes/%s/", remote->name);
if (starts_with(src, "refs/heads/"))
strbuf_addstr(&dst, src + 11);
else if (starts_with(src, "refs/"))
strbuf_addstr(&dst, src + 5);
else
strbuf_addstr(&dst, src);

refspec_appendf(&rs, "%s:%s", src, dst.buf);
strbuf_release(&dst);
}
return rs;
} else {
return remote->fetch;
}

}

Check failure on line 469 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / linux32 (daald/ubuntu32:xenial)

builtin/fetch.c:469:1: control reaches end of non-void function [-Werror=return-type]

Check failure on line 469 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / linux-musl (alpine)

builtin/fetch.c:469:1: control reaches end of non-void function [-Werror=return-type]

Check failure on line 469 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / pedantic (fedora)

builtin/fetch.c:469:1: control reaches end of non-void function [-Werror=return-type]

Check failure on line 469 in builtin/fetch.c

View workflow job for this annotation

GitHub Actions / win build

builtin/fetch.c:469:1: control reaches end of non-void function [-Werror=return-type]

static void filter_prefetch_refspec(struct refspec *rs)
{
int i;
Expand Down Expand Up @@ -502,8 +536,10 @@ static struct ref *get_ref_map(struct remote *remote,
int existing_refs_populated = 0;

filter_prefetch_refspec(rs);
if (remote)
filter_prefetch_refspec(&remote->fetch);
if (remote) {
struct refspec prefetch_rs = get_prefetch_refspec(remote);
filter_prefetch_refspec(&prefetch_rs);
}

if (rs->nr) {
struct refspec *fetch_refspec;
Expand Down
8 changes: 8 additions & 0 deletions remote.c
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ static struct remote *make_remote(struct remote_state *remote_state,
ret->prune = -1; /* unspecified */
ret->prune_tags = -1; /* unspecified */
ret->name = xstrndup(name, len);
string_list_init_dup(&ret->prefetch_refs);
refspec_init(&ret->push, REFSPEC_PUSH);
refspec_init(&ret->fetch, REFSPEC_FETCH);

Expand All @@ -166,6 +167,7 @@ static void remote_clear(struct remote *remote)
free((char *)remote->uploadpack);
FREE_AND_NULL(remote->http_proxy);
FREE_AND_NULL(remote->http_proxy_authmethod);
string_list_clear(&remote->prefetch_refs, 0);
}

static void add_merge(struct branch *branch, const char *name)
Expand Down Expand Up @@ -456,6 +458,12 @@ static int handle_config(const char *key, const char *value,
remote->prune = git_config_bool(key, value);
else if (!strcmp(subkey, "prunetags"))
remote->prune_tags = git_config_bool(key, value);
else if (!strcmp(subkey, "prefetchref")) {
if (!value)
return config_error_nonbool(key);
string_list_split(&remote->prefetch_refs, value, ' ', -1);
return 0;
}
else if (!strcmp(subkey, "url")) {
if (!value)
return config_error_nonbool(key);
Expand Down
3 changes: 3 additions & 0 deletions remote.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include "hashmap.h"
#include "refspec.h"
#include "strvec.h"
#include "string-list.h"

struct option;
struct transport_ls_refs_options;
Expand Down Expand Up @@ -77,6 +78,8 @@ struct remote {

struct refspec fetch;

struct string_list prefetch_refs;

/*
* The setting for whether to fetch tags (as a separate rule from the
* configured refspecs);
Expand Down
70 changes: 70 additions & 0 deletions t/t7900-maintenance.sh
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,76 @@ test_expect_success 'prefetch multiple remotes' '
test_subcommand git fetch remote2 $fetchargs <skip-remote1.txt
'

test_expect_success 'prefetch respects single and multiple remote.<name>.prefetchref patterns' '
test_create_repo prefetch-test-mixed-patterns &&
(
cd prefetch-test-mixed-patterns &&
test_commit initial &&
git clone . clone1 &&
git clone . clone2 &&
git remote add remote1 "file://$(pwd)/clone1" &&
git remote add remote2 "file://$(pwd)/clone2" &&
# Set single prefetchref pattern for remote1 and multiple for remote2
git config remote.remote1.prefetchref "refs/heads/main" &&
git config remote.remote2.prefetchref "refs/heads/feature/* refs/heads/topic" &&
# Create branches in clone1 and push
(
cd clone1 &&
git checkout -b main &&
test_commit main-commit &&
git checkout -b feature/a &&
test_commit feature-a-commit &&
git checkout -b other &&
test_commit other-commit &&
git push origin main feature/a other
) &&
# Create branches in clone2 and push
(
cd clone2 &&
git checkout -b topic &&
test_commit master-commit &&
git checkout -b feature/x &&
test_commit feature-x-commit &&
git checkout -b feature/y &&
test_commit feature-y-commit &&
git checkout -b dev &&
test_commit dev-commit &&
git push origin topic feature/x feature/y dev
) &&
# Run maintenance prefetch task
GIT_TRACE2_EVENT="$(pwd)/prefetch.txt" git maintenance run --task=prefetch 2>/dev/null &&
# Check that only specified refs were prefetched
fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" &&
test_subcommand git fetch remote1 $fetchargs <prefetch.txt &&
test_subcommand git fetch remote2 $fetchargs <prefetch.txt &&
ls -R .git/refs/prefetch &&
# Verify that only specified refs are in the prefetch refs for remote1
git rev-parse refs/prefetch/remotes/remote1/main &&
test_must_fail git rev-parse refs/prefetch/remotes/remote1/feature/a &&
test_must_fail git rev-parse refs/prefetch/remotes/remote1/other &&
# # Verify that only specified refs are in the prefetch refs for remote2
git rev-parse refs/prefetch/remotes/remote2/feature/x &&
git rev-parse refs/prefetch/remotes/remote2/feature/y &&
git rev-parse refs/prefetch/remotes/remote2/topic &&
test_must_fail git rev-parse refs/prefetch/remotes/remote2/dev &&
# Fetch all refs and compare
git fetch --all &&
test_cmp_rev refs/remotes/remote1/main refs/prefetch/remotes/remote1/main &&
test_cmp_rev refs/remotes/remote2/feature/x refs/prefetch/remotes/remote2/feature/x &&
test_cmp_rev refs/remotes/remote2/feature/y refs/prefetch/remotes/remote2/feature/y &&
test_cmp_rev refs/remotes/remote2/topic refs/prefetch/remotes/remote2/topic
)
'

test_expect_success 'loose-objects task' '
# Repack everything so we know the state of the object dir
git repack -adk &&
Expand Down

0 comments on commit 86a34d1

Please sign in to comment.