#include "packfile.h"
#include "object-store.h"
#include "promisor-remote.h"
+#include "submodule.h"
/* The maximum size for an object header. */
#define MAX_HEADER_LEN 32
return result;
}
-int raceproof_create_file(const char *path, create_file_fn fn, void *cb)
-{
- /*
- * The number of times we will try to remove empty directories
- * in the way of path. This is only 1 because if another
- * process is racily creating directories that conflict with
- * us, we don't want to fight against them.
- */
- int remove_directories_remaining = 1;
-
- /*
- * The number of times that we will try to create the
- * directories containing path. We are willing to attempt this
- * more than once, because another process could be trying to
- * clean up empty directories at the same time as we are
- * trying to create them.
- */
- int create_directories_remaining = 3;
-
- /* A scratch copy of path, filled lazily if we need it: */
- struct strbuf path_copy = STRBUF_INIT;
-
- int ret, save_errno;
-
- /* Sanity check: */
- assert(*path);
-
-retry_fn:
- ret = fn(path, cb);
- save_errno = errno;
- if (!ret)
- goto out;
-
- if (errno == EISDIR && remove_directories_remaining-- > 0) {
- /*
- * A directory is in the way. Maybe it is empty; try
- * to remove it:
- */
- if (!path_copy.len)
- strbuf_addstr(&path_copy, path);
-
- if (!remove_dir_recursively(&path_copy, REMOVE_DIR_EMPTY_ONLY))
- goto retry_fn;
- } else if (errno == ENOENT && create_directories_remaining-- > 0) {
- /*
- * Maybe the containing directory didn't exist, or
- * maybe it was just deleted by a process that is
- * racing with us to clean up empty directories. Try
- * to create it:
- */
- enum scld_error scld_result;
-
- if (!path_copy.len)
- strbuf_addstr(&path_copy, path);
-
- do {
- scld_result = safe_create_leading_directories(path_copy.buf);
- if (scld_result == SCLD_OK)
- goto retry_fn;
- } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0);
- }
-
-out:
- strbuf_release(&path_copy);
- errno = save_errno;
- return ret;
-}
-
static void fill_loose_path(struct strbuf *buf, const struct object_id *oid)
{
int i;
*/
static int alt_odb_usable(struct raw_object_store *o,
struct strbuf *path,
- const char *normalized_objdir)
+ const char *normalized_objdir, khiter_t *pos)
{
- struct object_directory *odb;
+ int r;
/* Detect cases where alternate disappeared */
if (!is_directory(path->buf)) {
* Prevent the common mistake of listing the same
* thing twice, or object directory itself.
*/
- for (odb = o->odb; odb; odb = odb->next) {
- if (!fspathcmp(path->buf, odb->path))
- return 0;
+ if (!o->odb_by_path) {
+ khiter_t p;
+
+ o->odb_by_path = kh_init_odb_path_map();
+ assert(!o->odb->next);
+ p = kh_put_odb_path_map(o->odb_by_path, o->odb->path, &r);
+ assert(r == 1); /* never used */
+ kh_value(o->odb_by_path, p) = o->odb;
}
- if (!fspathcmp(path->buf, normalized_objdir))
+ if (fspatheq(path->buf, normalized_objdir))
return 0;
-
- return 1;
+ *pos = kh_put_odb_path_map(o->odb_by_path, path->buf, &r);
+ /* r: 0 = exists, 1 = never used, 2 = deleted */
+ return r == 0 ? 0 : 1;
}
/*
static void read_info_alternates(struct repository *r,
const char *relative_base,
int depth);
-static int link_alt_odb_entry(struct repository *r, const char *entry,
+static int link_alt_odb_entry(struct repository *r, const struct strbuf *entry,
const char *relative_base, int depth, const char *normalized_objdir)
{
struct object_directory *ent;
struct strbuf pathbuf = STRBUF_INIT;
+ khiter_t pos;
- if (!is_absolute_path(entry) && relative_base) {
+ if (!is_absolute_path(entry->buf) && relative_base) {
strbuf_realpath(&pathbuf, relative_base, 1);
strbuf_addch(&pathbuf, '/');
}
- strbuf_addstr(&pathbuf, entry);
+ strbuf_addbuf(&pathbuf, entry);
if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
error(_("unable to normalize alternate object path: %s"),
while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
strbuf_setlen(&pathbuf, pathbuf.len - 1);
- if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir)) {
+ if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir, &pos)) {
strbuf_release(&pathbuf);
return -1;
}
CALLOC_ARRAY(ent, 1);
- ent->path = xstrdup(pathbuf.buf);
+ /* pathbuf.buf is already in r->objects->odb_by_path */
+ ent->path = strbuf_detach(&pathbuf, NULL);
/* add the alternate entry */
*r->objects->odb_tail = ent;
r->objects->odb_tail = &(ent->next);
ent->next = NULL;
+ assert(r->objects->odb_by_path);
+ kh_value(r->objects->odb_by_path, pos) = ent;
/* recursively add alternates */
- read_info_alternates(r, pathbuf.buf, depth + 1);
+ read_info_alternates(r, ent->path, depth + 1);
- strbuf_release(&pathbuf);
return 0;
}
alt = parse_alt_odb_entry(alt, sep, &entry);
if (!entry.len)
continue;
- link_alt_odb_entry(r, entry.buf,
+ link_alt_odb_entry(r, &entry,
relative_base, depth, objdirbuf.buf);
}
strbuf_release(&entry);
return ref_git;
}
+struct object_directory *find_odb(struct repository *r, const char *obj_dir)
+{
+ struct object_directory *odb;
+ char *obj_dir_real = real_pathdup(obj_dir, 1);
+ struct strbuf odb_path_real = STRBUF_INIT;
+
+ prepare_alt_odb(r);
+ for (odb = r->objects->odb; odb; odb = odb->next) {
+ strbuf_realpath(&odb_path_real, odb->path, 1);
+ if (!strcmp(obj_dir_real, odb_path_real.buf))
+ break;
+ }
+
+ free(obj_dir_real);
+ strbuf_release(&odb_path_real);
+
+ if (!odb)
+ die(_("could not find object directory matching %s"), obj_dir);
+ return odb;
+}
+
static void fill_alternate_refs_command(struct child_process *cmd,
const char *repo_path)
{
return ret;
}
+const char *mmap_os_err(void)
+{
+ static const char blank[] = "";
+#if defined(__linux__)
+ if (errno == ENOMEM) {
+ /* this continues an existing error message: */
+ static const char enomem[] =
+", check sys.vm.max_map_count and/or RLIMIT_DATA";
+ return enomem;
+ }
+#endif /* OS-specific bits */
+ return blank;
+}
+
void *xmmap(void *start, size_t length,
int prot, int flags, int fd, off_t offset)
{
void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
if (ret == MAP_FAILED)
- die_errno(_("mmap failed"));
+ die_errno(_("mmap failed%s"), mmap_os_err());
return ret;
}
* the streaming interface and rehash it to do the same.
*/
int check_object_signature(struct repository *r, const struct object_id *oid,
- void *map, unsigned long size, const char *type)
+ void *map, unsigned long size, const char *type,
+ struct object_id *real_oidp)
{
- struct object_id real_oid;
+ struct object_id tmp;
+ struct object_id *real_oid = real_oidp ? real_oidp : &tmp;
enum object_type obj_type;
struct git_istream *st;
git_hash_ctx c;
int hdrlen;
if (map) {
- hash_object_file(r->hash_algo, map, size, type, &real_oid);
- return !oideq(oid, &real_oid) ? -1 : 0;
+ hash_object_file(r->hash_algo, map, size, type, real_oid);
+ return !oideq(oid, real_oid) ? -1 : 0;
}
st = open_istream(r, oid, &obj_type, &size, NULL);
break;
r->hash_algo->update_fn(&c, buf, readlen);
}
- r->hash_algo->final_oid_fn(&real_oid, &c);
+ r->hash_algo->final_oid_fn(real_oid, &c);
close_istream(st);
- return !oideq(oid, &real_oid) ? -1 : 0;
+ return !oideq(oid, real_oid) ? -1 : 0;
}
int git_open_cloexec(const char *name, int flags)
prepare_alt_odb(r);
for (odb = r->objects->odb; odb; odb = odb->next) {
- if (oid_array_lookup(odb_loose_cache(odb, oid), oid) >= 0)
+ if (oidtree_contains(odb_loose_cache(odb, oid), oid))
return 1;
}
return 0;
return map_loose_object_1(r, NULL, oid, size);
}
-static int unpack_loose_short_header(git_zstream *stream,
- unsigned char *map, unsigned long mapsize,
- void *buffer, unsigned long bufsiz)
+enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
+ unsigned char *map,
+ unsigned long mapsize,
+ void *buffer,
+ unsigned long bufsiz,
+ struct strbuf *header)
{
- int ret;
+ int status;
/* Get the data stream */
memset(stream, 0, sizeof(*stream));
git_inflate_init(stream);
obj_read_unlock();
- ret = git_inflate(stream, 0);
+ status = git_inflate(stream, 0);
obj_read_lock();
-
- return ret;
-}
-
-int unpack_loose_header(git_zstream *stream,
- unsigned char *map, unsigned long mapsize,
- void *buffer, unsigned long bufsiz)
-{
- int status = unpack_loose_short_header(stream, map, mapsize,
- buffer, bufsiz);
-
- if (status < Z_OK)
- return status;
-
- /* Make sure we have the terminating NUL */
- if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
- return -1;
- return 0;
-}
-
-static int unpack_loose_header_to_strbuf(git_zstream *stream, unsigned char *map,
- unsigned long mapsize, void *buffer,
- unsigned long bufsiz, struct strbuf *header)
-{
- int status;
-
- status = unpack_loose_short_header(stream, map, mapsize, buffer, bufsiz);
if (status < Z_OK)
- return -1;
+ return ULHR_BAD;
/*
* Check if entire header is unpacked in the first iteration.
*/
if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
- return 0;
+ return ULHR_OK;
+
+ /*
+ * We have a header longer than MAX_HEADER_LEN. The "header"
+ * here is only non-NULL when we run "cat-file
+ * --allow-unknown-type".
+ */
+ if (!header)
+ return ULHR_TOO_LONG;
/*
* buffer[0..bufsiz] was not large enough. Copy the partial
stream->next_out = buffer;
stream->avail_out = bufsiz;
} while (status != Z_STREAM_END);
- return -1;
+ return ULHR_TOO_LONG;
}
static void *unpack_loose_rest(git_zstream *stream,
* too permissive for what we want to check. So do an anal
* object header parse by hand.
*/
-static int parse_loose_header_extended(const char *hdr, struct object_info *oi,
- unsigned int flags)
+int parse_loose_header(const char *hdr, struct object_info *oi)
{
const char *type_buf = hdr;
size_t size;
type = type_from_string_gently(type_buf, type_len, 1);
if (oi->type_name)
strbuf_add(oi->type_name, type_buf, type_len);
- /*
- * Set type to 0 if its an unknown object and
- * we're obtaining the type using '--allow-unknown-type'
- * option.
- */
- if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
- type = 0;
- else if (type < 0)
- die(_("invalid object type"));
if (oi->typep)
*oi->typep = type;
/*
* The length must be followed by a zero byte
*/
- return *hdr ? -1 : type;
-}
-
-int parse_loose_header(const char *hdr, unsigned long *sizep)
-{
- struct object_info oi = OBJECT_INFO_INIT;
+ if (*hdr)
+ return -1;
- oi.sizep = sizep;
- return parse_loose_header_extended(hdr, &oi, 0);
+ /*
+ * The format is valid, but the type may still be bogus. The
+ * Caller needs to check its oi->typep.
+ */
+ return 0;
}
static int loose_object_info(struct repository *r,
char hdr[MAX_HEADER_LEN];
struct strbuf hdrbuf = STRBUF_INIT;
unsigned long size_scratch;
+ enum object_type type_scratch;
+ int allow_unknown = flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE;
if (oi->delta_base_oid)
oidclr(oi->delta_base_oid);
if (!oi->sizep)
oi->sizep = &size_scratch;
+ if (!oi->typep)
+ oi->typep = &type_scratch;
if (oi->disk_sizep)
*oi->disk_sizep = mapsize;
- if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) {
- if (unpack_loose_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
- status = error(_("unable to unpack %s header with --allow-unknown-type"),
- oid_to_hex(oid));
- } else if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
+
+ switch (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr),
+ allow_unknown ? &hdrbuf : NULL)) {
+ case ULHR_OK:
+ if (parse_loose_header(hdrbuf.len ? hdrbuf.buf : hdr, oi) < 0)
+ status = error(_("unable to parse %s header"), oid_to_hex(oid));
+ else if (!allow_unknown && *oi->typep < 0)
+ die(_("invalid object type"));
+
+ if (!oi->contentp)
+ break;
+ *oi->contentp = unpack_loose_rest(&stream, hdr, *oi->sizep, oid);
+ if (*oi->contentp)
+ goto cleanup;
+
+ status = -1;
+ break;
+ case ULHR_BAD:
status = error(_("unable to unpack %s header"),
oid_to_hex(oid));
- if (status < 0)
- ; /* Do nothing */
- else if (hdrbuf.len) {
- if ((status = parse_loose_header_extended(hdrbuf.buf, oi, flags)) < 0)
- status = error(_("unable to parse %s header with --allow-unknown-type"),
- oid_to_hex(oid));
- } else if ((status = parse_loose_header_extended(hdr, oi, flags)) < 0)
- status = error(_("unable to parse %s header"), oid_to_hex(oid));
-
- if (status >= 0 && oi->contentp) {
- *oi->contentp = unpack_loose_rest(&stream, hdr,
- *oi->sizep, oid);
- if (!*oi->contentp) {
- git_inflate_end(&stream);
- status = -1;
- }
- } else
- git_inflate_end(&stream);
+ break;
+ case ULHR_TOO_LONG:
+ status = error(_("header for %s too long, exceeds %d bytes"),
+ oid_to_hex(oid), MAX_HEADER_LEN);
+ break;
+ }
+ git_inflate_end(&stream);
+cleanup:
munmap(map, mapsize);
- if (status && oi->typep)
- *oi->typep = status;
if (oi->sizep == &size_scratch)
oi->sizep = NULL;
strbuf_release(&hdrbuf);
+ if (oi->typep == &type_scratch)
+ oi->typep = NULL;
oi->whence = OI_LOOSE;
- return (status < 0) ? status : 0;
+ return status;
}
int obj_read_use_lock = 0;
break;
}
+ /*
+ * If r is the_repository, this might be an attempt at
+ * accessing a submodule object as if it were in the_repository
+ * (having called add_submodule_odb() on that submodule's ODB).
+ * If any such ODBs exist, register them and try again.
+ */
+ if (r == the_repository &&
+ register_all_submodule_odb_as_alternates())
+ /* We added some alternates; retry */
+ continue;
+
/* Check if it is a missing object */
- if (fetch_if_missing && has_promisor_remote() &&
- !already_retried && r == the_repository &&
+ if (fetch_if_missing && repo_has_promisor_remote(r) &&
+ !already_retried &&
!(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
/*
* TODO Investigate checking promisor_remote_get_direct()
* TODO return value and stopping on error here.
- * TODO Pass a repository struct through
- * promisor_remote_get_direct(), such that arbitrary
- * repositories work.
*/
promisor_remote_get_direct(r, real, 1);
already_retried = 1;
return 0;
rtype = packed_object_info(r, e.p, e.offset, oi);
if (rtype < 0) {
- mark_bad_packed_object(e.p, real->hash);
+ mark_bad_packed_object(e.p, real);
return do_oid_object_info_extended(r, real, oi, 0);
} else if (oi->whence == OI_PACKED) {
oi->u.packed.offset = e.offset;
die(_("loose object %s (stored in %s) is corrupt"),
oid_to_hex(repl), path);
- if ((p = has_packed_and_bad(r, repl->hash)) != NULL)
+ if ((p = has_packed_and_bad(r, repl)) != NULL)
die(_("packed object %s (stored in %s) is corrupt"),
oid_to_hex(repl), p->pack_name);
obj_read_unlock();
static int write_loose_object(const struct object_id *oid, char *hdr,
int hdrlen, const void *buf, unsigned long len,
- time_t mtime)
+ time_t mtime, unsigned flags)
{
int fd, ret;
unsigned char compressed[4096];
fd = create_tmpfile(&tmp_file, filename.buf);
if (fd < 0) {
- if (errno == EACCES)
+ if (flags & HASH_SILENT)
+ return -1;
+ else if (errno == EACCES)
return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
else
return error_errno(_("unable to create temporary file"));
struct utimbuf utb;
utb.actime = mtime;
utb.modtime = mtime;
- if (utime(tmp_file.buf, &utb) < 0)
+ if (utime(tmp_file.buf, &utb) < 0 &&
+ !(flags & HASH_SILENT))
warning_errno(_("failed utime() on %s"), tmp_file.buf);
}
return 1;
}
-int write_object_file(const void *buf, unsigned long len, const char *type,
- struct object_id *oid)
+int write_object_file_flags(const void *buf, unsigned long len,
+ const char *type, struct object_id *oid,
+ unsigned flags)
{
char hdr[MAX_HEADER_LEN];
int hdrlen = sizeof(hdr);
&hdrlen);
if (freshen_packed_object(oid) || freshen_loose_object(oid))
return 0;
- return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
+ return write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags);
}
int hash_object_file_literally(const void *buf, unsigned long len,
goto cleanup;
if (freshen_packed_object(oid) || freshen_loose_object(oid))
goto cleanup;
- status = write_loose_object(oid, header, hdrlen, buf, len, 0);
+ status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0);
cleanup:
free(header);
if (!buf)
return error(_("cannot read object for %s"), oid_to_hex(oid));
hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
- ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime);
+ ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0);
free(buf);
return ret;
static int append_loose_object(const struct object_id *oid, const char *path,
void *data)
{
- oid_array_append(data, oid);
+ oidtree_insert(data, oid);
return 0;
}
-struct oid_array *odb_loose_cache(struct object_directory *odb,
+struct oidtree *odb_loose_cache(struct object_directory *odb,
const struct object_id *oid)
{
int subdir_nr = oid->hash[0];
struct strbuf buf = STRBUF_INIT;
+ size_t word_bits = bitsizeof(odb->loose_objects_subdir_seen[0]);
+ size_t word_index = subdir_nr / word_bits;
+ size_t mask = 1u << (subdir_nr % word_bits);
+ uint32_t *bitmap;
if (subdir_nr < 0 ||
- subdir_nr >= ARRAY_SIZE(odb->loose_objects_subdir_seen))
+ subdir_nr >= bitsizeof(odb->loose_objects_subdir_seen))
BUG("subdir_nr out of range");
- if (odb->loose_objects_subdir_seen[subdir_nr])
- return &odb->loose_objects_cache[subdir_nr];
-
+ bitmap = &odb->loose_objects_subdir_seen[word_index];
+ if (*bitmap & mask)
+ return odb->loose_objects_cache;
+ if (!odb->loose_objects_cache) {
+ ALLOC_ARRAY(odb->loose_objects_cache, 1);
+ oidtree_init(odb->loose_objects_cache);
+ }
strbuf_addstr(&buf, odb->path);
for_each_file_in_obj_subdir(subdir_nr, &buf,
append_loose_object,
NULL, NULL,
- &odb->loose_objects_cache[subdir_nr]);
- odb->loose_objects_subdir_seen[subdir_nr] = 1;
+ odb->loose_objects_cache);
+ *bitmap |= mask;
strbuf_release(&buf);
- return &odb->loose_objects_cache[subdir_nr];
+ return odb->loose_objects_cache;
}
void odb_clear_loose_cache(struct object_directory *odb)
{
- int i;
-
- for (i = 0; i < ARRAY_SIZE(odb->loose_objects_cache); i++)
- oid_array_clear(&odb->loose_objects_cache[i]);
+ oidtree_clear(odb->loose_objects_cache);
+ FREE_AND_NULL(odb->loose_objects_cache);
memset(&odb->loose_objects_subdir_seen, 0,
sizeof(odb->loose_objects_subdir_seen));
}
int read_loose_object(const char *path,
const struct object_id *expected_oid,
- enum object_type *type,
- unsigned long *size,
- void **contents)
+ struct object_id *real_oid,
+ void **contents,
+ struct object_info *oi)
{
int ret = -1;
void *map = NULL;
unsigned long mapsize;
git_zstream stream;
char hdr[MAX_HEADER_LEN];
-
- *contents = NULL;
+ unsigned long *size = oi->sizep;
map = map_loose_object_1(the_repository, path, NULL, &mapsize);
if (!map) {
goto out;
}
- if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) {
+ if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr),
+ NULL) < 0) {
error(_("unable to unpack header of %s"), path);
goto out;
}
- *type = parse_loose_header(hdr, size);
- if (*type < 0) {
+ if (parse_loose_header(hdr, oi) < 0) {
error(_("unable to parse header of %s"), path);
git_inflate_end(&stream);
goto out;
}
- if (*type == OBJ_BLOB && *size > big_file_threshold) {
+ if (*oi->typep == OBJ_BLOB && *size > big_file_threshold) {
if (check_stream_oid(&stream, hdr, *size, path, expected_oid) < 0)
goto out;
} else {
}
if (check_object_signature(the_repository, expected_oid,
*contents, *size,
- type_name(*type))) {
- error(_("hash mismatch for %s (expected %s)"), path,
- oid_to_hex(expected_oid));
- free(*contents);
+ oi->type_name->buf, real_oid))
goto out;
- }
}
ret = 0; /* everything checks out */
projects / thirdparty / git.git / blobdiff