]> git.ipfire.org Git - thirdparty/git.git/blobdiff - sha1_file.c
Merge branches 'sp/maint-fetch-pack-stop-early' and 'sp/maint-upload-pack-stop-early'
[thirdparty/git.git] / sha1_file.c
index d949b35c3308ae5c06774da7f4fecdc84778d5eb..df0edbad1e810727e384a5753001dc8c85d4ecf9 100644 (file)
 #endif
 #endif
 
-#ifdef NO_C99_FORMAT
-#define SZ_FMT "lu"
-static unsigned long sz_fmt(size_t s) { return (unsigned long)s; }
-#else
-#define SZ_FMT "zu"
-static size_t sz_fmt(size_t s) { return s; }
-#endif
+#define SZ_FMT PRIuMAX
+static inline uintmax_t sz_fmt(size_t s) { return s; }
 
 const unsigned char null_sha1[20];
 
@@ -73,6 +68,35 @@ static struct cached_object *find_cached_object(const unsigned char *sha1)
        return NULL;
 }
 
+int mkdir_in_gitdir(const char *path)
+{
+       if (mkdir(path, 0777)) {
+               int saved_errno = errno;
+               struct stat st;
+               struct strbuf sb = STRBUF_INIT;
+
+               if (errno != EEXIST)
+                       return -1;
+               /*
+                * Are we looking at a path in a symlinked worktree
+                * whose original repository does not yet have it?
+                * e.g. .git/rr-cache pointing at its original
+                * repository in which the user hasn't performed any
+                * conflict resolution yet?
+                */
+               if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
+                   strbuf_readlink(&sb, path, st.st_size) ||
+                   !is_absolute_path(sb.buf) ||
+                   mkdir(sb.buf, 0777)) {
+                       strbuf_release(&sb);
+                       errno = saved_errno;
+                       return -1;
+               }
+               strbuf_release(&sb);
+       }
+       return adjust_shared_perm(path);
+}
+
 int safe_create_leading_directories(char *path)
 {
        char *pos = path + offset_1st_component(path);
@@ -418,6 +442,8 @@ static unsigned int pack_used_ctr;
 static unsigned int pack_mmap_calls;
 static unsigned int peak_pack_open_windows;
 static unsigned int pack_open_windows;
+static unsigned int pack_open_fds;
+static unsigned int pack_max_fds;
 static size_t peak_pack_mapped;
 static size_t pack_mapped;
 struct packed_git *packed_git;
@@ -595,8 +621,10 @@ static int unuse_one_window(struct packed_git *current, int keep_fd)
                        lru_l->next = lru_w->next;
                else {
                        lru_p->windows = lru_w->next;
-                       if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
+                       if (!lru_p->windows && lru_p->pack_fd != -1
+                               && lru_p->pack_fd != keep_fd) {
                                close(lru_p->pack_fd);
+                               pack_open_fds--;
                                lru_p->pack_fd = -1;
                        }
                }
@@ -681,8 +709,10 @@ void free_pack_by_name(const char *pack_name)
                if (strcmp(pack_name, p->pack_name) == 0) {
                        clear_delta_base_cache();
                        close_pack_windows(p);
-                       if (p->pack_fd != -1)
+                       if (p->pack_fd != -1) {
                                close(p->pack_fd);
+                               pack_open_fds--;
+                       }
                        close_pack_index(p);
                        free(p->bad_object_sha1);
                        *pp = p->next;
@@ -708,9 +738,29 @@ static int open_packed_git_1(struct packed_git *p)
        if (!p->index_data && open_pack_index(p))
                return error("packfile %s index unavailable", p->pack_name);
 
+       if (!pack_max_fds) {
+               struct rlimit lim;
+               unsigned int max_fds;
+
+               if (getrlimit(RLIMIT_NOFILE, &lim))
+                       die_errno("cannot get RLIMIT_NOFILE");
+
+               max_fds = lim.rlim_cur;
+
+               /* Save 3 for stdin/stdout/stderr, 22 for work */
+               if (25 < max_fds)
+                       pack_max_fds = max_fds - 25;
+               else
+                       pack_max_fds = 1;
+       }
+
+       while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1))
+               ; /* nothing */
+
        p->pack_fd = git_open_noatime(p->pack_name, p);
        if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
                return -1;
+       pack_open_fds++;
 
        /* If we created the struct before we had the pack we lack size. */
        if (!p->pack_size) {
@@ -762,6 +812,7 @@ static int open_packed_git(struct packed_git *p)
                return 0;
        if (p->pack_fd != -1) {
                close(p->pack_fd);
+               pack_open_fds--;
                p->pack_fd = -1;
        }
        return -1;
@@ -787,14 +838,13 @@ unsigned char *use_pack(struct packed_git *p,
 {
        struct pack_window *win = *w_cursor;
 
-       if (p->pack_fd == -1 && open_packed_git(p))
-               die("packfile %s cannot be accessed", p->pack_name);
-
        /* Since packfiles end in a hash of their content and it's
         * pointless to ask for an offset into the middle of that
         * hash, and the in_window function above wouldn't match
         * don't allow an offset too close to the end of the file.
         */
+       if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
+               die("packfile %s cannot be accessed", p->pack_name);
        if (offset > (p->pack_size - 20))
                die("offset beyond end of packfile (truncated pack?)");
 
@@ -808,6 +858,10 @@ unsigned char *use_pack(struct packed_git *p,
                if (!win) {
                        size_t window_align = packed_git_window_size / 2;
                        off_t len;
+
+                       if (p->pack_fd == -1 && open_packed_git(p))
+                               die("packfile %s cannot be accessed", p->pack_name);
+
                        win = xcalloc(1, sizeof(*win));
                        win->offset = (offset / window_align) * window_align;
                        len = p->pack_size - win->offset;
@@ -825,6 +879,12 @@ unsigned char *use_pack(struct packed_git *p,
                                die("packfile %s cannot be mapped: %s",
                                        p->pack_name,
                                        strerror(errno));
+                       if (!win->offset && win->len == p->pack_size
+                               && !p->do_not_close) {
+                               close(p->pack_fd);
+                               pack_open_fds--;
+                               p->pack_fd = -1;
+                       }
                        pack_mmap_calls++;
                        pack_open_windows++;
                        if (pack_mapped > peak_pack_mapped)
@@ -919,6 +979,9 @@ struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
 
 void install_packed_git(struct packed_git *pack)
 {
+       if (pack->pack_fd != -1)
+               pack_open_fds++;
+
        pack->next = packed_git;
        packed_git = pack;
 }
@@ -936,8 +999,6 @@ static void prepare_packed_git_one(char *objdir, int local)
        sprintf(path, "%s/pack", objdir);
        len = strlen(path);
        dir = opendir(path);
-       while (!dir && errno == EMFILE && unuse_one_window(NULL, -1))
-               dir = opendir(path);
        if (!dir) {
                if (errno != ENOENT)
                        error("unable to open object pack directory: %s: %s",
@@ -1093,14 +1154,6 @@ static int git_open_noatime(const char *name, struct packed_git *p)
                if (fd >= 0)
                        return fd;
 
-               /* Might the failure be insufficient file descriptors? */
-               if (errno == EMFILE) {
-                       if (unuse_one_window(p, -1))
-                               continue;
-                       else
-                               return -1;
-               }
-
                /* Might the failure be due to O_NOATIME? */
                if (errno != ENOENT && sha1_file_open_flag) {
                        sha1_file_open_flag = 0;
@@ -1932,6 +1985,27 @@ off_t find_pack_entry_one(const unsigned char *sha1,
        return 0;
 }
 
+static int is_pack_valid(struct packed_git *p)
+{
+       /* An already open pack is known to be valid. */
+       if (p->pack_fd != -1)
+               return 1;
+
+       /* If the pack has one window completely covering the
+        * file size, the pack is known to be valid even if
+        * the descriptor is not currently open.
+        */
+       if (p->windows) {
+               struct pack_window *w = p->windows;
+
+               if (!w->offset && w->len == p->pack_size)
+                       return 1;
+       }
+
+       /* Force the pack to open to prove its valid. */
+       return !open_packed_git(p);
+}
+
 static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
 {
        static struct packed_git *last_found = (void *)1;
@@ -1961,7 +2035,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
                         * it may have been deleted since the index
                         * was loaded!
                         */
-                       if (p->pack_fd == -1 && open_packed_git(p)) {
+                       if (!is_pack_valid(p)) {
                                error("packfile %s cannot be accessed", p->pack_name);
                                goto next;
                        }
@@ -2360,8 +2434,6 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
 
        filename = sha1_file_name(sha1);
        fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename);
-       while (fd < 0 && errno == EMFILE && unuse_one_window(NULL, -1))
-               fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename);
        if (fd < 0) {
                if (errno == EACCES)
                        return error("insufficient permission for adding an object to repository database %s\n", get_object_directory());