threaded delta search: proper locking for cache accounting

[thirdparty/git.git] / builtin-pack-objects.c

diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index b13558ee7ea2623d426bc4af7787e3e488990dda..b126fc8e727f8f9670ba85d9a5bf498f1baf5dfc 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -24,7 +24,7 @@ git-pack-objects [{ -q | --progress | --all-progress }] \n\
        [--max-pack-size=N] [--local] [--incremental] \n\
        [--window=N] [--window-memory=N] [--depth=N] \n\
        [--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
-       [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
+       [--threads=N] [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
        [--stdout | base-name] [<ref-list | <object-list]";
 
 struct object_entry {
@@ -72,6 +72,7 @@ static int progress = 1;
 static int window = 10;
 static uint32_t pack_size_limit;
 static int depth = 50;
+static int delta_search_threads = 1;
 static int pack_to_stdout;
 static int num_preferred_base;
 static struct progress progress_state;
@@ -1300,6 +1301,10 @@ static pthread_mutex_t read_mutex = PTHREAD_MUTEX_INITIALIZER;
 #define read_lock()            pthread_mutex_lock(&read_mutex)
 #define read_unlock()          pthread_mutex_unlock(&read_mutex)
 
+static pthread_mutex_t cache_mutex = PTHREAD_MUTEX_INITIALIZER;
+#define cache_lock()           pthread_mutex_lock(&cache_mutex)
+#define cache_unlock()         pthread_mutex_unlock(&cache_mutex)
+
 static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
 #define progress_lock()                pthread_mutex_lock(&progress_mutex)
 #define progress_unlock()      pthread_mutex_unlock(&progress_mutex)
@@ -1308,6 +1313,8 @@ static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 #define read_lock()            0
 #define read_unlock()          0
+#define cache_lock()           0
+#define cache_unlock()         0
 #define progress_lock()                0
 #define progress_unlock()      0
 
@@ -1422,17 +1429,27 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
        trg_entry->delta_size = delta_size;
        trg->depth = src->depth + 1;
 
+       /*
+        * Handle memory allocation outside of the cache
+        * accounting lock.  Compiler will optimize the strangeness
+        * away when THREADED_DELTA_SEARCH is not defined.
+        */
+       if (trg_entry->delta_data)
+               free(trg_entry->delta_data);
+       cache_lock();
        if (trg_entry->delta_data) {
                delta_cache_size -= trg_entry->delta_size;
-               free(trg_entry->delta_data);
                trg_entry->delta_data = NULL;
        }
-
        if (delta_cacheable(src_size, trg_size, delta_size)) {
-               trg_entry->delta_data = xrealloc(delta_buf, delta_size);
                delta_cache_size += trg_entry->delta_size;
-       } else
+               cache_unlock();
+               trg_entry->delta_data = xrealloc(delta_buf, delta_size);
+       } else {
+               cache_unlock();
                free(delta_buf);
+       }
+
        return 1;
 }
 
@@ -1596,6 +1613,7 @@ static void *threaded_find_deltas(void *arg)
                data_requester = me;
                pthread_mutex_unlock(&data_provider);
                pthread_mutex_lock(&data_ready);
+               pthread_mutex_unlock(&data_request);
 
                if (!me->list_size)
                        return NULL;
@@ -1605,19 +1623,22 @@ static void *threaded_find_deltas(void *arg)
        }
 }
 
-#define NR_THREADS     4
-
 static void ll_find_deltas(struct object_entry **list, unsigned list_size,
                           int window, int depth, unsigned *processed)
 {
-       struct thread_params p[NR_THREADS];
+       struct thread_params *target, p[delta_search_threads];
        int i, ret;
        unsigned chunk_size;
 
+       if (delta_search_threads <= 1) {
+               find_deltas(list, list_size, window, depth, processed);
+               return;
+       }
+
        pthread_mutex_lock(&data_provider);
        pthread_mutex_lock(&data_ready);
 
-       for (i = 0; i < NR_THREADS; i++) {
+       for (i = 0; i < delta_search_threads; i++) {
                p[i].window = window;
                p[i].depth = depth;
                p[i].processed = processed;
@@ -1641,17 +1662,17 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size,
                        sublist_size++;
 
                pthread_mutex_lock(&data_provider);
-               data_requester->list = list;
-               data_requester->list_size = sublist_size;
+               target = data_requester;
+               target->list = list;
+               target->list_size = sublist_size;
                pthread_mutex_unlock(&data_ready);
 
                list += sublist_size;
                list_size -= sublist_size;
                if (!sublist_size) {
-                       pthread_join(data_requester->thread, NULL);
+                       pthread_join(target->thread, NULL);
                        i--;
                }
-               pthread_mutex_unlock(&data_request);
        } while (i);
 }
 
@@ -1741,6 +1762,17 @@ static int git_pack_config(const char *k, const char *v)
                cache_max_small_delta_size = git_config_int(k, v);
                return 0;
        }
+       if (!strcmp(k, "pack.threads")) {
+               delta_search_threads = git_config_int(k, v);
+               if (delta_search_threads < 1)
+                       die("invalid number of threads specified (%d)",
+                           delta_search_threads);
+#ifndef THREADED_DELTA_SEARCH
+               if (delta_search_threads > 1)
+                       warning("no threads support, ignoring %s", k);
+#endif
+               return 0;
+       }
        return git_default_config(k, v);
 }
 
@@ -1900,6 +1932,18 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                                usage(pack_usage);
                        continue;
                }
+               if (!prefixcmp(arg, "--threads=")) {
+                       char *end;
+                       delta_search_threads = strtoul(arg+10, &end, 0);
+                       if (!arg[10] || *end || delta_search_threads < 1)
+                               usage(pack_usage);
+#ifndef THREADED_DELTA_SEARCH
+                       if (delta_search_threads > 1)
+                               warning("no threads support, "
+                                       "ignoring %s", arg);
+#endif
+                       continue;
+               }
                if (!prefixcmp(arg, "--depth=")) {
                        char *end;
                        depth = strtoul(arg+8, &end, 0);