negotiator/default: avoid stack overflow

mark_common() in negotiator/default.c may overflow the stack due to
recursive function calls. Avoid this by instead recursing using a
heap-allocated data structure.

This is the same case as 4654134976f (negotiator/skipping: avoid
stack overflow, 2022-10-25)

Reported-by: Xin Xing <xingxin.xx@bytedance.com>
Signed-off-by: Han Xin <hanxin.hx@bytedance.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/negotiator/default.c b/negotiator/default.c
index f4b78eb47ddd334fecfd53a53ad97b73bcde11ee..635cdd64838cd7b35d0babe225d6f8294f1ee94b 100644 (file)
--- a/negotiator/default.c
+++ b/negotiator/default.c
@@ -55,30 +55,49 @@ static int clear_marks(const char *refname, const struct object_id *oid,
 static void mark_common(struct negotiation_state *ns, struct commit *commit,
                int ancestors_only, int dont_parse)
 {
-       if (commit != NULL && !(commit->object.flags & COMMON)) {
-               struct object *o = (struct object *)commit;
+       struct prio_queue queue = { NULL };
+
+       if (!commit || (commit->object.flags & COMMON))
+               return;
+
+       prio_queue_put(&queue, commit);
+       if (!ancestors_only) {
+               commit->object.flags |= COMMON;
 
-               if (!ancestors_only)
-                       o->flags |= COMMON;
+               if ((commit->object.flags & SEEN) && !(commit->object.flags & POPPED))
+                       ns->non_common_revs--;
+       }
+       while ((commit = prio_queue_get(&queue))) {
+               struct object *o = (struct object *)commit;
 
                if (!(o->flags & SEEN))
                        rev_list_push(ns, commit, SEEN);
                else {
                        struct commit_list *parents;
 
-                       if (!ancestors_only && !(o->flags & POPPED))
-                               ns->non_common_revs--;
                        if (!o->parsed && !dont_parse)
                                if (repo_parse_commit(the_repository, commit))
-                                       return;
+                                       continue;
 
                        for (parents = commit->parents;
                                        parents;
-                                       parents = parents->next)
-                               mark_common(ns, parents->item, 0,
-                                           dont_parse);
+                                       parents = parents->next) {
+                               struct commit *p = parents->item;
+
+                               if (p->object.flags & COMMON)
+                                       continue;
+
+                               p->object.flags |= COMMON;
+
+                               if ((p->object.flags & SEEN) && !(p->object.flags & POPPED))
+                                       ns->non_common_revs--;
+
+                               prio_queue_put(&queue, parents->item);
+                       }
                }
        }
+
+       clear_prio_queue(&queue);
 }
 
 /*