libgomp/testsuite/libgomp.c/affinity-1.c

   1 /* Affinity tests.
   2    Copyright (C) 2013-2021 Free Software Foundation, Inc.
   3
   4    GCC is free software; you can redistribute it and/or modify it under
   5    the terms of the GNU General Public License as published by the Free
   6    Software Foundation; either version 3, or (at your option) any later
   7    version.
   8
   9    GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  10    WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12    for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with GCC; see the file COPYING3.  If not see
  16    <http://www.gnu.org/licenses/>.  */
  17
  18 /* { dg-do run } */
  19 /* { dg-set-target-env-var OMP_PROC_BIND "false" } */
  20 /* { dg-additional-options "-Wno-deprecated-declarations" } */
  21 /* { dg-additional-options "-DINTERPOSE_GETAFFINITY -DDO_FORK -ldl -Wno-deprecated-declarations" { target *-*-linux* } } */
  22
  23 #ifndef _GNU_SOURCE
  24 #define _GNU_SOURCE
  25 #endif
  26 #include "config.h"
  27 #include <omp.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <unistd.h>
  32
  33 #ifdef DO_FORK
  34 #include <signal.h>
  35 #include <sys/wait.h>
  36 #endif
  37 #ifdef HAVE_PTHREAD_AFFINITY_NP
  38 #include <sched.h>
  39 #include <pthread.h>
  40 #ifdef INTERPOSE_GETAFFINITY
  41 #include <dlfcn.h>
  42 #endif
  43 #endif
  44
  45 struct place
  46 {
  47   int start, len;
  48 };
  49 struct places
  50 {
  51   char name[40];
  52   int count;
  53   struct place places[8];
  54 } places_array[] = {
  55   { "", 1, { { -1, -1 } } },
  56   { "{0}:8", 8,
  57     { { 0, 1 }, { 1, 1 }, { 2, 1 }, { 3, 1 },
  58       { 4, 1 }, { 5, 1 }, { 6, 1 }, { 7, 1 } } },
  59   { "{7,6}:2:-3", 2, { { 6, 2 }, { 3, 2 } } },
  60   { "{6,7}:4:-2,!{2,3}", 3, { { 6, 2 }, { 4, 2 }, { 0, 2 } } },
  61   { "{1}:7:1", 7,
  62     { { 1, 1 }, { 2, 1 }, { 3, 1 },
  63       { 4, 1 }, { 5, 1 }, { 6, 1 }, { 7, 1 } } },
  64   { "{0,1},{3,2,4},{6,5,!6},{6},{7:2:-1,!6}", 5,
  65     { { 0, 2 }, { 2, 3 }, { 5, 1 }, { 6, 1 }, { 7, 1 } } }
  66 };
  67
  68 unsigned long contig_cpucount;
  69 unsigned long min_cpusetsize;
  70
  71 #if defined (HAVE_PTHREAD_AFFINITY_NP) && defined (_SC_NPROCESSORS_CONF) \
  72     && defined (CPU_ALLOC_SIZE)
  73
  74 #if defined (RTLD_NEXT) && defined (INTERPOSE_GETAFFINITY)
  75 int (*orig_getaffinity_np) (pthread_t, size_t, cpu_set_t *);
  76
  77 int
  78 pthread_getaffinity_np (pthread_t thread, size_t cpusetsize, cpu_set_t *cpuset)
  79 {
  80   int ret;
  81   unsigned long i, max;
  82   if (orig_getaffinity_np == NULL)
  83     {
  84       orig_getaffinity_np = (int (*) (pthread_t, size_t, cpu_set_t *))
  85                             dlsym (RTLD_NEXT, "pthread_getaffinity_np");
  86       if (orig_getaffinity_np == NULL)
  87         exit (0);
  88     }
  89   ret = orig_getaffinity_np (thread, cpusetsize, cpuset);
  90   if (ret != 0)
  91     return ret;
  92   if (contig_cpucount == 0)
  93     {
  94       max = 8 * cpusetsize;
  95       for (i = 0; i < max; i++)
  96         if (!CPU_ISSET_S (i, cpusetsize, cpuset))
  97           break;
  98       contig_cpucount = i;
  99       min_cpusetsize = cpusetsize;
 100     }
 101   return ret;
 102 }
 103 #endif
 104
 105 void
 106 print_affinity (struct place p)
 107 {
 108   static unsigned long size;
 109   if (size == 0)
 110     {
 111       if (min_cpusetsize)
 112         size = min_cpusetsize;
 113       else
 114         {
 115           size = sysconf (_SC_NPROCESSORS_CONF);
 116           size = CPU_ALLOC_SIZE (size);
 117           if (size < sizeof (cpu_set_t))
 118             size = sizeof (cpu_set_t);
 119         }
 120     }
 121   cpu_set_t *cpusetp = (cpu_set_t *) __builtin_alloca (size);
 122   if (pthread_getaffinity_np (pthread_self (), size, cpusetp) == 0)
 123     {
 124       unsigned long i, len, max = 8 * size;
 125       int notfirst = 0, unexpected = 1;
 126
 127       printf (" bound to {");
 128       for (i = 0, len = 0; i < max; i++)
 129         if (CPU_ISSET_S (i, size, cpusetp))
 130           {
 131             if (len == 0)
 132               {
 133                 if (notfirst)
 134                   {
 135                     unexpected = 1;
 136                     printf (",");
 137                   }
 138                 else if (i == (unsigned long) p.start)
 139                   unexpected = 0;
 140                 notfirst = 1;
 141                 printf ("%lu", i);
 142               }
 143             ++len;
 144           }
 145         else
 146           {
 147             if (len && len != (unsigned long) p.len)
 148               unexpected = 1;
 149             if (len > 1)
 150               printf (":%lu", len);
 151             len = 0;
 152           }
 153       if (len && len != (unsigned long) p.len)
 154         unexpected = 1;
 155       if (len > 1)
 156         printf (":%lu", len);
 157       printf ("}");
 158       if (p.start != -1 && unexpected)
 159         {
 160           printf (", expected {%d", p.start);
 161           if (p.len != 1)
 162             printf (":%d", p.len);
 163           printf ("} instead");
 164         }
 165       else if (p.start != -1)
 166         printf (", verified");
 167     }
 168 }
 169 #else
 170 void
 171 print_affinity (struct place p)
 172 {
 173   (void) p.start;
 174   (void) p.len;
 175 }
 176 #endif
 177
 178
 179 int
 180 main ()
 181 {
 182   char *env_proc_bind = getenv ("OMP_PROC_BIND");
 183   int test_false = env_proc_bind && strcmp (env_proc_bind, "false") == 0;
 184   int test_true = env_proc_bind && strcmp (env_proc_bind, "true") == 0;
 185   int test_spread_master_close
 186     = env_proc_bind && strcmp (env_proc_bind, "spread,master,close") == 0;
 187   char *env_places = getenv ("OMP_PLACES");
 188   int test_places = 0;
 189
 190 #ifdef DO_FORK
 191   if (env_places == NULL && contig_cpucount >= 8 && test_false
 192       && getenv ("GOMP_AFFINITY") == NULL)
 193     {
 194       int i, j, status;
 195       pid_t pid;
 196       for (j = 0; j < 2; j++)
 197         {
 198           if (setenv ("OMP_PROC_BIND", j ? "spread,master,close" : "true", 1)
 199               < 0)
 200             break;
 201           for (i = sizeof (places_array) / sizeof (places_array[0]) - 1;
 202                i; --i)
 203             {
 204               if (setenv ("OMP_PLACES", places_array[i].name, 1) < 0)
 205                 break;
 206               pid = fork ();
 207               if (pid == -1)
 208                 break;
 209               if (pid == 0)
 210                 {
 211                   execl ("/proc/self/exe", "affinity-1.exe", NULL);
 212                   _exit (1);
 213                 }
 214               if (waitpid (pid, &status, 0) < 0)
 215                 break;
 216               if (WIFSIGNALED (status) && WTERMSIG (status) == SIGABRT)
 217                 abort ();
 218               else if (!WIFEXITED (status) || WEXITSTATUS (status) != 0)
 219                 break;
 220             }
 221           if (i)
 222             break;
 223         }
 224     }
 225 #endif
 226
 227   int first = 1;
 228   if (env_proc_bind)
 229     {
 230       printf ("OMP_PROC_BIND='%s'", env_proc_bind);
 231       first = 0;
 232     }
 233   if (env_places)
 234     printf ("%sOMP_PLACES='%s'", first ? "" : " ", env_places);
 235   printf ("\n");
 236
 237   if (env_places && contig_cpucount >= 8
 238       && (test_true || test_spread_master_close))
 239     {
 240       for (test_places = sizeof (places_array) / sizeof (places_array[0]) - 1;
 241            test_places; --test_places)
 242         if (strcmp (env_places, places_array[test_places].name) == 0)
 243           break;
 244     }
 245
 246 #define verify(if_true, if_s_m_c) \
 247   if (test_false && omp_get_proc_bind () != omp_proc_bind_false)        \
 248     abort ();                                                           \
 249   if (test_true && omp_get_proc_bind () != if_true)                     \
 250     abort ();                                                           \
 251   if (test_spread_master_close && omp_get_proc_bind () != if_s_m_c)     \
 252     abort ();
 253
 254   verify (omp_proc_bind_true, omp_proc_bind_spread);
 255
 256   printf ("Initial thread");
 257   print_affinity (places_array[test_places].places[0]);
 258   printf ("\n");
 259   omp_set_nested (1);
 260   omp_set_dynamic (0);
 261
 262   #pragma omp parallel if (0)
 263   {
 264     verify (omp_proc_bind_true, omp_proc_bind_master);
 265     #pragma omp parallel if (0)
 266     {
 267       verify (omp_proc_bind_true, omp_proc_bind_close);
 268       #pragma omp parallel if (0)
 269       {
 270         verify (omp_proc_bind_true, omp_proc_bind_close);
 271       }
 272       #pragma omp parallel if (0) proc_bind (spread)
 273       {
 274         verify (omp_proc_bind_spread, omp_proc_bind_spread);
 275       }
 276     }
 277     #pragma omp parallel if (0) proc_bind (master)
 278     {
 279       verify (omp_proc_bind_master, omp_proc_bind_close);
 280       #pragma omp parallel if (0)
 281       {
 282         verify (omp_proc_bind_master, omp_proc_bind_close);
 283       }
 284       #pragma omp parallel if (0) proc_bind (spread)
 285       {
 286         verify (omp_proc_bind_spread, omp_proc_bind_spread);
 287       }
 288     }
 289   }
 290
 291   /* True/spread */
 292   #pragma omp parallel num_threads (4)
 293   {
 294     verify (omp_proc_bind_true, omp_proc_bind_master);
 295     #pragma omp critical
 296     {
 297       struct place p = places_array[0].places[0];
 298       int thr = omp_get_thread_num ();
 299       printf ("#1 thread %d", thr);
 300       if (omp_get_num_threads () == 4 && test_spread_master_close)
 301         switch (places_array[test_places].count)
 302           {
 303           case 8:
 304             /* T = 4, P = 8, each subpartition has 2 places.  */
 305           case 7:
 306             /* T = 4, P = 7, each subpartition has 2 places, but
 307                last partition, which has just one place.  */
 308             p = places_array[test_places].places[2 * thr];
 309             break;
 310           case 5:
 311             /* T = 4, P = 5, first subpartition has 2 places, the
 312                rest just one.  */
 313             p = places_array[test_places].places[thr ? 1 + thr : 0];
 314             break;
 315           case 3:
 316             /* T = 4, P = 3, unit sized subpartitions, first gets
 317                thr0 and thr3, second thr1, third thr2.  */
 318             p = places_array[test_places].places[thr == 3 ? 0 : thr];
 319             break;
 320           case 2:
 321             /* T = 4, P = 2, unit sized subpartitions, each with
 322                2 threads.  */
 323             p = places_array[test_places].places[thr / 2];
 324             break;
 325           }
 326       print_affinity (p);
 327       printf ("\n");
 328     }
 329     #pragma omp barrier
 330     if (omp_get_thread_num () == 3)
 331       {
 332         /* True/spread, true/master.  */
 333         #pragma omp parallel num_threads (3)
 334         {
 335           verify (omp_proc_bind_true, omp_proc_bind_close);
 336           #pragma omp critical
 337           {
 338             struct place p = places_array[0].places[0];
 339             int thr = omp_get_thread_num ();
 340             printf ("#1,#1 thread 3,%d", thr);
 341             if (omp_get_num_threads () == 3 && test_spread_master_close)
 342               /* Outer is spread, inner master, so just bind to the
 343                  place or the master thread, which is thr 3 above.  */
 344               switch (places_array[test_places].count)
 345                 {
 346                 case 8:
 347                 case 7:
 348                   p = places_array[test_places].places[6];
 349                   break;
 350                 case 5:
 351                   p = places_array[test_places].places[4];
 352                   break;
 353                 case 3:
 354                   p = places_array[test_places].places[0];
 355                   break;
 356                 case 2:
 357                   p = places_array[test_places].places[1];
 358                   break;
 359                 }
 360             print_affinity (p);
 361             printf ("\n");
 362           }
 363         }
 364         /* True/spread, spread.  */
 365         #pragma omp parallel num_threads (5) proc_bind (spread)
 366         {
 367           verify (omp_proc_bind_spread, omp_proc_bind_close);
 368           #pragma omp critical
 369           {
 370             struct place p = places_array[0].places[0];
 371             int thr = omp_get_thread_num ();
 372             printf ("#1,#2 thread 3,%d", thr);
 373             if (omp_get_num_threads () == 5 && test_spread_master_close)
 374               /* Outer is spread, inner spread.  */
 375               switch (places_array[test_places].count)
 376                 {
 377                 case 8:
 378                   /* T = 5, P = 2, unit sized subpartitions.  */
 379                   p = places_array[test_places].places[thr == 4 ? 6
 380                                                        : 6 + thr / 2];
 381                   break;
 382                 /* The rest are T = 5, P = 1.  */
 383                 case 7:
 384                   p = places_array[test_places].places[6];
 385                   break;
 386                 case 5:
 387                   p = places_array[test_places].places[4];
 388                   break;
 389                 case 3:
 390                   p = places_array[test_places].places[0];
 391                   break;
 392                 case 2:
 393                   p = places_array[test_places].places[1];
 394                   break;
 395                 }
 396             print_affinity (p);
 397             printf ("\n");
 398           }
 399           #pragma omp barrier
 400           if (omp_get_thread_num () == 3)
 401             {
 402               /* True/spread, spread, close.  */
 403               #pragma omp parallel num_threads (5) proc_bind (close)
 404               {
 405                 verify (omp_proc_bind_close, omp_proc_bind_close);
 406                 #pragma omp critical
 407                 {
 408                   struct place p = places_array[0].places[0];
 409                   int thr = omp_get_thread_num ();
 410                   printf ("#1,#2,#1 thread 3,3,%d", thr);
 411                   if (omp_get_num_threads () == 5 && test_spread_master_close)
 412                     /* Outer is spread, inner spread, innermost close.  */
 413                     switch (places_array[test_places].count)
 414                       {
 415                       /* All are T = 5, P = 1.  */
 416                       case 8:
 417                         p = places_array[test_places].places[7];
 418                         break;
 419                       case 7:
 420                         p = places_array[test_places].places[6];
 421                         break;
 422                       case 5:
 423                         p = places_array[test_places].places[4];
 424                         break;
 425                       case 3:
 426                         p = places_array[test_places].places[0];
 427                         break;
 428                       case 2:
 429                         p = places_array[test_places].places[1];
 430                         break;
 431                       }
 432                   print_affinity (p);
 433                   printf ("\n");
 434                 }
 435               }
 436             }
 437         }
 438         /* True/spread, master.  */
 439         #pragma omp parallel num_threads (4) proc_bind(master)
 440         {
 441           verify (omp_proc_bind_master, omp_proc_bind_close);
 442           #pragma omp critical
 443           {
 444             struct place p = places_array[0].places[0];
 445             int thr = omp_get_thread_num ();
 446             printf ("#1,#3 thread 3,%d", thr);
 447             if (omp_get_num_threads () == 4 && test_spread_master_close)
 448               /* Outer is spread, inner master, so just bind to the
 449                  place or the master thread, which is thr 3 above.  */
 450               switch (places_array[test_places].count)
 451                 {
 452                 case 8:
 453                 case 7:
 454                   p = places_array[test_places].places[6];
 455                   break;
 456                 case 5:
 457                   p = places_array[test_places].places[4];
 458                   break;
 459                 case 3:
 460                   p = places_array[test_places].places[0];
 461                   break;
 462                 case 2:
 463                   p = places_array[test_places].places[1];
 464                   break;
 465                 }
 466             print_affinity (p);
 467             printf ("\n");
 468           }
 469         }
 470         /* True/spread, close.  */
 471         #pragma omp parallel num_threads (6) proc_bind (close)
 472         {
 473           verify (omp_proc_bind_close, omp_proc_bind_close);
 474           #pragma omp critical
 475           {
 476             struct place p = places_array[0].places[0];
 477             int thr = omp_get_thread_num ();
 478             printf ("#1,#4 thread 3,%d", thr);
 479             if (omp_get_num_threads () == 6 && test_spread_master_close)
 480               /* Outer is spread, inner close.  */
 481               switch (places_array[test_places].count)
 482                 {
 483                 case 8:
 484                   /* T = 6, P = 2, unit sized subpartitions.  */
 485                   p = places_array[test_places].places[6 + thr / 3];
 486                   break;
 487                 /* The rest are T = 6, P = 1.  */
 488                 case 7:
 489                   p = places_array[test_places].places[6];
 490                   break;
 491                 case 5:
 492                   p = places_array[test_places].places[4];
 493                   break;
 494                 case 3:
 495                   p = places_array[test_places].places[0];
 496                   break;
 497                 case 2:
 498                   p = places_array[test_places].places[1];
 499                   break;
 500                 }
 501             print_affinity (p);
 502             printf ("\n");
 503           }
 504         }
 505       }
 506   }
 507
 508   /* Spread.  */
 509   #pragma omp parallel num_threads (5) proc_bind(spread)
 510   {
 511     verify (omp_proc_bind_spread, omp_proc_bind_master);
 512     #pragma omp critical
 513     {
 514       struct place p = places_array[0].places[0];
 515       int thr = omp_get_thread_num ();
 516       printf ("#2 thread %d", thr);
 517       if (omp_get_num_threads () == 5
 518           && (test_spread_master_close || test_true))
 519         switch (places_array[test_places].count)
 520           {
 521           case 8:
 522             /* T = 5, P = 8, first 3 subpartitions have 2 places, last
 523                2 one place.  */
 524             p = places_array[test_places].places[thr < 3 ? 2 * thr : 3 + thr];
 525             break;
 526           case 7:
 527             /* T = 5, P = 7, first 2 subpartitions have 2 places, last
 528                3 one place.  */
 529             p = places_array[test_places].places[thr < 2 ? 2 * thr : 2 + thr];
 530             break;
 531           case 5:
 532             /* T = 5, P = 5, unit sized subpartitions, each one with one
 533                thread.  */
 534             p = places_array[test_places].places[thr];
 535             break;
 536           case 3:
 537             /* T = 5, P = 3, unit sized subpartitions, first gets
 538                thr0 and thr3, second thr1 and thr4, third thr2.  */
 539             p = places_array[test_places].places[thr >= 3 ? thr - 3 : thr];
 540             break;
 541           case 2:
 542             /* T = 5, P = 2, unit sized subpartitions, first with
 543                thr{0,1,4} and second with thr{2,3}.  */
 544             p = places_array[test_places].places[thr == 4 ? 0 : thr / 2];
 545             break;
 546           }
 547       print_affinity (p);
 548       printf ("\n");
 549     }
 550     #pragma omp barrier
 551     if (omp_get_thread_num () == 3)
 552       {
 553         int pp = 0;
 554         switch (places_array[test_places].count)
 555           {
 556           case 8: pp = 6; break;
 557           case 7: pp = 5; break;
 558           case 5: pp = 3; break;
 559           case 2: pp = 1; break;
 560           }
 561         /* Spread, spread/master.  */
 562         #pragma omp parallel num_threads (3) firstprivate (pp)
 563         {
 564           verify (omp_proc_bind_spread, omp_proc_bind_close);
 565           #pragma omp critical
 566           {
 567             struct place p = places_array[0].places[0];
 568             int thr = omp_get_thread_num ();
 569             printf ("#2,#1 thread 3,%d", thr);
 570             if (test_spread_master_close || test_true)
 571               /* Outer is spread, inner spread resp. master, bit we have
 572                  just unit sized partitions.  */
 573               p = places_array[test_places].places[pp];
 574             print_affinity (p);
 575             printf ("\n");
 576           }
 577         }
 578         /* Spread, spread.  */
 579         #pragma omp parallel num_threads (5) proc_bind (spread) \
 580                              firstprivate (pp)
 581         {
 582           verify (omp_proc_bind_spread, omp_proc_bind_close);
 583           #pragma omp critical
 584           {
 585             struct place p = places_array[0].places[0];
 586             int thr = omp_get_thread_num ();
 587             printf ("#2,#2 thread 3,%d", thr);
 588             if (test_spread_master_close || test_true)
 589               /* Outer is spread, inner spread, bit we have
 590                  just unit sized partitions.  */
 591               p = places_array[test_places].places[pp];
 592             print_affinity (p);
 593             printf ("\n");
 594           }
 595         }
 596         /* Spread, master.  */
 597         #pragma omp parallel num_threads (4) proc_bind(master) \
 598                              firstprivate(pp)
 599         {
 600           verify (omp_proc_bind_master, omp_proc_bind_close);
 601           #pragma omp critical
 602           {
 603             struct place p = places_array[0].places[0];
 604             int thr = omp_get_thread_num ();
 605             printf ("#2,#3 thread 3,%d", thr);
 606             if (test_spread_master_close || test_true)
 607               /* Outer is spread, inner master, bit we have
 608                  just unit sized partitions.  */
 609               p = places_array[test_places].places[pp];
 610             print_affinity (p);
 611             printf ("\n");
 612           }
 613         }
 614         /* Spread, close.  */
 615         #pragma omp parallel num_threads (6) proc_bind (close) \
 616                              firstprivate (pp)
 617         {
 618           verify (omp_proc_bind_close, omp_proc_bind_close);
 619           #pragma omp critical
 620           {
 621             struct place p = places_array[0].places[0];
 622             int thr = omp_get_thread_num ();
 623             printf ("#2,#4 thread 3,%d", thr);
 624             if (test_spread_master_close || test_true)
 625               /* Outer is spread, inner close, bit we have
 626                  just unit sized partitions.  */
 627               p = places_array[test_places].places[pp];
 628             print_affinity (p);
 629             printf ("\n");
 630           }
 631         }
 632       }
 633   }
 634
 635   /* Master.  */
 636   #pragma omp parallel num_threads (3) proc_bind(master)
 637   {
 638     verify (omp_proc_bind_master, omp_proc_bind_master);
 639     #pragma omp critical
 640     {
 641       struct place p = places_array[0].places[0];
 642       int thr = omp_get_thread_num ();
 643       printf ("#3 thread %d", thr);
 644       if (test_spread_master_close || test_true)
 645         p = places_array[test_places].places[0];
 646       print_affinity (p);
 647       printf ("\n");
 648     }
 649     #pragma omp barrier
 650     if (omp_get_thread_num () == 2)
 651       {
 652         /* Master, master.  */
 653         #pragma omp parallel num_threads (4)
 654         {
 655           verify (omp_proc_bind_master, omp_proc_bind_close);
 656           #pragma omp critical
 657           {
 658             struct place p = places_array[0].places[0];
 659             int thr = omp_get_thread_num ();
 660             printf ("#3,#1 thread 2,%d", thr);
 661             if (test_spread_master_close || test_true)
 662               /* Outer is master, inner is master.  */
 663               p = places_array[test_places].places[0];
 664             print_affinity (p);
 665             printf ("\n");
 666           }
 667         }
 668         /* Master, spread.  */
 669         #pragma omp parallel num_threads (4) proc_bind (spread)
 670         {
 671           verify (omp_proc_bind_spread, omp_proc_bind_close);
 672           #pragma omp critical
 673           {
 674             struct place p = places_array[0].places[0];
 675             int thr = omp_get_thread_num ();
 676             printf ("#3,#2 thread 2,%d", thr);
 677             if (omp_get_num_threads () == 4
 678                 && (test_spread_master_close || test_true))
 679               /* Outer is master, inner is spread.  */
 680               switch (places_array[test_places].count)
 681                 {
 682                 case 8:
 683                   /* T = 4, P = 8, each subpartition has 2 places.  */
 684                 case 7:
 685                   /* T = 4, P = 7, each subpartition has 2 places, but
 686                      last partition, which has just one place.  */
 687                   p = places_array[test_places].places[2 * thr];
 688                   break;
 689                 case 5:
 690                   /* T = 4, P = 5, first subpartition has 2 places, the
 691                      rest just one.  */
 692                   p = places_array[test_places].places[thr ? 1 + thr : 0];
 693                   break;
 694                 case 3:
 695                   /* T = 4, P = 3, unit sized subpartitions, first gets
 696                      thr0 and thr3, second thr1, third thr2.  */
 697                   p = places_array[test_places].places[thr == 3 ? 0 : thr];
 698                   break;
 699                 case 2:
 700                   /* T = 4, P = 2, unit sized subpartitions, each with
 701                      2 threads.  */
 702                   p = places_array[test_places].places[thr / 2];
 703                   break;
 704                 }
 705             print_affinity (p);
 706             printf ("\n");
 707           }
 708           #pragma omp barrier
 709           if (omp_get_thread_num () == 0)
 710             {
 711               /* Master, spread, close.  */
 712               #pragma omp parallel num_threads (5) proc_bind (close)
 713               {
 714                 verify (omp_proc_bind_close, omp_proc_bind_close);
 715                 #pragma omp critical
 716                 {
 717                   struct place p = places_array[0].places[0];
 718                   int thr = omp_get_thread_num ();
 719                   printf ("#3,#2,#1 thread 2,0,%d", thr);
 720                   if (omp_get_num_threads () == 5
 721                       && (test_spread_master_close || test_true))
 722                     /* Outer is master, inner spread, innermost close.  */
 723                     switch (places_array[test_places].count)
 724                       {
 725                       /* First 3 are T = 5, P = 2.  */
 726                       case 8:
 727                       case 7:
 728                       case 5:
 729                         p = places_array[test_places].places[(thr & 2) / 2];
 730                         break;
 731                       /* All the rest are T = 5, P = 1.  */
 732                       case 3:
 733                       case 2:
 734                         p = places_array[test_places].places[0];
 735                         break;
 736                       }
 737                   print_affinity (p);
 738                   printf ("\n");
 739                 }
 740               }
 741             }
 742           #pragma omp barrier
 743           if (omp_get_thread_num () == 3)
 744             {
 745               /* Master, spread, close.  */
 746               #pragma omp parallel num_threads (5) proc_bind (close)
 747               {
 748                 verify (omp_proc_bind_close, omp_proc_bind_close);
 749                 #pragma omp critical
 750                 {
 751                   struct place p = places_array[0].places[0];
 752                   int thr = omp_get_thread_num ();
 753                   printf ("#3,#2,#2 thread 2,3,%d", thr);
 754                   if (omp_get_num_threads () == 5
 755                       && (test_spread_master_close || test_true))
 756                     /* Outer is master, inner spread, innermost close.  */
 757                     switch (places_array[test_places].count)
 758                       {
 759                       case 8:
 760                         /* T = 5, P = 2.  */
 761                         p = places_array[test_places].places[6
 762                                                              + (thr & 2) / 2];
 763                         break;
 764                       /* All the rest are T = 5, P = 1.  */
 765                       case 7:
 766                         p = places_array[test_places].places[6];
 767                         break;
 768                       case 5:
 769                         p = places_array[test_places].places[4];
 770                         break;
 771                       case 3:
 772                         p = places_array[test_places].places[0];
 773                         break;
 774                       case 2:
 775                         p = places_array[test_places].places[1];
 776                         break;
 777                       }
 778                   print_affinity (p);
 779                   printf ("\n");
 780                 }
 781               }
 782             }
 783         }
 784         /* Master, master.  */
 785         #pragma omp parallel num_threads (4) proc_bind(master)
 786         {
 787           verify (omp_proc_bind_master, omp_proc_bind_close);
 788           #pragma omp critical
 789           {
 790             struct place p = places_array[0].places[0];
 791             int thr = omp_get_thread_num ();
 792             printf ("#3,#3 thread 2,%d", thr);
 793             if (test_spread_master_close || test_true)
 794               /* Outer is master, inner master.  */
 795               p = places_array[test_places].places[0];
 796             print_affinity (p);
 797             printf ("\n");
 798           }
 799         }
 800         /* Master, close.  */
 801         #pragma omp parallel num_threads (6) proc_bind (close)
 802         {
 803           verify (omp_proc_bind_close, omp_proc_bind_close);
 804           #pragma omp critical
 805           {
 806             struct place p = places_array[0].places[0];
 807             int thr = omp_get_thread_num ();
 808             printf ("#3,#4 thread 2,%d", thr);
 809             if (omp_get_num_threads () == 6
 810                 && (test_spread_master_close || test_true))
 811               switch (places_array[test_places].count)
 812                 {
 813                 case 8:
 814                   /* T = 6, P = 8.  */
 815                 case 7:
 816                   /* T = 6, P = 7.  */
 817                   p = places_array[test_places].places[thr];
 818                   break;
 819                 case 5:
 820                   /* T = 6, P = 5.  thr{0,5} go into the first place.  */
 821                   p = places_array[test_places].places[thr == 5 ? 0 : thr];
 822                   break;
 823                 case 3:
 824                   /* T = 6, P = 3, two threads into each place.  */
 825                   p = places_array[test_places].places[thr / 2];
 826                   break;
 827                 case 2:
 828                   /* T = 6, P = 2, 3 threads into each place.  */
 829                   p = places_array[test_places].places[thr / 3];
 830                   break;
 831                 }
 832             print_affinity (p);
 833             printf ("\n");
 834           }
 835         }
 836       }
 837   }
 838
 839   #pragma omp parallel num_threads (5) proc_bind(close)
 840   {
 841     verify (omp_proc_bind_close, omp_proc_bind_master);
 842     #pragma omp critical
 843     {
 844       struct place p = places_array[0].places[0];
 845       int thr = omp_get_thread_num ();
 846       printf ("#4 thread %d", thr);
 847       if (omp_get_num_threads () == 5
 848           && (test_spread_master_close || test_true))
 849         switch (places_array[test_places].count)
 850           {
 851           case 8:
 852             /* T = 5, P = 8.  */
 853           case 7:
 854             /* T = 5, P = 7.  */
 855           case 5:
 856             /* T = 5, P = 5.  */
 857             p = places_array[test_places].places[thr];
 858             break;
 859           case 3:
 860             /* T = 5, P = 3, thr{0,3} in first place, thr{1,4} in second,
 861                thr2 in third.  */
 862             p = places_array[test_places].places[thr >= 3 ? thr - 3 : thr];
 863             break;
 864           case 2:
 865             /* T = 5, P = 2, thr{0,1,4} in first place, thr{2,3} in second.  */
 866             p = places_array[test_places].places[thr == 4 ? 0 : thr / 2];
 867             break;
 868           }
 869       print_affinity (p);
 870       printf ("\n");
 871     }
 872     #pragma omp barrier
 873     if (omp_get_thread_num () == 2)
 874       {
 875         int pp = 0;
 876         switch (places_array[test_places].count)
 877           {
 878           case 8:
 879           case 7:
 880           case 5:
 881           case 3:
 882             pp = 2;
 883             break;
 884           case 2:
 885             pp = 1;
 886             break;
 887           }
 888         /* Close, close/master.  */
 889         #pragma omp parallel num_threads (4) firstprivate (pp)
 890         {
 891           verify (omp_proc_bind_close, omp_proc_bind_close);
 892           #pragma omp critical
 893           {
 894             struct place p = places_array[0].places[0];
 895             int thr = omp_get_thread_num ();
 896             printf ("#4,#1 thread 2,%d", thr);
 897             if (test_spread_master_close)
 898               /* Outer is close, inner is master.  */
 899               p = places_array[test_places].places[pp];
 900             else if (omp_get_num_threads () == 4 && test_true)
 901               /* Outer is close, inner is close.  */
 902               switch (places_array[test_places].count)
 903                 {
 904                 case 8:
 905                   /* T = 4, P = 8.  */
 906                 case 7:
 907                   /* T = 4, P = 7.  */
 908                   p = places_array[test_places].places[2 + thr];
 909                   break;
 910                 case 5:
 911                   /* T = 4, P = 5.  There is wrap-around for thr3.  */
 912                   p = places_array[test_places].places[thr == 3 ? 0 : 2 + thr];
 913                   break;
 914                 case 3:
 915                   /* T = 4, P = 3, thr{0,3} go into p2, thr1 into p0, thr2
 916                      into p1.  */
 917                   p = places_array[test_places].places[(2 + thr) % 3];
 918                   break;
 919                 case 2:
 920                   /* T = 4, P = 2, 2 threads into each place.  */
 921                   p = places_array[test_places].places[1 - thr / 2];
 922                   break;
 923                 }
 924
 925             print_affinity (p);
 926             printf ("\n");
 927           }
 928         }
 929         /* Close, spread.  */
 930         #pragma omp parallel num_threads (4) proc_bind (spread)
 931         {
 932           verify (omp_proc_bind_spread, omp_proc_bind_close);
 933           #pragma omp critical
 934           {
 935             struct place p = places_array[0].places[0];
 936             int thr = omp_get_thread_num ();
 937             printf ("#4,#2 thread 2,%d", thr);
 938             if (omp_get_num_threads () == 4
 939                 && (test_spread_master_close || test_true))
 940               /* Outer is close, inner is spread.  */
 941               switch (places_array[test_places].count)
 942                 {
 943                 case 8:
 944                   /* T = 4, P = 8, each subpartition has 2 places.  */
 945                 case 7:
 946                   /* T = 4, P = 7, each subpartition has 2 places, but
 947                      last partition, which has just one place.  */
 948                   p = places_array[test_places].places[thr == 3 ? 0
 949                                                        : 2 + 2 * thr];
 950                   break;
 951                 case 5:
 952                   /* T = 4, P = 5, first subpartition has 2 places, the
 953                      rest just one.  */
 954                   p = places_array[test_places].places[thr == 3 ? 0
 955                                                        : 2 + thr];
 956                   break;
 957                 case 3:
 958                   /* T = 4, P = 3, unit sized subpartitions, third gets
 959                      thr0 and thr3, first thr1, second thr2.  */
 960                   p = places_array[test_places].places[thr == 0 ? 2 : thr - 1];
 961                   break;
 962                 case 2:
 963                   /* T = 4, P = 2, unit sized subpartitions, each with
 964                      2 threads.  */
 965                   p = places_array[test_places].places[1 - thr / 2];
 966                   break;
 967                 }
 968             print_affinity (p);
 969             printf ("\n");
 970           }
 971           #pragma omp barrier
 972           if (omp_get_thread_num () == 0)
 973             {
 974               /* Close, spread, close.  */
 975               #pragma omp parallel num_threads (5) proc_bind (close)
 976               {
 977                 verify (omp_proc_bind_close, omp_proc_bind_close);
 978                 #pragma omp critical
 979                 {
 980                   struct place p = places_array[0].places[0];
 981                   int thr = omp_get_thread_num ();
 982                   printf ("#4,#2,#1 thread 2,0,%d", thr);
 983                   if (omp_get_num_threads () == 5
 984                       && (test_spread_master_close || test_true))
 985                     /* Outer is close, inner spread, innermost close.  */
 986                     switch (places_array[test_places].count)
 987                       {
 988                       case 8:
 989                       case 7:
 990                         /* T = 5, P = 2.  */
 991                         p = places_array[test_places].places[2
 992                                                              + (thr & 2) / 2];
 993                         break;
 994                       /* All the rest are T = 5, P = 1.  */
 995                       case 5:
 996                       case 3:
 997                         p = places_array[test_places].places[2];
 998                         break;
 999                       case 2:
1000                         p = places_array[test_places].places[1];
1001                         break;
1002                       }
1003                   print_affinity (p);
1004                   printf ("\n");
1005                 }
1006               }
1007             }
1008           #pragma omp barrier
1009           if (omp_get_thread_num () == 2)
1010             {
1011               /* Close, spread, close.  */
1012               #pragma omp parallel num_threads (5) proc_bind (close)
1013               {
1014                 verify (omp_proc_bind_close, omp_proc_bind_close);
1015                 #pragma omp critical
1016                 {
1017                   struct place p = places_array[0].places[0];
1018                   int thr = omp_get_thread_num ();
1019                   printf ("#4,#2,#2 thread 2,2,%d", thr);
1020                   if (omp_get_num_threads () == 5
1021                       && (test_spread_master_close || test_true))
1022                     /* Outer is close, inner spread, innermost close.  */
1023                     switch (places_array[test_places].count)
1024                       {
1025                       case 8:
1026                         /* T = 5, P = 2.  */
1027                         p = places_array[test_places].places[6
1028                                                              + (thr & 2) / 2];
1029                         break;
1030                       /* All the rest are T = 5, P = 1.  */
1031                       case 7:
1032                         p = places_array[test_places].places[6];
1033                         break;
1034                       case 5:
1035                         p = places_array[test_places].places[4];
1036                         break;
1037                       case 3:
1038                         p = places_array[test_places].places[1];
1039                         break;
1040                       case 2:
1041                         p = places_array[test_places].places[0];
1042                         break;
1043                       }
1044                   print_affinity (p);
1045                   printf ("\n");
1046                 }
1047               }
1048             }
1049           #pragma omp barrier
1050           if (omp_get_thread_num () == 3)
1051             {
1052               /* Close, spread, close.  */
1053               #pragma omp parallel num_threads (5) proc_bind (close)
1054               {
1055                 verify (omp_proc_bind_close, omp_proc_bind_close);
1056                 #pragma omp critical
1057                 {
1058                   struct place p = places_array[0].places[0];
1059                   int thr = omp_get_thread_num ();
1060                   printf ("#4,#2,#3 thread 2,3,%d", thr);
1061                   if (omp_get_num_threads () == 5
1062                       && (test_spread_master_close || test_true))
1063                     /* Outer is close, inner spread, innermost close.  */
1064                     switch (places_array[test_places].count)
1065                       {
1066                       case 8:
1067                       case 7:
1068                       case 5:
1069                         /* T = 5, P = 2.  */
1070                         p = places_array[test_places].places[(thr & 2) / 2];
1071                         break;
1072                       /* All the rest are T = 5, P = 1.  */
1073                       case 3:
1074                         p = places_array[test_places].places[2];
1075                         break;
1076                       case 2:
1077                         p = places_array[test_places].places[0];
1078                         break;
1079                       }
1080                   print_affinity (p);
1081                   printf ("\n");
1082                 }
1083               }
1084             }
1085         }
1086         /* Close, master.  */
1087         #pragma omp parallel num_threads (4) proc_bind(master) \
1088                              firstprivate (pp)
1089         {
1090           verify (omp_proc_bind_master, omp_proc_bind_close);
1091           #pragma omp critical
1092           {
1093             struct place p = places_array[0].places[0];
1094             int thr = omp_get_thread_num ();
1095             printf ("#4,#3 thread 2,%d", thr);
1096             if (test_spread_master_close || test_true)
1097               /* Outer is close, inner master.  */
1098               p = places_array[test_places].places[pp];
1099             print_affinity (p);
1100             printf ("\n");
1101           }
1102         }
1103         /* Close, close.  */
1104         #pragma omp parallel num_threads (6) proc_bind (close)
1105         {
1106           verify (omp_proc_bind_close, omp_proc_bind_close);
1107           #pragma omp critical
1108           {
1109             struct place p = places_array[0].places[0];
1110             int thr = omp_get_thread_num ();
1111             printf ("#4,#4 thread 2,%d", thr);
1112             if (omp_get_num_threads () == 6
1113                 && (test_spread_master_close || test_true))
1114               switch (places_array[test_places].count)
1115                 {
1116                 case 8:
1117                   /* T = 6, P = 8.  */
1118                   p = places_array[test_places].places[2 + thr];
1119                   break;
1120                 case 7:
1121                   /* T = 6, P = 7.  */
1122                   p = places_array[test_places].places[thr == 5 ? 0 : 2 + thr];
1123                   break;
1124                 case 5:
1125                   /* T = 6, P = 5.  thr{0,5} go into the third place.  */
1126                   p = places_array[test_places].places[thr >= 3 ? thr - 3
1127                                                        : 2 + thr];
1128                   break;
1129                 case 3:
1130                   /* T = 6, P = 3, two threads into each place.  */
1131                   p = places_array[test_places].places[thr < 2 ? 2
1132                                                        : thr / 2 - 1];
1133                   break;
1134                 case 2:
1135                   /* T = 6, P = 2, 3 threads into each place.  */
1136                   p = places_array[test_places].places[1 - thr / 3];
1137                   break;
1138                 }
1139             print_affinity (p);
1140             printf ("\n");
1141           }
1142         }
1143       }
1144   }
1145
1146   return 0;
1147 }