t/helper/test-run-command.c

   1 /*
   2  * test-run-command.c: test run command API.
   3  *
   4  * (C) 2009 Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
   5  *
   6  * This code is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #define DISABLE_SIGN_COMPARE_WARNINGS
  12
  13 #include "test-tool.h"
  14 #include "run-command.h"
  15 #include "strvec.h"
  16 #include "strbuf.h"
  17 #include "parse-options.h"
  18 #include "string-list.h"
  19 #include "thread-utils.h"
  20 #include "wildmatch.h"
  21
  22 static int number_callbacks;
  23 static int parallel_next(struct child_process *cp,
  24                          struct strbuf *err,
  25                          void *cb,
  26                          void **task_cb UNUSED)
  27 {
  28         struct child_process *d = cb;
  29         if (number_callbacks >= 4)
  30                 return 0;
  31
  32         strvec_pushv(&cp->args, d->args.v);
  33         if (err)
  34                 strbuf_addstr(err, "preloaded output of a child\n");
  35         else
  36                 fprintf(stderr, "preloaded output of a child\n");
  37
  38         number_callbacks++;
  39         return 1;
  40 }
  41
  42 static int no_job(struct child_process *cp UNUSED,
  43                   struct strbuf *err,
  44                   void *cb UNUSED,
  45                   void **task_cb UNUSED)
  46 {
  47         if (err)
  48                 strbuf_addstr(err, "no further jobs available\n");
  49         else
  50                 fprintf(stderr, "no further jobs available\n");
  51         return 0;
  52 }
  53
  54 static int task_finished(int result UNUSED,
  55                          struct strbuf *err,
  56                          void *pp_cb UNUSED,
  57                          void *pp_task_cb UNUSED)
  58 {
  59         if (err)
  60                 strbuf_addstr(err, "asking for a quick stop\n");
  61         else
  62                 fprintf(stderr, "asking for a quick stop\n");
  63         return 1;
  64 }
  65
  66 struct testsuite {
  67         struct string_list tests, failed;
  68         int next;
  69         int quiet, immediate, verbose, verbose_log, trace, write_junit_xml;
  70         const char *shell_path;
  71 };
  72 #define TESTSUITE_INIT { \
  73         .tests = STRING_LIST_INIT_DUP, \
  74         .failed = STRING_LIST_INIT_DUP, \
  75 }
  76
  77 static int next_test(struct child_process *cp, struct strbuf *err, void *cb,
  78                      void **task_cb)
  79 {
  80         struct testsuite *suite = cb;
  81         const char *test;
  82         if (suite->next >= suite->tests.nr)
  83                 return 0;
  84
  85         test = suite->tests.items[suite->next++].string;
  86         if (suite->shell_path)
  87                 strvec_push(&cp->args, suite->shell_path);
  88         strvec_push(&cp->args, test);
  89         if (suite->quiet)
  90                 strvec_push(&cp->args, "--quiet");
  91         if (suite->immediate)
  92                 strvec_push(&cp->args, "-i");
  93         if (suite->verbose)
  94                 strvec_push(&cp->args, "-v");
  95         if (suite->verbose_log)
  96                 strvec_push(&cp->args, "-V");
  97         if (suite->trace)
  98                 strvec_push(&cp->args, "-x");
  99         if (suite->write_junit_xml)
 100                 strvec_push(&cp->args, "--write-junit-xml");
 101
 102         strbuf_addf(err, "Output of '%s':\n", test);
 103         *task_cb = (void *)test;
 104
 105         return 1;
 106 }
 107
 108 static int test_finished(int result, struct strbuf *err, void *cb,
 109                          void *task_cb)
 110 {
 111         struct testsuite *suite = cb;
 112         const char *name = (const char *)task_cb;
 113
 114         if (result)
 115                 string_list_append(&suite->failed, name);
 116
 117         strbuf_addf(err, "%s: '%s'\n", result ? "FAIL" : "SUCCESS", name);
 118
 119         return 0;
 120 }
 121
 122 static int test_failed(struct strbuf *out, void *cb, void *task_cb)
 123 {
 124         struct testsuite *suite = cb;
 125         const char *name = (const char *)task_cb;
 126
 127         string_list_append(&suite->failed, name);
 128         strbuf_addf(out, "FAILED TO START: '%s'\n", name);
 129
 130         return 0;
 131 }
 132
 133 static const char * const testsuite_usage[] = {
 134         "test-run-command testsuite [<options>] [<pattern>...]",
 135         NULL
 136 };
 137
 138 static int testsuite(int argc, const char **argv)
 139 {
 140         struct testsuite suite = TESTSUITE_INIT;
 141         int max_jobs = 1, i, ret = 0;
 142         DIR *dir;
 143         struct dirent *d;
 144         struct option options[] = {
 145                 OPT_BOOL('i', "immediate", &suite.immediate,
 146                          "stop at first failed test case(s)"),
 147                 OPT_INTEGER('j', "jobs", &max_jobs, "run <N> jobs in parallel"),
 148                 OPT_BOOL('q', "quiet", &suite.quiet, "be terse"),
 149                 OPT_BOOL('v', "verbose", &suite.verbose, "be verbose"),
 150                 OPT_BOOL('V', "verbose-log", &suite.verbose_log,
 151                          "be verbose, redirected to a file"),
 152                 OPT_BOOL('x', "trace", &suite.trace, "trace shell commands"),
 153                 OPT_BOOL(0, "write-junit-xml", &suite.write_junit_xml,
 154                          "write JUnit-style XML files"),
 155                 OPT_END()
 156         };
 157         struct run_process_parallel_opts opts = {
 158                 .get_next_task = next_test,
 159                 .start_failure = test_failed,
 160                 .task_finished = test_finished,
 161                 .data = &suite,
 162         };
 163         struct strbuf progpath = STRBUF_INIT;
 164         size_t path_prefix_len;
 165
 166         argc = parse_options(argc, argv, NULL, options,
 167                         testsuite_usage, PARSE_OPT_STOP_AT_NON_OPTION);
 168
 169         if (max_jobs <= 0)
 170                 max_jobs = online_cpus();
 171
 172         /*
 173          * If we run without a shell, execute the programs directly from CWD.
 174          */
 175         suite.shell_path = getenv("TEST_SHELL_PATH");
 176         if (!suite.shell_path)
 177                 strbuf_addstr(&progpath, "./");
 178         path_prefix_len = progpath.len;
 179
 180         dir = opendir(".");
 181         if (!dir)
 182                 die("Could not open the current directory");
 183         while ((d = readdir(dir))) {
 184                 const char *p = d->d_name;
 185
 186                 if (!strcmp(p, ".") || !strcmp(p, ".."))
 187                         continue;
 188
 189                 /* No pattern: match all */
 190                 if (!argc) {
 191                         strbuf_setlen(&progpath, path_prefix_len);
 192                         strbuf_addstr(&progpath, p);
 193                         string_list_append(&suite.tests, progpath.buf);
 194                         continue;
 195                 }
 196
 197                 for (i = 0; i < argc; i++)
 198                         if (!wildmatch(argv[i], p, 0)) {
 199                                 strbuf_setlen(&progpath, path_prefix_len);
 200                                 strbuf_addstr(&progpath, p);
 201                                 string_list_append(&suite.tests, progpath.buf);
 202                                 break;
 203                         }
 204         }
 205         closedir(dir);
 206
 207         if (!suite.tests.nr)
 208                 die("No tests match!");
 209         if (max_jobs > suite.tests.nr)
 210                 max_jobs = suite.tests.nr;
 211
 212         fprintf(stderr, "Running %"PRIuMAX" tests (%d at a time)\n",
 213                 (uintmax_t)suite.tests.nr, max_jobs);
 214
 215         opts.processes = max_jobs;
 216         run_processes_parallel(&opts);
 217
 218         if (suite.failed.nr > 0) {
 219                 ret = 1;
 220                 fprintf(stderr, "%"PRIuMAX" tests failed:\n\n",
 221                         (uintmax_t)suite.failed.nr);
 222                 for (i = 0; i < suite.failed.nr; i++)
 223                         fprintf(stderr, "\t%s\n", suite.failed.items[i].string);
 224         }
 225
 226         string_list_clear(&suite.tests, 0);
 227         string_list_clear(&suite.failed, 0);
 228         strbuf_release(&progpath);
 229
 230         return ret;
 231 }
 232
 233 static uint64_t my_random_next = 1234;
 234
 235 static uint64_t my_random(void)
 236 {
 237         uint64_t res = my_random_next;
 238         my_random_next = my_random_next * 1103515245 + 12345;
 239         return res;
 240 }
 241
 242 static int quote_stress_test(int argc, const char **argv)
 243 {
 244         /*
 245          * We are running a quote-stress test.
 246          * spawn a subprocess that runs quote-stress with a
 247          * special option that echoes back the arguments that
 248          * were passed in.
 249          */
 250         char special[] = ".?*\\^_\"'`{}()[]<>@~&+:;$%"; // \t\r\n\a";
 251         int i, j, k, trials = 100, skip = 0, msys2 = 0;
 252         struct strbuf out = STRBUF_INIT;
 253         struct strvec args = STRVEC_INIT;
 254         struct option options[] = {
 255                 OPT_INTEGER('n', "trials", &trials, "number of trials"),
 256                 OPT_INTEGER('s', "skip", &skip, "skip <n> trials"),
 257                 OPT_BOOL('m', "msys2", &msys2, "test quoting for MSYS2's sh"),
 258                 OPT_END()
 259         };
 260         const char * const usage[] = {
 261                 "test-tool run-command quote-stress-test <options>",
 262                 NULL
 263         };
 264
 265         argc = parse_options(argc, argv, NULL, options, usage, 0);
 266
 267         setenv("MSYS_NO_PATHCONV", "1", 0);
 268
 269         for (i = 0; i < trials; i++) {
 270                 struct child_process cp = CHILD_PROCESS_INIT;
 271                 size_t arg_count, arg_offset;
 272                 int ret = 0;
 273
 274                 strvec_clear(&args);
 275                 if (msys2)
 276                         strvec_pushl(&args, "sh", "-c",
 277                                      "printf %s\\\\0 \"$@\"", "skip", NULL);
 278                 else
 279                         strvec_pushl(&args, "test-tool", "run-command",
 280                                      "quote-echo", NULL);
 281                 arg_offset = args.nr;
 282
 283                 if (argc > 0) {
 284                         trials = 1;
 285                         arg_count = argc;
 286                         for (j = 0; j < arg_count; j++)
 287                                 strvec_push(&args, argv[j]);
 288                 } else {
 289                         arg_count = 1 + (my_random() % 5);
 290                         for (j = 0; j < arg_count; j++) {
 291                                 char buf[20];
 292                                 size_t min_len = 1;
 293                                 size_t arg_len = min_len +
 294                                         (my_random() % (ARRAY_SIZE(buf) - min_len));
 295
 296                                 for (k = 0; k < arg_len; k++)
 297                                         buf[k] = special[my_random() %
 298                                                 ARRAY_SIZE(special)];
 299                                 buf[arg_len] = '\0';
 300
 301                                 strvec_push(&args, buf);
 302                         }
 303                 }
 304
 305                 if (i < skip)
 306                         continue;
 307
 308                 strvec_pushv(&cp.args, args.v);
 309                 strbuf_reset(&out);
 310                 if (pipe_command(&cp, NULL, 0, &out, 0, NULL, 0) < 0)
 311                         return error("Failed to spawn child process");
 312
 313                 for (j = 0, k = 0; j < arg_count; j++) {
 314                         const char *arg = args.v[j + arg_offset];
 315
 316                         if (strcmp(arg, out.buf + k))
 317                                 ret = error("incorrectly quoted arg: '%s', "
 318                                             "echoed back as '%s'",
 319                                              arg, out.buf + k);
 320                         k += strlen(out.buf + k) + 1;
 321                 }
 322
 323                 if (k != out.len)
 324                         ret = error("got %d bytes, but consumed only %d",
 325                                      (int)out.len, (int)k);
 326
 327                 if (ret) {
 328                         fprintf(stderr, "Trial #%d failed. Arguments:\n", i);
 329                         for (j = 0; j < arg_count; j++)
 330                                 fprintf(stderr, "arg #%d: '%s'\n",
 331                                         (int)j, args.v[j + arg_offset]);
 332
 333                         strbuf_release(&out);
 334                         strvec_clear(&args);
 335
 336                         return ret;
 337                 }
 338
 339                 if (i && (i % 100) == 0)
 340                         fprintf(stderr, "Trials completed: %d\n", (int)i);
 341         }
 342
 343         strbuf_release(&out);
 344         strvec_clear(&args);
 345
 346         return 0;
 347 }
 348
 349 static int quote_echo(int argc, const char **argv)
 350 {
 351         while (argc > 1) {
 352                 fwrite(argv[1], strlen(argv[1]), 1, stdout);
 353                 fputc('\0', stdout);
 354                 argv++;
 355                 argc--;
 356         }
 357
 358         return 0;
 359 }
 360
 361 static int inherit_handle(const char *argv0)
 362 {
 363         struct child_process cp = CHILD_PROCESS_INIT;
 364         char path[PATH_MAX];
 365         int tmp;
 366
 367         /* First, open an inheritable handle */
 368         xsnprintf(path, sizeof(path), "out-XXXXXX");
 369         tmp = xmkstemp(path);
 370
 371         strvec_pushl(&cp.args,
 372                      "test-tool", argv0, "inherited-handle-child", NULL);
 373         cp.in = -1;
 374         cp.no_stdout = cp.no_stderr = 1;
 375         if (start_command(&cp) < 0)
 376                 die("Could not start child process");
 377
 378         /* Then close it, and try to delete it. */
 379         close(tmp);
 380         if (unlink(path))
 381                 die("Could not delete '%s'", path);
 382
 383         if (close(cp.in) < 0 || finish_command(&cp) < 0)
 384                 die("Child did not finish");
 385
 386         return 0;
 387 }
 388
 389 static int inherit_handle_child(void)
 390 {
 391         struct strbuf buf = STRBUF_INIT;
 392
 393         if (strbuf_read(&buf, 0, 0) < 0)
 394                 die("Could not read stdin");
 395         printf("Received %s\n", buf.buf);
 396         strbuf_release(&buf);
 397
 398         return 0;
 399 }
 400
 401 int cmd__run_command(int argc, const char **argv)
 402 {
 403         struct child_process proc = CHILD_PROCESS_INIT;
 404         int jobs;
 405         int ret;
 406         struct run_process_parallel_opts opts = {
 407                 .data = &proc,
 408         };
 409
 410         if (argc > 1 && !strcmp(argv[1], "testsuite"))
 411                 return testsuite(argc - 1, argv + 1);
 412         if (!strcmp(argv[1], "inherited-handle"))
 413                 return inherit_handle(argv[0]);
 414         if (!strcmp(argv[1], "inherited-handle-child"))
 415                 return inherit_handle_child();
 416
 417         if (argc >= 2 && !strcmp(argv[1], "quote-stress-test"))
 418                 return !!quote_stress_test(argc - 1, argv + 1);
 419
 420         if (argc >= 2 && !strcmp(argv[1], "quote-echo"))
 421                 return !!quote_echo(argc - 1, argv + 1);
 422
 423         if (argc < 3)
 424                 return 1;
 425         while (!strcmp(argv[1], "env")) {
 426                 if (!argv[2])
 427                         die("env specifier without a value");
 428                 strvec_push(&proc.env, argv[2]);
 429                 argv += 2;
 430                 argc -= 2;
 431         }
 432         if (argc < 3) {
 433                 ret = 1;
 434                 goto cleanup;
 435         }
 436         strvec_pushv(&proc.args, (const char **)argv + 2);
 437
 438         if (!strcmp(argv[1], "start-command-ENOENT")) {
 439                 if (start_command(&proc) < 0 && errno == ENOENT) {
 440                         ret = 0;
 441                         goto cleanup;
 442                 }
 443                 fprintf(stderr, "FAIL %s\n", argv[1]);
 444                 return 1;
 445         }
 446         if (!strcmp(argv[1], "run-command")) {
 447                 ret = run_command(&proc);
 448                 goto cleanup;
 449         }
 450
 451         if (!strcmp(argv[1], "--ungroup")) {
 452                 argv += 1;
 453                 argc -= 1;
 454                 opts.ungroup = 1;
 455         }
 456
 457         jobs = atoi(argv[2]);
 458         strvec_clear(&proc.args);
 459         strvec_pushv(&proc.args, (const char **)argv + 3);
 460
 461         if (!strcmp(argv[1], "run-command-parallel")) {
 462                 opts.get_next_task = parallel_next;
 463         } else if (!strcmp(argv[1], "run-command-abort")) {
 464                 opts.get_next_task = parallel_next;
 465                 opts.task_finished = task_finished;
 466         } else if (!strcmp(argv[1], "run-command-no-jobs")) {
 467                 opts.get_next_task = no_job;
 468                 opts.task_finished = task_finished;
 469         } else {
 470                 ret = 1;
 471                 fprintf(stderr, "check usage\n");
 472                 goto cleanup;
 473         }
 474         opts.processes = jobs;
 475         run_processes_parallel(&opts);
 476         ret = 0;
 477 cleanup:
 478         child_process_clear(&proc);
 479         return ret;
 480 }