BEE Backport bacula/src/stored/stored.c

This commit is the result of the squash of the following main commits:

Author: Eric Bollengier <eric@baculasystems.com>
Date:   Fri Apr 24 20:16:22 2020 +0200

    Reliably log user activity in a way that satisfies auditors

    Messages {
      name = Standard
      append = /tmp/bacula.log = all

      # To send the events into a log and in the catalog
      append = /tmp/audit.log = events, !events.bweb
      catalog = all, events
    }

    Users can create custom events with a console command:

    .events

    They can list events in the catalog with

    * list events
    +---------------------+------------+-----------+--------------------------------+
    | time                | type       | source    | events                         |
    +---------------------+------------+-----------+--------------------------------+
    | 2020-04-24 17:04:07 | daemon     | *Daemon*  | Director startup               |
    | 2020-04-24 17:04:12 | connection | *Console* | Connection from 127.0.0.1:8101 |
    | 2020-04-24 17:04:20 | command    | *Console* | purge jobid=1                  |
    +---------------------+------------+-----------+--------------------------------+

    The event format in the log is:

    24-Apr 17:04 Events: code=DC0001 from=zog8-dir ref=0x1fa5 type=daemon source=*Daemon* text=Director startup

    24-Apr 17:04 Events: code=DC0002 from=zog8-dir ref=0x7fb58000c4b8 type=connection source=*Console* text=Connection from 127.0.0.1:8101

    The reference (ref) is used to identify a console session, all activity
    from the same console will have the same reference.

    The source is the name of the restricted console, or *Console* if this
    is the default console.

    We have the following events:
     - cancel a job
     - delete volume
     - delete job
     - purge job
     - delete pool
     - delete client
     - daemon startup
     - daemon shutdown
     - console connection
     - console disconnection

    To add a new events:
    ua->send_events(type, format, arguments);

Author: Eric Bollengier <eric@baculasystems.com>
Date:   Wed Mar 4 15:41:15 2020 +0100

    Fix compilation of community version + move dedup configuration functions to external file

Author: Eric Bollengier <eric@baculasystems.com>
Date:   Tue Jan 14 15:09:05 2020 +0100

    Add specific checks when TLS PSK is not available in SSL library

Author: Eric Bollengier <eric@baculasystems.com>
Date:   Wed Oct 30 11:58:32 2019 +0100

    Disable device debug hook only for windows

Author: Alain Spineux <alain@baculasystems.com>
Date:   Tue Sep 24 10:17:51 2019 +0200

    dedup: move all dedup code into the plugin

    - remove global variables: dedupengine and bucker_manager
    - update protocol SD<->DIR
    - extend dedup_dev and BufferedMsgSD to suuport all function
    - create "dummy" functions in dev.h
    Warning :
    - include a dedupengine use counter
    - .status storage=XXX dedupengine display all the dedupengine on the SD
     instead of the one(s) related to the "device"

Author: Eric Bollengier <eric@baculasystems.com>
Date:   Wed Apr 10 11:58:00 2019 +0200

    Fix windows compilation

Author: Alain Spineux <alain@baculasystems.com>
Date:   Mon Apr 8 14:56:33 2019 +0200

    PSK: Add new "TLS PSK Enable" directive to all resources

    - add the field to the resources
    - create default "psk_ctx" CONTEXT for each of them at startup

Author: Norbert Bizet <norbert.bizet@baculasystems.com>
Date:   Fri Mar 22 10:50:06 2019 +0100

    cloud: Handle was driver has a generic drv

Author: Eric Bollengier <eric@baculasystems.com>
Date:   Thu Nov 1 17:33:35 2018 +0100

    cloud: Accept empty Hostname, SecretKey and AccessKey for Generic drivers

diff --git a/bacula/src/stored/stored.c b/bacula/src/stored/stored.c
index 8cd107b5c2f7a327eccecbc9227ce23a9503664a..fa4982183d80c1db47ce2a00ae7127c67bd29a96 100644 (file)
--- a/bacula/src/stored/stored.c
+++ b/bacula/src/stored/stored.c
@@ -91,7 +91,7 @@ static void usage()
       "     -u <user>         userid to <user>\n"
       "     -v                verbose user messages\n"
       "     -?                print this message.\n"
-      "\n"), 2000, "", VERSION, BDATE);
+      "\n"), 2000, BDEMO, VERSION, BDATE);
    exit(1);
 }
 
@@ -130,6 +130,8 @@ int main (int argc, char *argv[])
    char *uid = NULL;
    char *gid = NULL;
 
+   device_default_open_mode = omd_write;
+
    start_heap = sbrk(0);
    setlocale(LC_ALL, "");
    bindtextdomain("bacula", LOCALEDIR);
@@ -143,11 +145,11 @@ int main (int argc, char *argv[])
 
    /* Sanity checks */
    if (TAPE_BSIZE % B_DEV_BSIZE != 0 || TAPE_BSIZE / B_DEV_BSIZE == 0) {
-      Jmsg2(NULL, M_ABORT, 0, _("Tape block size (%d) not multiple of system size (%d)\n"),
+      Emsg2(M_ABORT, 0, _("Tape block size (%d) not multiple of system size (%d)\n"),
          TAPE_BSIZE, B_DEV_BSIZE);
    }
    if (TAPE_BSIZE != (1 << (ffs(TAPE_BSIZE)-1))) {
-      Jmsg1(NULL, M_ABORT, 0, _("Tape block size (%d) is not a power of 2\n"), TAPE_BSIZE);
+      Emsg1(M_ABORT, 0, _("Tape block size (%d) is not a power of 2\n"), TAPE_BSIZE);
    }
 
    while ((ch = getopt(argc, argv, "c:d:fg:mpPstu:v?Ti")) != -1) {
@@ -268,6 +270,10 @@ int main (int argc, char *argv[])
       Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile);
    }
 
+   if (crypto_check_fips(me->require_fips) < 0) {
+      Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Unable to set FIPS mode\n"));
+   }
+
    init_reservations_lock();
 
    if (test_config) {
@@ -311,7 +317,7 @@ int main (int argc, char *argv[])
    create_volume_lists();             /* do before device_init */
    if (pthread_create(&thid, NULL, device_initialization, NULL) != 0) {
       berrno be;
-      Jmsg1(NULL, M_ABORT, 0, _("Unable to create thread. ERR=%s\n"), be.bstrerror());
+      Emsg1(M_ABORT, 0, _("Unable to create thread. ERR=%s\n"), be.bstrerror());
    }
 
    start_watchdog();                  /* start watchdog thread */
@@ -320,6 +326,11 @@ int main (int argc, char *argv[])
 
    start_collector_threads();    /* start collector thread for every Collector resource */
 
+   /* Keep track of important events */
+   events_send_msg(NULL, "SD0001", EVENTS_TYPE_DAEMON, "*Daemon*",
+                   (intptr_t)get_first_port_host_order(me->sdaddrs),
+                   "Storage startup");
+
    /* Single server used for Director and File daemon */
    server_tid = pthread_self();
    server_tid_valid = true;
@@ -392,7 +403,9 @@ static int check_resources()
       /* tls_require implies tls_enable */
       if (store->tls_require) {
          if (have_tls) {
-            store->tls_enable = true;
+            if (store->tls_certfile || store->tls_keyfile) {
+               store->tls_enable = true;
+            }
          } else {
             Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n"));
             OK = false;
@@ -424,7 +437,7 @@ static int check_resources()
       }
 
       /* If everything is well, attempt to initialize our per-resource TLS context */
-      if (OK && (tls_needed || store->tls_require)) {
+      if (OK && tls_needed) {
          /* Initialize TLS context:
           * Args: CA certfile, CA certdir, Certfile, Keyfile,
           * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */
@@ -439,12 +452,23 @@ static int check_resources()
             OK = false;
          }
       }
+      store->psk_ctx = new_psk_context(NULL); /* shared key generated by DIR */
+      /* In this case, we have TLS Require=Yes and TLS not setup and no PSK */
+      if (OK && tls_needed == false && store->tls_require) {
+         if (!store->psk_ctx) {
+            Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS PSK context for Storage \"%s\" in %s.\n"),
+                 store->hdr.name, configfile);
+            OK = false;
+         }
+      }
    }
 
    foreach_res(director, R_DIRECTOR) {
       /* tls_require implies tls_enable */
       if (director->tls_require) {
-         director->tls_enable = true;
+         if (director->tls_certfile || director->tls_keyfile) {
+            director->tls_enable = true;
+         }
       }
 
       tls_needed = director->tls_enable || director->tls_authenticate;
@@ -471,7 +495,7 @@ static int check_resources()
       }
 
       /* If everything is well, attempt to initialize our per-resource TLS context */
-      if (OK && (tls_needed || director->tls_require)) {
+      if (OK && tls_needed) {
          /* Initialize TLS context:
           * Args: CA certfile, CA certdir, Certfile, Keyfile,
           * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */
@@ -486,6 +510,15 @@ static int check_resources()
             OK = false;
          }
       }
+      director->psk_ctx = new_psk_context(director->password);
+      /* In this case, we have TLS Require=Yes and TLS not setup and no PSK */
+      if (OK && (tls_needed == false && director->tls_require)) {
+         if (!director->psk_ctx) {
+            Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS PSK context for Director \"%s\" in %s.\n"),
+                 director->hdr.name, configfile);
+            OK = false;
+         }
+      }
    }
 
    foreach_res(changer, R_AUTOCHANGER) {
@@ -494,6 +527,52 @@ static int check_resources()
       }
    }
 
+   CLOUD *cloud;
+   /* TODO: Can use a table */
+   foreach_res(cloud, R_CLOUD) {
+      if (cloud->driver_type == C_S3_DRIVER  ||
+          cloud->driver_type == C_FILE_DRIVER)
+      {
+         if (cloud->host_name == NULL) {
+            Jmsg(NULL, M_FATAL, 0,
+                 _("Failed to initialize Cloud. Hostname not defined for Cloud \"%s\"\n"),
+                 cloud->hdr.name);
+            OK = false;
+         }
+      }
+      if (cloud->driver_type == C_WAS_DRIVER ||
+          cloud->driver_type == C_S3_DRIVER)
+      {
+         if (cloud->access_key == NULL) {
+            Jmsg(NULL, M_FATAL, 0,
+                 _("Failed to initialize Cloud. AccessKey not set for Cloud \"%s\"\n"),
+                 cloud->hdr.name);
+            OK = false;
+         }
+         if (cloud->secret_key == NULL) {
+            Jmsg(NULL, M_FATAL, 0,
+                 _("Failed to initialize Cloud. SecretKey not set for Cloud \"%s\"\n"),
+                 cloud->hdr.name);
+            OK = false;
+         }
+      }
+   }
+
+#ifdef SD_DEDUP_SUPPORT
+   DEDUPRES *dedup;
+   foreach_res(dedup, R_DEDUP) {
+      if (dedup->driver_type == D_LEGACY_DRIVER)
+      {
+         if (dedup->dedup_dir == NULL) {
+            Jmsg(NULL, M_FATAL, 0,
+                 _("Failed to initialize Dedup. DedupDirectory not defined for Dedup \"%s\"\n"),
+                 dedup->hdr.name);
+            OK = false;
+         }
+      }
+   }
+#endif
+
    if (OK) {
       OK = init_autochangers();
    }
@@ -561,7 +640,7 @@ static void cleanup_old_files()
       }
       /* Exclude any name with ., .., not my_name or containing a space */
       if (strcmp(dname.c_str(), ".") == 0 || strcmp(dname.c_str(), "..") == 0 ||
-         strncmp(dname.c_str(), my_name, my_name_len) != 0) {
+          strncmp(dname.c_str(), my_name, my_name_len) != 0) {
          Dmsg1(500, "Skipped: %s\n", dname.c_str());
          continue;
       }
@@ -583,6 +662,16 @@ get_out2:
    free_pool_memory(basename);
 }
 
+void dbg_print_devices(FILE *fp)
+{
+   DEVRES *device;
+   foreach_res(device, R_DEVICE) {
+      if (device->dev) {
+         device->dev->dbg_print(fp);
+      }
+   }
+}
+
 /*
  * Here we attempt to init and open each device. This is done
  *  once at startup in a separate thread.
@@ -620,6 +709,11 @@ void *device_initialization(void *arg)
       jcr->dcr = dcr = new_dcr(jcr, NULL, dev);
       generate_plugin_event(jcr, bsdEventDeviceInit, dcr);
 
+      /* Keep track of important events */
+      events_send_msg(jcr, "SD0002", EVENTS_TYPE_DAEMON, "*Daemon*",
+                      get_first_port_host_order(me->sdaddrs),
+                      "Device initialization %s", device->hdr.name);
+
       if (device->control_name && stat(device->control_name, &statp) < 0) {
          berrno be;
          Jmsg2(jcr, M_ERROR_TERM, 0, _("Unable to stat ControlDevice %s: ERR=%s\n"),
@@ -636,14 +730,9 @@ void *device_initialization(void *arg)
                dev->print_name());
       }
 
-      if (device->min_block_size > device->max_block_size) {
-         Jmsg1(jcr, M_ERROR_TERM, 0, _("MaximumBlockSize must be greater or equal than MinimumBlockSize for Device \"%s\"\n"),
-               dev->print_name());
-      }
-
 #ifdef HAVE_WIN32
       if (device->cap_bits & CAP_SYNCONCLOSE) {
-         device->cap_bits &= ~CAP_SYNCONCLOSE; /* Clear bit not used on Windows */
+         device->cap_bits &= ~CAP_SYNCONCLOSE; /* Not available on windows */
       }
 #endif
       /*
@@ -690,6 +779,9 @@ void *device_initialization(void *arg)
    }
 
    UnlockRes();
+#ifndef HAVE_WIN32
+   dbg_add_hook(dbg_print_devices);
+#endif
 
 #ifdef xxx
    if (jcr->dcr) {
@@ -710,6 +802,7 @@ void terminate_stored(int sig)
 {
    static bool in_here = false;
    DEVRES *device;
+   DEDUPRES *dedup;
    JCR *jcr;
 
    if (in_here) {                     /* prevent loops */
@@ -768,6 +861,11 @@ void terminate_stored(int sig)
          delete_pid_file(me->pid_directory,
                          "bacula-sd", get_first_port_host_order(me->sdaddrs));
       }
+
+      /* Keep track of important events */
+      events_send_msg(NULL, "SD0003", EVENTS_TYPE_DAEMON, "*Daemon*",
+                      get_first_port_host_order(me->sdaddrs),
+                      "Storage shutdown");
    }
 
    Dmsg1(200, "In terminate_stored() sig=%d\n", sig);
@@ -787,6 +885,15 @@ void terminate_stored(int sig)
          Dmsg2(10, "No dev structure %s %s\n", device->hdr.name, device->device_name);
       }
    }
+
+   foreach_res(dedup, R_DEDUP) {
+      if (dedup->dedupengine) {
+         dedup->dedupengine = NULL; // Should have been closed by the last device
+      } else {
+         Dmsg1(10, "No dev Dedupengine structure %s\n", dedup->hdr.name);
+      }
+   }
+
    if (server_tid_valid) {
       server_tid_valid = false;
       bnet_stop_thread_server(server_tid);