Fix a Coverity-reported overrun.

A Coverity scan of open-vm-tools reports a buffer overrun in
Escape_Unescape.  The problem is that Escape_Unescape uses
sizeof('\0') to specify the size of a buffer that consists of
a single character in the variable nulByte (previously named
nullbyte).  However, character literals in C are ints, so
sizeof('\0') is equivalent to sizeof int rather than sizeof char.
Use "sizeof nulByte" instead.

diff --git a/open-vm-tools/lib/misc/escape.c b/open-vm-tools/lib/misc/escape.c
index f4b66337ab7f962b768f43949fdcc590c9952da1..79002deeb9bc3d96b27980a9cec1c99470492c3c 100644 (file)
--- a/open-vm-tools/lib/misc/escape.c
+++ b/open-vm-tools/lib/misc/escape.c
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2017,2020 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -718,7 +718,7 @@ Escape_Unescape(char escByte,       // IN
 {
    DynBuf result;
    Bool escaped = FALSE;
-   char nullbyte = '\0';
+   char nulByte = '\0';
    int i;
 
    ASSERT(bufIn);
@@ -734,7 +734,7 @@ Escape_Unescape(char escByte,       // IN
       }
    }
 
-   DynBuf_Append(&result, &nullbyte, sizeof('\0'));
+   DynBuf_Append(&result, &nulByte, sizeof nulByte);
 
    return DynBuf_Get(&result);
 }