From: John Wolfe <jwolfe@vmware.com>
Date: Wed, 23 Dec 2020 15:33:50 +0000 (-0800)
Subject: tools: Fix Coverity errors in resolution plugin.
X-Git-Tag: stable-11.2.5~6
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=686fcb305ffbd3bcbba2461da9db8b41e2217f3c;p=thirdparty%2Fopen-vm-tools.git

tools: Fix Coverity errors in resolution plugin.

Fix multiplication overflow and fgetc usage to avoid Coverity errors.
---

diff --git a/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c b/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c
index 668924850..574e004ac 100644
--- a/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c
+++ b/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c
@@ -400,7 +400,7 @@ resolutionXorgDriverVersion(int numPaths,               // IN: Number of strings
 {
    FILE *driver = NULL;
    const char *curMatch;
-   char curFileChar;
+   int curFileChar;
    int i;
 
    g_debug("%s: Scanning for VMWare Xorg drivers.\n", __func__);
@@ -423,7 +423,7 @@ resolutionXorgDriverVersion(int numPaths,               // IN: Number of strings
 	 goto outNotFound;
 
       curFileChar = fgetc(driver);
-      if (curFileChar == *curMatch) {
+      if (curFileChar != EOF && curFileChar == *curMatch) {
 	 curMatch++;
 	 continue;
       } else if (curMatch != versionString) {
diff --git a/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c b/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
index c5591139d..63b5812c9 100644
--- a/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
+++ b/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
@@ -533,7 +533,7 @@ SelectResolution(uint32 width,
     * and height.
     */
    for (i = 0; i < xrrNumSizes; i++) {
-      potentialSize = xrrSizes[i].width * xrrSizes[i].height;
+      potentialSize = (uint64)xrrSizes[i].width * xrrSizes[i].height;
       if (xrrSizes[i].width <= width && xrrSizes[i].height <= height &&
           potentialSize > bestFitSize ) {
          bestFitSize = potentialSize;