tools: Fix Coverity errors in resolution plugin.

Fix multiplication overflow and fgetc usage to avoid Coverity errors.

diff --git a/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c b/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c
index 668924850ea62f49c6fb03819af939eba5cf07a5..574e004ac9aefc3a23fb7d30978409251518f07f 100644 (file)
--- a/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c
+++ b/open-vm-tools/services/plugins/resolutionSet/resolutionCommon.c
@@ -400,7 +400,7 @@ resolutionXorgDriverVersion(int numPaths,               // IN: Number of strings
 {
    FILE *driver = NULL;
    const char *curMatch;
-   char curFileChar;
+   int curFileChar;
    int i;
 
    g_debug("%s: Scanning for VMWare Xorg drivers.\n", __func__);
@@ -423,7 +423,7 @@ resolutionXorgDriverVersion(int numPaths,               // IN: Number of strings
         goto outNotFound;
 
       curFileChar = fgetc(driver);
-      if (curFileChar == *curMatch) {
+      if (curFileChar != EOF && curFileChar == *curMatch) {
         curMatch++;
         continue;
       } else if (curMatch != versionString) {

diff --git a/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c b/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
index c5591139d07fad22c35ac2b65dcbf7fa7d5103b8..63b5812c9695daeca70d119811f5689383500d40 100644 (file)
--- a/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
+++ b/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
@@ -533,7 +533,7 @@ SelectResolution(uint32 width,
     * and height.
     */
    for (i = 0; i < xrrNumSizes; i++) {
-      potentialSize = xrrSizes[i].width * xrrSizes[i].height;
+      potentialSize = (uint64)xrrSizes[i].width * xrrSizes[i].height;
       if (xrrSizes[i].width <= width && xrrSizes[i].height <= height &&
           potentialSize > bestFitSize ) {
          bestFitSize = potentialSize;