open-src/lib/libXvMC/CVE-2013-1990.patch
author Alan Coopersmith <Alan.Coopersmith@Oracle.COM>
Wed, 15 May 2013 13:44:02 -0700
changeset 1345 d5dacbb8de2b
child 1349 f430f604f391
permissions -rw-r--r--
16673783 problem in X11/LIBRARIES 16674478 problem in X11/LIBRARIES

From cf1a1dc1b9ca34a29d0471da9389f8eae70ddbd9 Mon Sep 17 00:00:00 2001
From: Alan Coopersmith <[email protected]>
Date: Sat, 13 Apr 2013 00:47:57 -0700
Subject: [PATCH:libXvMC 1/5] Use _XEatDataWords to avoid overflow of
 rep.length shifting

rep.length is a CARD32, so rep.length << 2 could overflow in 32-bit builds

Signed-off-by: Alan Coopersmith <[email protected]>
---
 configure.ac |    6 ++++++
 src/XvMC.c   |   24 ++++++++++++++++++------
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/configure.ac b/configure.ac
index b44f80d..f9d59a1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -42,6 +42,12 @@ XORG_CHECK_MALLOC_ZERO
 # Obtain compiler/linker options for depedencies
 PKG_CHECK_MODULES(XVMC, x11 xext xv xextproto videoproto)
 
+# Check for _XEatDataWords function that may be patched into older Xlib release
+SAVE_LIBS="$LIBS"
+LIBS="$XVMC_LIBS"
+AC_CHECK_FUNCS([_XEatDataWords])
+LIBS="$SAVE_LIBS"
+
 # Checks for library functions.
 AC_CHECK_FUNCS([shmat])
 
diff --git a/src/XvMC.c b/src/XvMC.c
index 5a4cf0d..b3e97ec 100644
--- a/src/XvMC.c
+++ b/src/XvMC.c
@@ -16,6 +16,18 @@
 #include <sys/time.h>
 #include <X11/extensions/Xext.h>
 #include <X11/extensions/extutil.h>
+#include <limits.h>
+
+#ifndef HAVE__XEATDATAWORDS
+static inline void _XEatDataWords(Display *dpy, unsigned long n)
+{
+# ifndef LONG64
+    if (n >= (ULONG_MAX >> 2))
+        _XIOError(dpy);
+# endif
+    _XEatData (dpy, n << 2);
+}
+#endif
 
 static XExtensionInfo _xvmc_info_data;
 static XExtensionInfo *xvmc_info = &_xvmc_info_data;
@@ -134,7 +146,7 @@ XvMCSurfaceInfo * XvMCListSurfaceTypes(Display *dpy, XvPortID port, int *num)
 	       surface_info[i].flags = sinfo.flags;
 	    }
 	} else
-	   _XEatData(dpy, rep.length << 2);
+	   _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -207,7 +219,7 @@ XvImageFormatValues * XvMCListSubpictureTypes (
               ret[i].scanline_order = Info.scanline_order;
             }
         } else
-	   _XEatData(dpy, rep.length << 2);
+	   _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -278,7 +290,7 @@ Status _xvmc_create_context (
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
 	    *priv_count = rep.length;
 	} else
-	    _XEatData(dpy, rep.length << 2);
+	    _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -359,7 +371,7 @@ Status _xvmc_create_surface (
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;
         } else
-            _XEatData(dpy, rep.length << 2);
+            _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -449,7 +461,7 @@ Status _xvmc_create_subpicture (
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;
         } else
-            _XEatData(dpy, rep.length << 2);
+            _XEatDataWords(dpy, rep.length);
     }
 
     UnlockDisplay (dpy);
@@ -579,7 +591,7 @@ Status XvMCGetDRInfo(Display *dpy, XvPortID port,
 
 	} else {
 
-	    _XEatData(dpy, realSize);
+	    _XEatDataWords(dpy, rep.length);
 	    UnlockDisplay (dpy);
 	    SyncHandle ();
 	    return -1;
-- 
1.7.9.2

From 2712383813b26475dc6713888414d842be57f8ca Mon Sep 17 00:00:00 2001
From: Alan Coopersmith <[email protected]>
Date: Sat, 13 Apr 2013 00:50:02 -0700
Subject: [PATCH:libXvMC 2/5] integer overflow in XvMCListSurfaceTypes()
 [CVE-2013-1990 1/2]

rep.num is a CARD32 and needs to be bounds checked before multiplying
by sizeof(XvMCSurfaceInfo) to come up with the total size to allocate,
to avoid integer overflow leading to underallocation and writing data from
the network past the end of the allocated buffer.

Reported-by: Ilja Van Sprundel <[email protected]>
Signed-off-by: Alan Coopersmith <[email protected]>
---
 src/XvMC.c |    4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/XvMC.c b/src/XvMC.c
index b3e97ec..5d8c2cf 100644
--- a/src/XvMC.c
+++ b/src/XvMC.c
@@ -123,8 +123,8 @@ XvMCSurfaceInfo * XvMCListSurfaceTypes(Display *dpy, XvPortID port, int *num)
     }
 
     if(rep.num > 0) {
-	surface_info =
-	    (XvMCSurfaceInfo*)Xmalloc(rep.num * sizeof(XvMCSurfaceInfo));
+        if (rep.num < (INT_MAX / sizeof(XvMCSurfaceInfo)))
+            surface_info = Xmalloc(rep.num * sizeof(XvMCSurfaceInfo));
 
         if(surface_info) {
 	    xvmcSurfaceInfo sinfo;
-- 
1.7.9.2

From 478d4e5873eeee2ebdce6673e4e3469816ab63b8 Mon Sep 17 00:00:00 2001
From: Alan Coopersmith <[email protected]>
Date: Sat, 13 Apr 2013 00:50:02 -0700
Subject: [PATCH:libXvMC 3/5] integer overflow in XvMCListSubpictureTypes()
 [CVE-2013-1990 2/2]

rep.num is a CARD32 and needs to be bounds checked before multiplying by
sizeof(XvImageFormatValues) to come up with the total size to allocate,
to avoid integer overflow leading to underallocation and writing data from
the network past the end of the allocated buffer.

Reported-by: Ilja Van Sprundel <[email protected]>
Signed-off-by: Alan Coopersmith <[email protected]>
---
 src/XvMC.c |    4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/XvMC.c b/src/XvMC.c
index 5d8c2cf..8d602ec 100644
--- a/src/XvMC.c
+++ b/src/XvMC.c
@@ -184,8 +184,8 @@ XvImageFormatValues * XvMCListSubpictureTypes (
     }
 
     if(rep.num > 0) {
-        ret =
-	   (XvImageFormatValues*)Xmalloc(rep.num * sizeof(XvImageFormatValues));
+        if (rep.num < (INT_MAX / sizeof(XvImageFormatValues)))
+            ret = Xmalloc(rep.num * sizeof(XvImageFormatValues));
 
         if(ret) {
             xvImageFormatInfo Info;
-- 
1.7.9.2

From 5fd871e5f878810f8f8837725d548e07e89577ab Mon Sep 17 00:00:00 2001
From: Alan Coopersmith <[email protected]>
Date: Sat, 13 Apr 2013 00:50:02 -0700
Subject: [PATCH:libXvMC 4/5] integer overflow in _xvmc_create_*()

rep.length is a CARD32 and should be bounds checked before left-shifting
by 2 bits to come up with the total size to allocate, though in these
cases, no buffer overflow should occur here, since the XRead call is passed
the same rep.length << 2 length argument, but the *priv_count returned to
the caller could be interpreted or used to calculate a larger buffer size
than was actually allocated, leading them to go out of bounds.

Signed-off-by: Alan Coopersmith <[email protected]>
---
 src/XvMC.c |    9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/XvMC.c b/src/XvMC.c
index 8d602ec..d8bc59d 100644
--- a/src/XvMC.c
+++ b/src/XvMC.c
@@ -285,7 +285,8 @@ Status _xvmc_create_context (
     context->flags = rep.flags_return;
 
     if(rep.length) {
-	*priv_data = Xmalloc(rep.length << 2);
+	if (rep.length < (INT_MAX >> 2))
+	    *priv_data = Xmalloc(rep.length << 2);
 	if(*priv_data) {
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
 	    *priv_count = rep.length;
@@ -366,7 +367,8 @@ Status _xvmc_create_surface (
     }
 
     if(rep.length) {
-        *priv_data = Xmalloc(rep.length << 2);
+        if (rep.length < (INT_MAX >> 2))
+            *priv_data = Xmalloc(rep.length << 2);
         if(*priv_data) {
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;
@@ -456,7 +458,8 @@ Status _xvmc_create_subpicture (
     subpicture->component_order[3] = rep.component_order[3];
 
     if(rep.length) {
-        *priv_data = Xmalloc(rep.length << 2);
+        if (rep.length < (INT_MAX >> 2))
+            *priv_data = Xmalloc(rep.length << 2);
         if(*priv_data) {
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;
-- 
1.7.9.2

From e9415ddef2ac81d4139bd32d5e9cda9394a60051 Mon Sep 17 00:00:00 2001
From: Alan Coopersmith <[email protected]>
Date: Sat, 13 Apr 2013 01:20:08 -0700
Subject: [PATCH:libXvMC 5/5] Multiple unvalidated assumptions in
 XvMCGetDRInfo() [CVE-2013-1999]

The individual string sizes is assumed to not be more than the amount of
data read from the network, and could cause buffer overflow if they are.

The strings returned from the X server are assumed to be null terminated,
and could cause callers to read past the end of the buffer if they are not.

Also be sure to set the returned pointers to NULL, so callers don't try
accessing bad pointers on failure cases.

Reported-by: Ilja Van Sprundel <[email protected]>
Signed-off-by: Alan Coopersmith <[email protected]>
---
 src/XvMC.c |   36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/src/XvMC.c b/src/XvMC.c
index d8bc59d..cb42487 100644
--- a/src/XvMC.c
+++ b/src/XvMC.c
@@ -499,7 +499,6 @@ Status XvMCGetDRInfo(Display *dpy, XvPortID port,
     XExtDisplayInfo *info = xvmc_find_display(dpy);
     xvmcGetDRInfoReply rep;
     xvmcGetDRInfoReq  *req;
-    char *tmpBuf = NULL;
     CARD32 magic;
 
 #ifdef HAVE_SHMAT
@@ -510,6 +509,9 @@ Status XvMCGetDRInfo(Display *dpy, XvPortID port,
     here.tz_dsttime = 0;
 #endif
 
+    *name = NULL;
+    *busID = NULL;
+
     XvMCCheckExtension (dpy, info, BadImplementation);
 
     LockDisplay (dpy);
@@ -568,31 +570,31 @@ Status XvMCGetDRInfo(Display *dpy, XvPortID port,
 #endif
 
     if (rep.length > 0) {
-
-        int realSize = rep.length << 2;
-
-	tmpBuf = (char *) Xmalloc(realSize);
-	if (tmpBuf) {
-	    *name = (char *) Xmalloc(rep.nameLen);
-	    if (*name) {
-		*busID = (char *) Xmalloc(rep.busIDLen);
-		if (! *busID) {
-		    XFree(*name);
-		    XFree(tmpBuf);
-		}
-	    } else {
-		XFree(tmpBuf);
+	unsigned long realSize = 0;
+	char *tmpBuf = NULL;
+
+	if (rep.length < (INT_MAX >> 2)) {
+	    realSize = rep.length << 2;
+	    if (realSize >= (rep.nameLen + rep.busIDLen)) {
+		tmpBuf = Xmalloc(realSize);
+		*name = Xmalloc(rep.nameLen);
+		*busID = Xmalloc(rep.busIDLen);
 	    }
 	}
 
 	if (*name && *busID && tmpBuf) {
-
 	    _XRead(dpy, tmpBuf, realSize);
 	    strncpy(*name,tmpBuf,rep.nameLen);
+	    name[rep.nameLen - 1] = '\0';
 	    strncpy(*busID,tmpBuf+rep.nameLen,rep.busIDLen);
+	    busID[rep.busIDLen - 1] = '\0';
 	    XFree(tmpBuf);
-
 	} else {
+	    XFree(*name);
+	    *name = NULL;
+	    XFree(*busID);
+	    *name = NULL;
+	    XFree(tmpBuf);
 
 	    _XEatDataWords(dpy, rep.length);
 	    UnlockDisplay (dpy);
-- 
1.7.9.2