6646626 [X.Org bug #4947] Xv extension not byte-swapping properly
authorAlan Coopersmith <Alan.Coopersmith@Sun.COM>
Fri, 11 Jan 2008 15:42:48 -0800
changeset 268 ca267932a176
parent 267 5928c19cc0de
child 269 4de1d496f68b
6646626 [X.Org bug #4947] Xv extension not byte-swapping properly
open-src/xserver/xorg/patch-list
open-src/xserver/xorg/xv-swap.patch
--- a/open-src/xserver/xorg/patch-list	Fri Jan 11 14:46:21 2008 -0800
+++ b/open-src/xserver/xorg/patch-list	Fri Jan 11 15:42:48 2008 -0800
@@ -47,3 +47,4 @@
 6425775.patch
 KB_EncodingAtXCrash.patch
 6278039.patch
+xv-swap.patch
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/open-src/xserver/xorg/xv-swap.patch	Fri Jan 11 15:42:48 2008 -0800
@@ -0,0 +1,261 @@
+###############################################################################
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use subject to license terms.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, and/or sell copies of the Software, and to permit persons
+# to whom the Software is furnished to do so, provided that the above
+# copyright notice(s) and this permission notice appear in all copies of
+# the Software and that both the above copyright notice(s) and this
+# permission notice appear in supporting documentation.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+# OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL
+# INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# 
+# Except as contained in this notice, the name of a copyright holder
+# shall not be used in advertising or otherwise to promote the sale, use
+# or other dealings in this Software without prior written authorization
+# of the copyright holder.
+#
+
+Fixes Xv extension byte swapping bugs:
+    X.Org Bugzilla #4947 <https://bugs.freedesktop.org/show_bug.cgi?id=4947>
+    Sun bug 6646626 <http://bugs.opensolaris.org/bugdatabase/view_bug.do?bug_id=6646626>
+    
+    Don't use swapped data after swapping it.   When done swapping data,
+    send the swapped data, not the address of the pointer to it, to the client.
+    Make sure we swap all the right fields.
+
+Contributed upstream for Xorg server 1.5 in commits:
+	dfd682b582636a36345144bcf835e3ee46718d90
+	6844bd2e63490870bab3c469eec6030354ef2865
+
+--- Xext/xvdisp.c	2006-09-17 23:04:17.000000000 -0700
++++ Xext/xvdisp.c	2008-01-02 16:11:40.967091000 -0800
+@@ -374,6 +374,7 @@
+   xvAdaptorInfo ainfo;
+   xvQueryAdaptorsReply rep;
+   int totalSize;
++  int nameSize;
+   int na;
+   XvAdaptorPtr pa;
+   int nf;
+@@ -439,12 +440,12 @@
+       ainfo.base_id = pa->base_id;
+       ainfo.num_ports = pa->nPorts;
+       ainfo.type = pa->type;
+-      ainfo.name_size = strlen(pa->name);
++      ainfo.name_size = nameSize = strlen(pa->name);
+       ainfo.num_formats = pa->nFormats;
+ 
+       _WriteAdaptorInfo(client, &ainfo);
+ 
+-      WriteToClient(client, ainfo.name_size, pa->name);
++      WriteToClient(client, nameSize, pa->name);
+ 
+       nf = pa->nFormats;
+       pf = pa->pFormats;
+@@ -470,6 +471,7 @@
+   xvEncodingInfo einfo;
+   xvQueryEncodingsReply rep;
+   int totalSize;
++  int nameSize;
+   XvPortPtr pPort;
+   int ne;
+   XvEncodingPtr pe;
+@@ -514,13 +516,13 @@
+   while (ne--) 
+     {
+       einfo.encoding = pe->id;
+-      einfo.name_size = strlen(pe->name);
++      einfo.name_size = nameSize = strlen(pe->name);
+       einfo.width = pe->width;
+       einfo.height = pe->height;
+       einfo.rate.numerator = pe->rate.numerator;
+       einfo.rate.denominator = pe->rate.denominator;
+       _WriteEncodingInfo(client, &einfo);
+-      WriteToClient(client, einfo.name_size, pe->name);
++      WriteToClient(client, nameSize, pe->name);
+       pe++;
+     }
+ 
+@@ -1001,18 +1003,19 @@
+   rep.text_size = 0;
+ 
+   for(i = 0, pAtt = pPort->pAdaptor->pAttributes; 
+-      i < rep.num_attributes; i++, pAtt++) 
++      i < pPort->pAdaptor->nAttributes; i++, pAtt++) 
+   {    
+       rep.text_size += (strlen(pAtt->name) + 1 + 3) & ~3L;
+   }
+ 
+-  rep.length = (rep.num_attributes * sz_xvAttributeInfo) + rep.text_size;
++  rep.length = (pPort->pAdaptor->nAttributes * sz_xvAttributeInfo)
++      + rep.text_size;
+   rep.length >>= 2;
+ 
+   _WriteQueryPortAttributesReply(client, &rep);
+ 
+   for(i = 0, pAtt = pPort->pAdaptor->pAttributes; 
+-      i < rep.num_attributes; i++, pAtt++) 
++      i < pPort->pAdaptor->nAttributes; i++, pAtt++) 
+   {
+       size = strlen(pAtt->name) + 1;  /* pass the NULL */
+       Info.flags = pAtt->flags;
+@@ -1223,6 +1226,7 @@
+   XvPortPtr pPort;
+   int *offsets;
+   int *pitches;
++  int planeLength;
+   REQUEST(xvQueryImageAttributesReq);
+ 
+   REQUEST_SIZE_MATCH(xvQueryImageAttributesReq);
+@@ -1262,7 +1266,7 @@
+ 
+   rep.type = X_Reply;
+   rep.sequenceNumber = client->sequence;
+-  rep.length = num_planes << 1;
++  rep.length = planeLength = num_planes << 1;
+   rep.num_planes = num_planes;
+   rep.width = width;
+   rep.height = height;
+@@ -1270,8 +1274,8 @@
+  
+   _WriteQueryImageAttributesReply(client, &rep);
+   if(client->swapped)
+-    SwapLongs((CARD32*)offsets, rep.length);
+-  WriteToClient(client, rep.length << 2, (char*)offsets);
++    SwapLongs((CARD32*)offsets, planeLength);
++  WriteToClient(client, planeLength << 2, (char*)offsets);
+ 
+   xfree(offsets);
+ 
+@@ -1305,7 +1309,7 @@
+ 
+   pImage = pPort->pAdaptor->pImages;
+   
+-  for(i = 0; i < rep.num_formats; i++, pImage++) {
++  for(i = 0; i < pPort->pAdaptor->nImages; i++, pImage++) {
+      info.id = pImage->id; 	
+      info.type = pImage->type; 	
+      info.byte_order = pImage->byte_order; 
+@@ -1504,6 +1508,7 @@ SProcXvShmPutImage(ClientPtr client)
+   swapl(&stuff->gc, n);
+   swapl(&stuff->shmseg, n);
+   swapl(&stuff->id, n);
++  swapl(&stuff->offset, n);
+   swaps(&stuff->src_x, n);
+   swaps(&stuff->src_y, n);
+   swaps(&stuff->src_w, n);
+@@ -1512,7 +1517,6 @@ SProcXvShmPutImage(ClientPtr client)
+   swaps(&stuff->drw_y, n);
+   swaps(&stuff->drw_w, n);
+   swaps(&stuff->drw_h, n);
+-  swaps(&stuff->offset, n);
+   swaps(&stuff->width, n);
+   swaps(&stuff->height, n);
+   return ProcXvShmPutImage(client);
+@@ -1603,9 +1607,10 @@ SProcXvQueryImageAttributes(ClientPtr cl
+   register char n;
+   REQUEST(xvQueryImageAttributesReq);
+   swaps(&stuff->length, n);
++  swapl(&stuff->port, n);
+   swapl(&stuff->id, n);
+   swaps(&stuff->width, n);
+-  swaps(&stuff->width, n);
++  swaps(&stuff->height, n);
+   return ProcXvQueryImageAttributes(client);
+ }
+ 
+@@ -1632,7 +1636,7 @@
+   swaps(&rep->version, n);
+   swaps(&rep->revision, n);
+   
+-  (void)WriteToClient(client, sz_xvQueryExtensionReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvQueryExtensionReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1648,7 +1652,7 @@
+   swapl(&rep->length, n);
+   swaps(&rep->num_adaptors, n);
+   
+-  (void)WriteToClient(client, sz_xvQueryAdaptorsReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvQueryAdaptorsReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1664,7 +1668,7 @@
+   swapl(&rep->length, n);
+   swaps(&rep->num_encodings, n);
+   
+-  (void)WriteToClient(client, sz_xvQueryEncodingsReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvQueryEncodingsReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1771,7 +1775,7 @@
+   swaps(&rep->sequenceNumber, n);
+   swapl(&rep->length, n);
+ 
+-  (void)WriteToClient(client, sz_xvGrabPortReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvGrabPortReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1787,7 +1791,7 @@
+   swapl(&rep->length, n);
+   swapl(&rep->value, n);
+ 
+-  (void)WriteToClient(client, sz_xvGetPortAttributeReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvGetPortAttributeReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1804,7 +1808,7 @@
+   swaps(&rep->actual_width, n);
+   swaps(&rep->actual_height, n);
+ 
+-  (void)WriteToClient(client, sz_xvQueryBestSizeReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvQueryBestSizeReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1821,7 +1825,7 @@
+   swapl(&rep->num_attributes, n);
+   swapl(&rep->text_size, n);
+ 
+-  (void)WriteToClient(client, sz_xvQueryPortAttributesReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvQueryPortAttributesReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1840,7 +1844,7 @@
+   swaps(&rep->width, n);
+   swaps(&rep->height, n);
+ 
+-  (void)WriteToClient(client, sz_xvQueryImageAttributesReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvQueryImageAttributesReply, (char *)rep);
+ 
+   return Success;
+ }
+@@ -1857,7 +1861,7 @@
+   swapl(&rep->length, n);
+   swapl(&rep->num_formats, n);
+ 
+-  (void)WriteToClient(client, sz_xvListImageFormatsReply, (char *)&rep);
++  (void)WriteToClient(client, sz_xvListImageFormatsReply, (char *)rep);
+ 
+   return Success;
+ }