changeset 1722:b0f4bd357814

avoid integer overflow in Polygon::linesCross0() The product of the two zOfVectorProduct() values could overflow. Check both values for different sign instead. Testcase: <html> <body> <img src="doesnt_matter.gif" width="250" height="700" usemap="#themap"> <map name="themap"> <area shape="poly" coords="1,250, 245,270, 223,513" href="http://www.dillo.org"> </map> </body> </html> Reported-by and Testcase-by: corvid <corvid@lavabit.com>
author Johannes Hofmann <Johannes.Hofmann@gmx.de>
date Fri, 15 Oct 2010 22:26:14 +0200
parents 8a4b1d7c0b0d
children b0cb317b40e0
files dw/types.cc
diffstat 1 files changed, 4 insertions(+), 3 deletions(-) [+]
line wrap: on
line diff
--- a/dw/types.cc	Fri Oct 15 19:33:01 2010 +0000
+++ b/dw/types.cc	Fri Oct 15 22:26:14 2010 +0200
@@ -177,9 +177,10 @@
    /** TODO Some more description */
    // If the scalar product is 0, it means that one point is on the second
    // line, so we check for <= 0, not < 0.
-   return
-      zOfVectorProduct (ax1 - bx1, ay1 - by1, bx2 - bx1, by2 - by1) *
-      zOfVectorProduct (ax2 - bx1, ay2 - by1, bx2 - bx1, by2 - by1) <= 0;
+   int z1 = zOfVectorProduct (ax1 - bx1, ay1 - by1, bx2 - bx1, by2 - by1);
+   int z2 = zOfVectorProduct (ax2 - bx1, ay2 - by1, bx2 - bx1, by2 - by1);
+
+   return (z1 <= 0 && z2 >= 0) || (z1 >= 0 && z2 <= 0); 
 }
 
 /**