Changeset 1ba61ed in git for libpolys

Timestamp:

Sep 13, 2017, 3:49:06 PM (7 years ago)

Author:

Hans Schoenemann <hannes@…>

Branches:

(u'spielwiese', '2a584933abf2a2d3082034c7586d38bb6de1a30a')

Children:

6932bc7f98e394d56f60843c8b3372bd4c4e4b46

Parents:

b034c347789bcd8c6a686ffe3872a93603f716ae

Message:

fix: avoid int overflow in ntSize

File:

• libpolys/polys/ext_fields/transext.cc (modified) (3 diffs)

libpolys/polys/ext_fields/transext.cc

@@ -1799 +1799 @@
   fraction f = (fraction)a;
   poly p = NUM(f);
-  int noOfTerms = 0;
-  int numDegree = 0;
+  unsigned long noOfTerms = 0;
+  unsigned long numDegree = 0;
   if (p!=NULL)
   {
@@ -1806 +1806 @@
     noOfTerms = pLength(p);
   }
-  int denDegree = 0;
+  unsigned long denDegree = 0;
   if (!DENIS1(f))
   {
@@ -1813 +1813 @@
   }
   ntTest(a); // !!!!
-  return ((numDegree + denDegree)*(numDegree + denDegree) + 1) * noOfTerms; // must be >0
+  // avoid int overflow:
+  unsigned long t= ((numDegree + denDegree)*(numDegree + denDegree) + 1) * noOfTerms; // must be >0
+  if (t>INT_MAX) return INT_MAX;
+  else return (int)t;
 }