Skip to content

Commit e388c78

Browse files
wrandelshoferwrandelshofer
committed
Fix issue #80: Bug in large array size inputs for JavaBigDecimalParser.parseBigDecimal.
AbstractBigDecimalParserTest.java: - Add a test for the input value that reproduces the problem. JavaBigDecimalFromByteArray.java, JavaBigDecimalFromCharArray.java, JavaBigDecimalFromCharSequence.java: - Fix the off-by-one issue in JavaBigDecimalFromCharArray. - Harmonize the code in all 3 implementations. Strings.java, FftMultiplierTest.java, JmhBigDecimalScalability.java, JmhDoubleScalability.java, JmhJavaBigIntegerFromByteArrayScalability.java, JmhJavaDoubleFromCharSequenceScalability.java: - Implement a better method for creating Strings from repeated characters.
1 parent 13272f5 commit e388c78

File tree

10 files changed

+107
-54
lines changed

10 files changed

+107
-54
lines changed

‎fastdoubleparser-dev/src/main/java/ch.randelshofer.fastdoubleparser/ch/randelshofer/fastdoubleparser/JavaBigDecimalFromByteArray.java

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -146,8 +146,8 @@ public BigDecimal parseBigDecimalString(byte[] str, int offset, int length) {
146146
BigDecimal parseBigDecimalStringWithManyDigits(byte[] str, int offset, int length) {
147147
final int integerPartIndex;
148148
final int nonZeroIntegerPartIndex;
149-
int decimalPointIndex = -1;
150149
int nonZeroFractionalPartIndex = -1;
150+
int decimalPointIndex = -1;
151151
final int exponentIndicatorIndex;
152152

153153
final int endIndex = offset + length;
@@ -169,15 +169,17 @@ BigDecimal parseBigDecimalStringWithManyDigits(byte[] str, int offset, int lengt
169169
// -----------------
170170
// skip leading zeroes
171171
integerPartIndex = index;
172-
while (index < endIndex - 8 && FastDoubleSwar.isEightZeroes(str, index)) {
172+
// swarLimit: We can process blocks of eight chars with SWAR, we must process the remaining chars individually.
173+
int swarLimit = Math.min(endIndex - 8, 1 << 30);
174+
while (index < swarLimit && FastDoubleSwar.isEightZeroes(str, index)) {
173175
index += 8;
174176
}
175177
while (index < endIndex && str[index] == '0') {
176178
index++;
177179
}
178180
// Count digits of integer part
179181
nonZeroIntegerPartIndex = index;
180-
while (index < endIndex - 8 && FastDoubleSwar.isEightDigits(str, index)) {
182+
while (index < swarLimit && FastDoubleSwar.isEightDigits(str, index)) {
181183
index += 8;
182184
}
183185
while (index < endIndex && FastDoubleSwar.isDigit(ch = str[index])) {
@@ -186,15 +188,15 @@ BigDecimal parseBigDecimalStringWithManyDigits(byte[] str, int offset, int lengt
186188
if (ch == '.') {
187189
decimalPointIndex = index++;
188190
// skip leading zeroes
189-
while (index < endIndex - 8 && FastDoubleSwar.isEightZeroes(str, index)) {
191+
while (index < swarLimit && FastDoubleSwar.isEightZeroes(str, index)) {
190192
index += 8;
191193
}
192194
while (index < endIndex && str[index] == '0') {
193195
index++;
194196
}
195197
nonZeroFractionalPartIndex = index;
196198
// Count digits of fraction part
197-
while (index < endIndex - 8 && FastDoubleSwar.isEightDigits(str, index)) {
199+
while (index < swarLimit && FastDoubleSwar.isEightDigits(str, index)) {
198200
index += 8;
199201
}
200202
while (index < endIndex && FastDoubleSwar.isDigit(ch = str[index])) {
@@ -247,8 +249,7 @@ BigDecimal parseBigDecimalStringWithManyDigits(byte[] str, int offset, int lengt
247249
illegal |= integerPartIndex == decimalPointIndex && decimalPointIndex == exponentIndicatorIndex;
248250
checkParsedBigDecimalBounds(illegal, index, endIndex, digitCountWithoutLeadingZeros, exponent);
249251

250-
return valueOfBigDecimalString(str, nonZeroIntegerPartIndex, decimalPointIndex, nonZeroFractionalPartIndex, exponentIndicatorIndex, isNegative, (int) exponent
251-
);
252+
return valueOfBigDecimalString(str, nonZeroIntegerPartIndex, decimalPointIndex, nonZeroFractionalPartIndex, exponentIndicatorIndex, isNegative, (int) exponent);
252253
}
253254

254255
/**
@@ -313,7 +314,7 @@ BigDecimal valueOfBigDecimalString(byte[] str, int integerPartIndex, int decimal
313314
} else {
314315
fractionalPart = ParseDigitsTaskByteArray.parseDigitsIterative(str, nonZeroFractionalPartIndex, exponentIndicatorIndex);
315316
}
316-
// If the integer part is not 0, we combine it with the fraction part.
317+
// If the integer part is 0, we can just use the fractional part.
317318
if (integerPart.signum() == 0) {
318319
significand = fractionalPart;
319320
} else {

‎fastdoubleparser-dev/src/main/java/ch.randelshofer.fastdoubleparser/ch/randelshofer/fastdoubleparser/JavaBigDecimalFromCharArray.java

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,8 @@ public BigDecimal parseBigDecimalString(char[] str, int offset, int length) {
141141
* Parses a big decimal string that has many digits.
142142
*/
143143
BigDecimal parseBigDecimalStringWithManyDigits(char[] str, int offset, int length) {
144-
final int nonZeroIntegerPartIndex;
145144
final int integerPartIndex;
145+
final int nonZeroIntegerPartIndex;
146146
int nonZeroFractionalPartIndex = -1;
147147
int decimalPointIndex = -1;
148148
final int exponentIndicatorIndex;
@@ -164,7 +164,9 @@ BigDecimal parseBigDecimalStringWithManyDigits(char[] str, int offset, int lengt
164164

165165
// Count digits of significand
166166
// -----------------
167+
// skip leading zeroes
167168
integerPartIndex = index;
169+
// swarLimit: We can process blocks of eight chars with SWAR, we must process the remaining chars individually.
168170
int swarLimit = Math.min(endIndex - 8, 1 << 30);
169171
while (index < swarLimit && FastDoubleSwar.isEightZeroes(str, index)) {
170172
index += 8;
@@ -244,10 +246,10 @@ BigDecimal parseBigDecimalStringWithManyDigits(char[] str, int offset, int lengt
244246
illegal |= integerPartIndex == decimalPointIndex && decimalPointIndex == exponentIndicatorIndex;
245247
checkParsedBigDecimalBounds(illegal, index, endIndex, digitCountWithoutLeadingZeros, exponent);
246248

247-
return valueOfBigDecimalString(str, nonZeroIntegerPartIndex, decimalPointIndex, nonZeroFractionalPartIndex, exponentIndicatorIndex, isNegative, (int) exponent);
249+
return valueOfBigDecimalString(str, nonZeroIntegerPartIndex, decimalPointIndex, nonZeroFractionalPartIndex, exponentIndicatorIndex, isNegative, (int) exponent
250+
);
248251
}
249252

250-
251253
/**
252254
* Parses a big decimal string after we have identified the parts of the significand,
253255
* and after we have obtained the exponent value.
@@ -273,8 +275,7 @@ BigDecimal parseBigDecimalStringWithManyDigits(char[] str, int offset, int lengt
273275
* @return the parsed big decimal
274276
*/
275277
BigDecimal valueOfBigDecimalString(char[] str, int integerPartIndex, int decimalPointIndex, int nonZeroFractionalPartIndex, int exponentIndicatorIndex, boolean isNegative, int exponent) {
276-
int integerExponent = exponentIndicatorIndex - decimalPointIndex - 1;
277-
int fractionDigitsCount = exponentIndicatorIndex - nonZeroFractionalPartIndex;
278+
int fractionDigitsCount = exponentIndicatorIndex - decimalPointIndex - 1;
278279
int nonZeroFractionDigitsCount = exponentIndicatorIndex - nonZeroFractionalPartIndex;
279280
int integerDigitsCount = decimalPointIndex - integerPartIndex;
280281
NavigableMap<Integer, BigInteger> powersOfTen = null;
@@ -306,16 +307,16 @@ BigDecimal valueOfBigDecimalString(char[] str, int integerPartIndex, int decimal
306307
if (powersOfTen == null) {
307308
powersOfTen = createPowersOfTenFloor16Map();
308309
}
309-
fillPowersOfNFloor16Recursive(powersOfTen, decimalPointIndex + 1, exponentIndicatorIndex);
310-
fractionalPart = ParseDigitsTaskCharArray.parseDigitsRecursive(str, decimalPointIndex + 1, exponentIndicatorIndex, powersOfTen, RECURSION_THRESHOLD);
310+
fillPowersOfNFloor16Recursive(powersOfTen, nonZeroFractionalPartIndex, exponentIndicatorIndex);
311+
fractionalPart = ParseDigitsTaskCharArray.parseDigitsRecursive(str, nonZeroFractionalPartIndex, exponentIndicatorIndex, powersOfTen, RECURSION_THRESHOLD);
311312
} else {
312-
fractionalPart = ParseDigitsTaskCharArray.parseDigitsIterative(str, decimalPointIndex + 1, exponentIndicatorIndex);
313+
fractionalPart = ParseDigitsTaskCharArray.parseDigitsIterative(str, nonZeroFractionalPartIndex, exponentIndicatorIndex);
313314
}
314-
// If the integer part is not 0, we combine it with the fraction part.
315+
// If the integer part is 0, we can just use the fractional part.
315316
if (integerPart.signum() == 0) {
316317
significand = fractionalPart;
317318
} else {
318-
BigInteger integerFactor = computePowerOfTen(powersOfTen, integerExponent);
319+
BigInteger integerFactor = computePowerOfTen(powersOfTen, fractionDigitsCount);
319320
significand = FftMultiplier.multiply(integerPart, integerFactor).add(fractionalPart);
320321
}
321322
} else {

‎fastdoubleparser-dev/src/main/java/ch.randelshofer.fastdoubleparser/ch/randelshofer/fastdoubleparser/JavaBigDecimalFromCharSequence.java

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ public JavaBigDecimalFromCharSequence() {
3838
*/
3939
public BigDecimal parseBigDecimalString(CharSequence str, int offset, int length) {
4040
try {
41-
int size = str.length();
42-
final int endIndex = checkBounds(size, offset, length);
41+
final int endIndex = checkBounds(str.length(), offset, length);
4342
if (hasManyDigits(length)) {
4443
return parseBigDecimalStringWithManyDigits(str, offset, length);
4544
}
@@ -137,7 +136,6 @@ public BigDecimal parseBigDecimalString(CharSequence str, int offset, int length
137136
nfe.initCause(e);
138137
throw nfe;
139138
}
140-
141139
}
142140

143141
/**
@@ -146,8 +144,8 @@ public BigDecimal parseBigDecimalString(CharSequence str, int offset, int length
146144
BigDecimal parseBigDecimalStringWithManyDigits(CharSequence str, int offset, int length) {
147145
final int integerPartIndex;
148146
final int nonZeroIntegerPartIndex;
149-
int decimalPointIndex = -1;
150147
int nonZeroFractionalPartIndex = -1;
148+
int decimalPointIndex = -1;
151149
final int exponentIndicatorIndex;
152150

153151
final int endIndex = offset + length;
@@ -169,15 +167,17 @@ BigDecimal parseBigDecimalStringWithManyDigits(CharSequence str, int offset, int
169167
// -----------------
170168
// skip leading zeroes
171169
integerPartIndex = index;
172-
while (index < endIndex - 8 && FastDoubleSwar.isEightZeroes(str, index)) {
170+
// swarLimit: We can process blocks of eight chars with SWAR, we must process the remaining chars individually.
171+
int swarLimit = Math.min(endIndex - 8, 1 << 30);
172+
while (index < swarLimit && FastDoubleSwar.isEightZeroes(str, index)) {
173173
index += 8;
174174
}
175175
while (index < endIndex && str.charAt(index) == '0') {
176176
index++;
177177
}
178178
// Count digits of integer part
179179
nonZeroIntegerPartIndex = index;
180-
while (index < endIndex - 8 && FastDoubleSwar.isEightDigits(str, index)) {
180+
while (index < swarLimit && FastDoubleSwar.isEightDigits(str, index)) {
181181
index += 8;
182182
}
183183
while (index < endIndex && FastDoubleSwar.isDigit(ch = str.charAt(index))) {
@@ -186,15 +186,15 @@ BigDecimal parseBigDecimalStringWithManyDigits(CharSequence str, int offset, int
186186
if (ch == '.') {
187187
decimalPointIndex = index++;
188188
// skip leading zeroes
189-
while (index < endIndex - 8 && FastDoubleSwar.isEightZeroes(str, index)) {
189+
while (index < swarLimit && FastDoubleSwar.isEightZeroes(str, index)) {
190190
index += 8;
191191
}
192192
while (index < endIndex && str.charAt(index) == '0') {
193193
index++;
194194
}
195195
nonZeroFractionalPartIndex = index;
196196
// Count digits of fraction part
197-
while (index < endIndex - 8 && FastDoubleSwar.isEightDigits(str, index)) {
197+
while (index < swarLimit && FastDoubleSwar.isEightDigits(str, index)) {
198198
index += 8;
199199
}
200200
while (index < endIndex && FastDoubleSwar.isDigit(ch = str.charAt(index))) {
@@ -312,6 +312,7 @@ BigDecimal valueOfBigDecimalString(CharSequence str, int integerPartIndex, int d
312312
} else {
313313
fractionalPart = ParseDigitsTaskCharSequence.parseDigitsIterative(str, nonZeroFractionalPartIndex, exponentIndicatorIndex);
314314
}
315+
// If the integer part is 0, we can just use the fractional part.
315316
if (integerPart.signum() == 0) {
316317
significand = fractionalPart;
317318
} else {

‎fastdoubleparser-dev/src/test/java/ch.randelshofer.fastdoubleparser/ch/randelshofer/fastdoubleparser/AbstractBigDecimalParserTest.java

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -135,12 +135,15 @@ protected List<NumberTestDataSupplier> createDataForLegalDecStrings() {
135135
new NumberTestDataSupplier("min exponent", () -> new NumberTestData("1e" + (Integer.MIN_VALUE + 1), BigDecimal.ONE.scaleByPowerOfTen(Integer.MIN_VALUE + 1))),
136136
new NumberTestDataSupplier("max exponent", () -> new NumberTestData("1e" + Integer.MAX_VALUE, BigDecimal.ONE.scaleByPowerOfTen(Integer.MAX_VALUE))),
137137

138-
new NumberTestDataSupplier("8.99...99e68", () -> new NumberTestData("8." + (repeat("9", 19)) + "e68", new BigDecimal("8." + (repeat("9", 19)) + "e68"))),
138+
new NumberTestDataSupplier("8.99...99e68", () -> new NumberTestData("8." + (repeat('9', 19)) + "e68", new BigDecimal("8." + (repeat('9', 19)) + "e68"))),
139139
new NumberTestDataSupplier("103203303403503603703803903.122232425262728292", () -> new NumberTestData("103203303403503603703803903.122232425262728292", new BigDecimal("103203303403503603703803903.122232425262728292"))),
140140
new NumberTestDataSupplier("122232425262728292.103203303403503603703803903", () -> new NumberTestData("122232425262728292.103203303403503603703803903", new BigDecimal("122232425262728292.103203303403503603703803903"))),
141141
new NumberTestDataSupplier("-103203303403503603703803903.122232425262728292e6789", () -> new NumberTestData("-103203303403503603703803903.122232425262728292e6789", new BigDecimal("-103203303403503603703803903.122232425262728292e6789"))),
142142
new NumberTestDataSupplier("122232425262728292.103203303403503603703803903e-6789", () -> new NumberTestData("122232425262728292.103203303403503603703803903e-6789", new BigDecimal("122232425262728292.103203303403503603703803903e-6789"))),
143-
new NumberTestDataSupplier("-122232425262728292.103203303403503603703803903e-6789", () -> new NumberTestData("-122232425262728292.103203303403503603703803903e-6789", new BigDecimal("-122232425262728292.103203303403503603703803903e-6789")))
143+
new NumberTestDataSupplier("-122232425262728292.103203303403503603703803903e-6789", () -> new NumberTestData("-122232425262728292.103203303403503603703803903e-6789", new BigDecimal("-122232425262728292.103203303403503603703803903e-6789"))),
144+
new NumberTestDataSupplier("-11000.0..0(652 fractional digits)",
145+
() -> new NumberTestData("-11000." + repeat('0', 652),
146+
new BigDecimal("-11000." + repeat('0', 652))))
144147
);
145148
}
146149

@@ -239,16 +242,16 @@ protected List<NumberTestDataSupplier> createTestDataForInputClassesInMethodPars
239242
*/
240243
protected List<NumberTestDataSupplier> createTestDataForInputClassesInMethodParseBigDecimalStringWithManyDigits() {
241244
return Arrays.asList(
242-
new NumberTestDataSupplier("illegal only negative sign", () -> new NumberTestData("-" + repeat("\000", 32), AbstractNumberParser.SYNTAX_ERROR, NumberFormatException.class)),
243-
new NumberTestDataSupplier("illegal only positive sign", () -> new NumberTestData("+" + repeat("\000", 32), AbstractNumberParser.SYNTAX_ERROR, NumberFormatException.class)),
245+
new NumberTestDataSupplier("illegal only negative sign", () -> new NumberTestData("-" + repeat('\000', 32), AbstractNumberParser.SYNTAX_ERROR, NumberFormatException.class)),
246+
new NumberTestDataSupplier("illegal only positive sign", () -> new NumberTestData("+" + repeat('\000', 32), AbstractNumberParser.SYNTAX_ERROR, NumberFormatException.class)),
244247

245248

246249
new NumberTestDataSupplier("significand with 40 zeroes in integer part", () -> new NumberTestData("significand with 40 zeroes in integer part", repeat("0", 40), BigDecimal::new)),
247-
new NumberTestDataSupplier("significand with 40 zeroes in fraction part", () -> new NumberTestData("." + repeat("0", 40), BigDecimal::new)),
250+
new NumberTestDataSupplier("significand with 40 zeroes in fraction part", () -> new NumberTestData("." + repeat('0', 40), BigDecimal::new)),
248251

249-
new NumberTestDataSupplier("significand with 10 leading zeros and 30 digits in integer part", () -> new NumberTestData("significand with 10 leading zeros and 30 digits in integer part", repeat("0", 10) + repeat("9", 30), BigDecimal::new)),
250-
new NumberTestDataSupplier("significand with 10 leading zeros and 30 digits in fraction part", () -> new NumberTestData("." + repeat("0", 10) + repeat("9", 30), BigDecimal::new)),
251-
new NumberTestDataSupplier("significand with 10 leading zeros and 30 digits in integer part and in fraction part", () -> new NumberTestData("significand with 10 leading zeros and 30 digits in integer part and in fraction part", repeat("0", 10) + repeat("9", 30) + "." + repeat("0", 10) + repeat("9", 30), BigDecimal::new)),
252+
new NumberTestDataSupplier("significand with 10 leading zeros and 30 digits in integer part", () -> new NumberTestData("significand with 10 leading zeros and 30 digits in integer part", repeat('0', 10) + repeat('9', 30), BigDecimal::new)),
253+
new NumberTestDataSupplier("significand with 10 leading zeros and 30 digits in fraction part", () -> new NumberTestData("." + repeat('0', 10) + repeat('9', 30), BigDecimal::new)),
254+
new NumberTestDataSupplier("significand with 10 leading zeros and 30 digits in integer part and in fraction part", () -> new NumberTestData("significand with 10 leading zeros and 30 digits in integer part and in fraction part", repeat('0', 10) + repeat("9", 30) + "." + repeat("0", 10) + repeat("9", 30), BigDecimal::new)),
252255

253256
new NumberTestDataSupplier("significand with 40 digits in integer part and exponent", () -> new NumberTestData("-1234567890123456789012345678901234567890e887799", BigDecimal::new)),
254257
new NumberTestDataSupplier("no significand but exponent 40 digits", () -> new NumberTestData("-e12345678901234567890123456789012345678901234567890", AbstractNumberParser.SYNTAX_ERROR, NumberFormatException.class)),

‎fastdoubleparser-dev/src/test/java/ch.randelshofer.fastdoubleparser/ch/randelshofer/fastdoubleparser/FftMultiplierTest.java

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,17 @@ public List<DynamicTest> dynamicTestsMultiply() {
3131
"2147483647",
3232
"9223372036854775807")),
3333
dynamicTest("'3','0'**84 * '4','0'**84", () -> shouldMultiplyFft(
34-
"3" + repeat("0", 84),
35-
"4" + repeat("0", 84))),
34+
"3" + repeat('0', 84),
35+
"4" + repeat('0', 84))),
3636
dynamicTest("'-','3','0'**84 * '4','0'**84", () -> shouldMultiplyFft(
37-
"-3" + repeat("0", 84),
38-
"4" + repeat("0", 84))),
37+
"-3" + repeat('0', 84),
38+
"4" + repeat('0', 84))),
3939
dynamicTest("'-','3','0'**84 * '-','4','0'**84", () -> shouldMultiplyFft(
40-
"-3" + repeat("0", 84),
41-
"-4" + repeat("0", 84))),
40+
"-3" + repeat('0', 84),
41+
"-4" + repeat('0', 84))),
4242
dynamicTest("'3','0'**100_000 * '4','0'**100_000", () -> shouldMultiplyFft(
43-
"3" + repeat("0", 100_000),
44-
"4" + repeat("0", 100_000)))
43+
"3" + repeat('0', 100_000),
44+
"4" + repeat('0', 100_000)))
4545
);
4646
}
4747

@@ -140,16 +140,16 @@ private void shouldMultiplyFft(BigInteger a, BigInteger b, BigInteger expected)
140140
public List<DynamicTest> dynamicTestsSquare() {
141141
return Arrays.asList(
142142
dynamicTest("'3','0'**84", () -> shouldSquare(
143-
"3" + repeat("0", 84)
143+
"3" + repeat('0', 84)
144144
)),
145145
dynamicTest("'-','3','0'**84", () -> shouldSquare(
146-
"-3" + repeat("0", 84)
146+
"-3" + repeat('0', 84)
147147
)),
148148
dynamicTest("'-','3','0'**84", () -> shouldSquare(
149-
"-3" + repeat("0", 84)
149+
"-3" + repeat('0', 84)
150150
)),
151151
dynamicTest("'3','0'**100_000", () -> shouldSquare(
152-
"3" + repeat("0", 100_000)
152+
"3" + repeat('0', 100_000)
153153
))
154154
);
155155

0 commit comments

Comments
 (0)