From b6a06a62d28977d98a0f3fdaac0be017832b3d7d Mon Sep 17 00:00:00 2001 From: Kostas Tsiounis Date: Thu, 5 Mar 2026 13:07:49 -0500 Subject: [PATCH] Check that integer representation of message is smaller than modulus In RSA cipher operations, the message is converted into an integer that needs to be smaller than the public key's modulus. A check is added to fail early with the proper message, instead of failing during the actual operation with a not so helpful message. Additional tests to verify that behaviour are introduced as well. Signed-off-by: Kostas Tsiounis --- .../com/ibm/crypto/plus/provider/RSA.java | 29 +++++++---- .../ibm/jceplus/junit/tests/BaseTestRSA.java | 50 +++++++++++++++++++ 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/ibm/crypto/plus/provider/RSA.java b/src/main/java/com/ibm/crypto/plus/provider/RSA.java index 347ed3a18..155eeec93 100644 --- a/src/main/java/com/ibm/crypto/plus/provider/RSA.java +++ b/src/main/java/com/ibm/crypto/plus/provider/RSA.java @@ -11,6 +11,7 @@ import com.ibm.crypto.plus.provider.base.NativeException; import com.ibm.crypto.plus.provider.base.RSACipher; import com.ibm.crypto.plus.provider.base.RSAPadding; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.security.AlgorithmParameters; import java.security.InvalidAlgorithmParameterException; @@ -45,6 +46,7 @@ public final class RSA extends CipherSpi { private int keyType = -1; private boolean initialized = false; private boolean encrypting = true; + private RSAPublicKey rsaPub = null; private static final boolean doTypeChecking; private static final String DO_TYPE_CHECKING = "com.ibm.crypto.provider.DoRSATypeChecking"; @@ -104,14 +106,21 @@ protected int engineDoFinal(byte[] input, int inOffset, int inLen, byte[] output try { int outLen = 0; if (this.encrypting) { - if (this.padding.isPadding(RSAPadding.RSAPAD_NONE) - && msgLength != engineGetOutputSize(0)) { - byte[] paddedInput = new byte[engineGetOutputSize(0)]; - System.arraycopy(this.msgBuffer.array(), 0, paddedInput, - paddedInput.length - msgLength, msgLength); - this.msgBuffer.clear(); - this.msgBuffer.put(paddedInput); - this.msgLength = paddedInput.length; + if (this.padding.isPadding(RSAPadding.RSAPAD_NONE)) { + if (msgLength != engineGetOutputSize(0)) { + byte[] paddedInput = new byte[engineGetOutputSize(0)]; + System.arraycopy(this.msgBuffer.array(), 0, paddedInput, + paddedInput.length - msgLength, msgLength); + this.msgBuffer.clear(); + this.msgBuffer.put(paddedInput); + this.msgLength = paddedInput.length; + } + + BigInteger m = new BigInteger(1, this.msgBuffer.array()); + BigInteger n = this.rsaPub.getModulus(); + if (m.compareTo(n) >= 0) { + throw new BadPaddingException("Message is larger than modulus"); + } } else if (this.padding.isPadding(RSAPadding.RSAPAD_PKCS1) && msgLength > pkcs1InputLimit()) { throw new IllegalBlockSizeException( @@ -286,7 +295,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params) } } try { - RSAPublicKey rsaPub = (RSAPublicKey) rsaKey; + this.rsaPub = (RSAPublicKey) rsaKey; rsaCipher.initialize(rsaPub.getOCKKey(), false); this.keyType = Cipher.PUBLIC_KEY; } catch (Exception e) { @@ -300,6 +309,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params) } try { RSAPrivateCrtKey rsaPriv = (RSAPrivateCrtKey) rsaKey; + this.rsaPub = null; rsaCipher.initialize(rsaPriv.getOCKKey(), false); this.keyType = Cipher.PRIVATE_KEY; } catch (Exception e) { @@ -313,6 +323,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params) } try { RSAPrivateKey rsaPriv = (RSAPrivateKey) rsaKey; + this.rsaPub = null; rsaCipher.initialize(rsaPriv.getOCKKey(), true); this.keyType = Cipher.PRIVATE_KEY; } catch (Exception e) { diff --git a/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java b/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java index 482aef919..b3fbedaa7 100644 --- a/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java +++ b/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java @@ -633,6 +633,56 @@ public void testRSACipherExceedInput() throws Exception { } } + @Test + public void testRSACipherNoPaddingExceedInput() throws Exception { + // FIPS does not support non-OAEP paddings. + assumeFalse("OpenJCEPlusFIPS".equals(getProviderName())); + + rsaKeyPairGen.initialize(2048); + KeyPair rsaKeyPair = rsaKeyPairGen.generateKeyPair(); + RSAPublicKey pubKey = (RSAPublicKey) rsaKeyPair.getPublic(); + Cipher cp = Cipher.getInstance("RSA/ECB/NoPadding", getProviderName()); + + BigInteger modulus = ((RSAKey) pubKey).getModulus(); + byte[] modulusPlusOne = modulus.add(BigInteger.ONE).toByteArray(); + byte[] plaintext = Arrays.copyOfRange(modulusPlusOne, 1, modulusPlusOne.length); // BigInteger has an extra byte for sign. + try { + cp.init(Cipher.ENCRYPT_MODE, pubKey); + cp.doFinal(plaintext); + + fail("Did not get expected BadPaddingException."); + } catch (BadPaddingException bpe) { + assertEquals("Message is larger than modulus", bpe.getMessage(), "Exception message is not what's expected."); + } + + byte[] modulusArray = modulus.toByteArray(); + plaintext = Arrays.copyOfRange(modulusArray, 1, modulusArray.length); // BigInteger has an extra byte for sign. + try { + cp.init(Cipher.ENCRYPT_MODE, pubKey); + cp.doFinal(plaintext); + + fail("Did not get expected BadPaddingException."); + } catch (BadPaddingException bpe) { + assertEquals("Message is larger than modulus", bpe.getMessage(), "Exception message is not what's expected."); + } + } + + @Test + public void testRSACipherSmallMessageLargeFirstByte() throws Exception { + // FIPS does not support non-OAEP paddings. + assumeFalse("OpenJCEPlusFIPS".equals(getProviderName())); + + rsaKeyPairGen.initialize(2048); + KeyPair rsaKeyPair = rsaKeyPairGen.generateKeyPair(); + RSAPublicKey pubKey = (RSAPublicKey) rsaKeyPair.getPublic(); + Cipher cp = Cipher.getInstance("RSA/ECB/NoPadding", getProviderName()); + cp.init(Cipher.ENCRYPT_MODE, pubKey); + + byte[] plaintext = {(byte) 0xFF, (byte) 0x01, (byte) 0x01, (byte) 0x01, (byte) 0x01}; + + cp.doFinal(plaintext); + } + @Test public void testRSACipher_init_cert() throws Exception { // FIXME