Incorrect maximum encryption size for RSA OAEP (#423)

Incorrect maximum encryption size for RSA OAEP

### Checklist
- [X] I've run tests to see all new and existing tests pass
- [X] I've followed the code style of the rest of the project
- [X] I've read the [Contribution Guidelines](CONTRIBUTING.md)
- [X] I've updated the documentation if necessary

#### If you've made changes to `gyb` files
- [n/a] I've run `./scripts/generate_boilerplate_files_with_gyb.sh` and
included updated generated files in a commit of this pull request

### Motivation:
The `maximumEncryptSize` function is hardcoded to use 42 as the hash
offset, but the RFC actually says it's "2*hLen-2" so 42 is only valid
for SHA1. SHA256 should be 62 (2*32-2). This adds a hash length onto the
Digest enum, which can then be used in the length calculation.

In writing the tests for this, I also stumbled on the unsafe PEM
representation for RSA Public Keys don't allow 1024-bit keys as
documented, so this also fixes that.

### Modifications:
* Added a `hashBitLength` to the RSA Digest which is then used in the
`maximumEncryptSize` to properly compute the maximum length
* Corrected the minimum key size for RSA unsafe PEM public keys to 1024
from 2048

### Result:
* The `maximumEncryptSize` function will return the expected value for
RSA OAEP SHA256 keys. I don't believe anything calls this internally, so
this would be for external consumers.
* Unsafe construction of RSA public keys now allow 1024 bit keys. It was
documented as supporting them, but the check was still 2048 (probably
copy-paste error from the safe variant). This only expands the potential
uses, so it shouldn't introduce any new failures.

Co-authored-by: Mike Schrag <schrag@apple.com>
This commit is contained in:
Mike SchragandMike Schrag authored and GitHub committed 2025-10-29 10:58:46 +00:00
1 parent 87a9e066d9
commit e8ed8867ec
2 files changed
+42 -4

No files matched your search

+13 -4
View File
@@ -508,7 +508,7 @@ extension _RSA.Encryption {
/// - Warning: Key sizes less than 2048 are not recommended and should only be used for compatibility reasons.
public init(unsafePEMRepresentation pemRepresentation: String) throws {
self.backing = try BackingPublicKey(pemRepresentation: pemRepresentation)
guard self.keySizeInBits >= 2048, self.keySizeInBits % 8 == 0 else { throw CryptoKitError.incorrectParameterSize }
guard self.keySizeInBits >= 1024, self.keySizeInBits % 8 == 0 else { throw CryptoKitError.incorrectParameterSize }
}
/// Construct an RSA public key from a DER representation.
@@ -675,6 +675,14 @@ extension _RSA.Encryption {
internal enum Digest {
case sha1
case sha256
/// Returns the number of bits in the resulting hash
var hashBitLength: Int {
switch self {
case .sha1: return 160
case .sha256: return 256
}
}
}
}
@@ -694,7 +702,7 @@ extension _RSA.Encryption.PublicKey {
/// Return the maximum amount of data in bytes this key can encrypt in a single operation when using
/// the specified padding mode.
///
/// ## Common values:
/// ## Common values (for PKCS1 OAEP SHA1):
///
/// Key size|Padding|Max length
/// -|-|-
@@ -703,8 +711,9 @@ extension _RSA.Encryption.PublicKey {
/// 4096|PKCS-OAEP|470 bytes
public func maximumEncryptSize(with padding: _RSA.Encryption.Padding) -> Int {
switch padding.backing {
case .pkcs1_oaep:
return (self.keySizeInBits / 8) - 42
case let .pkcs1_oaep(Digest):
// https://datatracker.ietf.org/doc/html/rfc8017#section-7.1.1
return (self.keySizeInBits / 8) - (2 * Digest.hashBitLength / 8) - 2
}
}
@@ -140,6 +140,35 @@ final class TestRSAEncryption: XCTestCase {
XCTAssertEqual(primitives.publicExponent, e)
}
}
func testMaximumEncryptSize() throws {
let pemRepresentation1024 = """
-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCTLbu1QZhtXWKCHOjavP5NUCwJ
5DwjoMKGlEM/PQOMiY+wup8R1kCOHV6g+FvJ86laHJc0gqwFf1U51YxtQFy7cGV4
W2zJeTkqadO2fvTCjbZU+Oa78iVtTynq5h4yRWrTmveyzInhdVpi075Ql2hpGuET
H1qYVxqaDIJEHyETDQIDAQAB
-----END PUBLIC KEY-----
"""
let pubKey1024 = try _RSA.Encryption.PublicKey(unsafePEMRepresentation: pemRepresentation1024)
XCTAssertEqual(86, pubKey1024.maximumEncryptSize(with: .PKCS1_OAEP))
XCTAssertEqual(62, pubKey1024.maximumEncryptSize(with: .PKCS1_OAEP_SHA256))
let pemRepresentation2048 = """
-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8IRKcs5FrHlWye2lfwc
Hr0Pi8g5iZhGMOOwmyIVsNULAvUIGZlIw38NNqebH3eF6ZxPiSRpwPwIs6QRcH5/
IwbHkUc0KdBbUwXDrLs0w00I7Flu5RP7IEfkOZdDGEWFY1pA3H1HaogxKFc5k3mM
s7pW6oty1eP4O7aVa/Pp363Vba7EZ2nru9lz4Ta+JU8UIHbpoddMGikGEKHrQ/Ge
n9RMNzSIy/e7TgTwC39GKn8fwN6VfcdNjvIhJrFNha/ORNArpzup7FUUauGLKt3a
jgsIjrAPBp63+Sy7+aFVoGTvI7DCkZ/Wv3JCFRuTAdYOa0A1xiqhTb1pcypvrd2T
ZQIDAQAB
-----END PUBLIC KEY-----
"""
let pubKey2048 = try _RSA.Encryption.PublicKey(pemRepresentation: pemRepresentation2048)
XCTAssertEqual(214, pubKey2048.maximumEncryptSize(with: .PKCS1_OAEP))
XCTAssertEqual(190, pubKey2048.maximumEncryptSize(with: .PKCS1_OAEP_SHA256))
}
}
struct RSAEncryptionOAEPTestGroup: Codable {