crypto: fix RSA-PSS oversized salt handling

Let OpenSSL handle representable salt lengths so verification of an
impossible length resolves false. Guard values outside the native int32
parameter range to prevent SignJob from silently ignoring them.

Move the digest-size helper next to HKDF, its remaining consumer.

Signed-off-by: Filip Skokan <panva.ip@gmail.com>
PR-URL: #65550
Reviewed-By: Aviv Keller <me@aviv.sh>
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: panva

lib/internal/crypto/hkdf.js

@@ -22,7 +22,6 @@ const {
 const { kMaxLength } = require('buffer');
 
 const {
-  getDigestSizeInBytes,
   jobPromise,
   normalizeHashName,
   toBuf,
@@ -142,6 +141,22 @@ function hkdfSync(hash, key, salt, info, length) {
   return bits;
 }
 
+function getDigestSizeInBytes(name) {
+  switch (name) {
+    case 'SHA-1':
+      return 20;
+    case 'SHA-256': // Fall through
+    case 'SHA3-256':
+      return 32;
+    case 'SHA-384': // Fall through
+    case 'SHA3-384':
+      return 48;
+    case 'SHA-512': // Fall through
+    case 'SHA3-512':
+      return 64;
+  }
+}
+
 function validateHkdfDeriveBitsLength(length, hash) {
   if (length === null)
     throw lazyDOMException('length cannot be null', 'OperationError');

lib/internal/crypto/rsa.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const {
-  MathCeil,
   TypedArrayPrototypeGetBuffer,
   Uint8Array,
 } = primordials;
@@ -24,12 +23,11 @@ const {
 } = internalBinding('crypto');
 
 const {
-  validateInt32,
+  isInt32,
 } = require('internal/validators');
 
 const {
   bigIntArrayToUnsignedInt,
-  getDigestSizeInBytes,
   getUsagesMask,
   jobPromise,
   normalizeHashName,
@@ -241,19 +239,16 @@ function rsaSignVerify(key, data, { saltLength }, signature) {
     throw lazyDOMException(`Key must be a ${type} key`, 'InvalidAccessError');
 
   const algorithm = getCryptoKeyAlgorithm(key);
-  if (algorithm.name === 'RSA-PSS') {
-    try {
-      validateInt32(
-        saltLength,
-        'algorithm.saltLength',
-        0,
-        MathCeil((algorithm.modulusLength - 1) / 8) -
-          getDigestSizeInBytes(algorithm.hash.name) - 2);
-    } catch (err) {
-      throw lazyDOMException(
-        'The operation failed for an operation-specific reason',
-        { name: 'OperationError', cause: err });
-    }
+  // RsaPssParams converts saltLength to an unsigned long, but SignJob only
+  // accepts int32 values.
+  if (algorithm.name === 'RSA-PSS' && !isInt32(saltLength)) {
+    // EMSA-PSS-VERIFY treats an impossible salt length as inconsistent.
+    if (mode === kSignJobModeVerify)
+      return false;
+
+    throw lazyDOMException(
+      'The operation failed for an operation-specific reason',
+      'OperationError');
   }
 
   return jobPromise(() => new SignJob(

lib/internal/crypto/util.js

@@ -1070,22 +1070,6 @@ function getBlockSize(name) {
   }
 }
 
-function getDigestSizeInBytes(name) {
-  switch (name) {
-    case 'SHA-1':
-      return 20;
-    case 'SHA-256': // Fall through
-    case 'SHA3-256':
-      return 32;
-    case 'SHA-384': // Fall through
-    case 'SHA3-384':
-      return 48;
-    case 'SHA-512': // Fall through
-    case 'SHA3-512':
-      return 64;
-  }
-}
-
 function validateKeyOps(keyOps, usagesSet) {
   if (keyOps === undefined) return;
   validateArray(keyOps, 'keyData.key_ops');
@@ -1159,7 +1143,6 @@ module.exports = {
   bigIntArrayToUnsignedBigInt,
   bigIntArrayToUnsignedInt,
   getBlockSize,
-  getDigestSizeInBytes,
   getStringOption,
   getUsagesMask,
   getUsagesFromMask,

test/parallel/test-webcrypto-sign-verify-rsa.js

@@ -227,22 +227,26 @@ async function testSaltLength(keyLength, hash, hLen) {
 
   const signature = await subtle.sign(
     { name: 'RSA-PSS', saltLength: max }, privateKey, data);
-  await assert.rejects(
-    subtle.sign({ name: 'RSA-PSS', saltLength: max + 1 }, privateKey, data), (err) => {
-      assert.strictEqual(err.name, 'OperationError');
-      assert.strictEqual(err.cause?.code, 'ERR_OUT_OF_RANGE');
-      assert.strictEqual(err.cause?.message, `The value of "algorithm.saltLength" is out of range. It must be >= 0 && <= ${max}. Received ${max + 1}`);
-      return true;
-    });
-  await subtle.verify(
-    { name: 'RSA-PSS', saltLength: max }, publicKey, signature, data);
-  await assert.rejects(
-    subtle.verify({ name: 'RSA-PSS', saltLength: max + 1 }, publicKey, signature, data), (err) => {
-      assert.strictEqual(err.name, 'OperationError');
-      assert.strictEqual(err.cause?.code, 'ERR_OUT_OF_RANGE');
-      assert.strictEqual(err.cause?.message, `The value of "algorithm.saltLength" is out of range. It must be >= 0 && <= ${max}. Received ${max + 1}`);
-      return true;
-    });
+  assert.strictEqual(await subtle.verify(
+    { name: 'RSA-PSS', saltLength: max }, publicKey, signature, data), true);
+
+  for (const saltLength of [max + 1, 0x7fffffff]) {
+    await assert.rejects(
+      subtle.sign({ name: 'RSA-PSS', saltLength }, privateKey, data), {
+        name: 'OperationError',
+      });
+    assert.strictEqual(await subtle.verify(
+      { name: 'RSA-PSS', saltLength }, publicKey, signature, data), false);
+  }
+
+  for (const saltLength of [0x80000000, 0xffffffff]) {
+    await assert.rejects(
+      subtle.sign({ name: 'RSA-PSS', saltLength }, privateKey, data), {
+        name: 'OperationError',
+      });
+    assert.strictEqual(await subtle.verify(
+      { name: 'RSA-PSS', saltLength }, publicKey, signature, data), false);
+  }
 }
 
 (async function() {