Handle NIST format signatures

BoringSSL expects EC signatures in DER format, but COSE transports
them in NIST format (bare concatenation of R and S).

Add code to support conversion from NIST to DER, and provide separate
signature verification methods for each format.

Loosely based on aosp/3064203.

Bug: 338745127
Test: atest VmAttestationTestApp
Change-Id: If5dd07e5e5b94a4c7491b2f828cfe73a2492df64
diff --git a/libs/bssl/error/src/lib.rs b/libs/bssl/error/src/lib.rs
index 82a2d5e..0bdb821 100644
--- a/libs/bssl/error/src/lib.rs
+++ b/libs/bssl/error/src/lib.rs
@@ -88,6 +88,8 @@
     EC_KEY_new_by_curve_name,
     EC_KEY_set_public_key_affine_coordinates,
     EC_POINT_get_affine_coordinates,
+    ECDSA_SIG_new,
+    ECDSA_SIG_set0,
     ECDSA_sign,
     ECDSA_size,
     ECDSA_verify,
@@ -105,6 +107,7 @@
     EVP_DigestVerifyInit,
     HKDF,
     HMAC,
+    i2d_ECDSA_SIG,
     RAND_bytes,
     SHA256,
 }
diff --git a/libs/bssl/src/ec_key.rs b/libs/bssl/src/ec_key.rs
index 897f8a1..9eb5e5c 100644
--- a/libs/bssl/src/ec_key.rs
+++ b/libs/bssl/src/ec_key.rs
@@ -22,15 +22,17 @@
 use alloc::vec::Vec;
 use bssl_avf_error::{ApiName, Error, Result};
 use bssl_sys::{
-    BN_bin2bn, BN_bn2bin_padded, BN_clear_free, BN_new, CBB_flush, CBB_len, ECDSA_sign, ECDSA_size,
-    ECDSA_verify, EC_GROUP_get_curve_name, EC_GROUP_new_by_curve_name, EC_KEY_check_key,
-    EC_KEY_free, EC_KEY_generate_key, EC_KEY_get0_group, EC_KEY_get0_public_key,
-    EC_KEY_marshal_private_key, EC_KEY_new_by_curve_name, EC_KEY_parse_private_key,
-    EC_KEY_set_public_key_affine_coordinates, EC_POINT_get_affine_coordinates,
-    NID_X9_62_prime256v1, NID_secp384r1, BIGNUM, EC_GROUP, EC_KEY, EC_POINT,
+    i2d_ECDSA_SIG, BN_bin2bn, BN_bn2bin_padded, BN_clear_free, BN_new, CBB_flush, CBB_len,
+    ECDSA_SIG_free, ECDSA_SIG_new, ECDSA_SIG_set0, ECDSA_sign, ECDSA_size, ECDSA_verify,
+    EC_GROUP_get_curve_name, EC_GROUP_new_by_curve_name, EC_KEY_check_key, EC_KEY_free,
+    EC_KEY_generate_key, EC_KEY_get0_group, EC_KEY_get0_public_key, EC_KEY_marshal_private_key,
+    EC_KEY_new_by_curve_name, EC_KEY_parse_private_key, EC_KEY_set_public_key_affine_coordinates,
+    EC_POINT_get_affine_coordinates, NID_X9_62_prime256v1, NID_secp384r1, BIGNUM, ECDSA_SIG,
+    EC_GROUP, EC_KEY, EC_POINT,
 };
 use cbor_util::{get_label_value, get_label_value_as_bytes};
 use ciborium::Value;
+use core::mem;
 use core::ptr::{self, NonNull};
 use coset::{
     iana::{self, EnumI64},
@@ -144,7 +146,7 @@
     /// Verifies the DER-encoded ECDSA `signature` of the `digest` with the current `EcKey`.
     ///
     /// Returns Ok(()) if the verification succeeds, otherwise an error will be returned.
-    pub fn ecdsa_verify(&self, signature: &[u8], digest: &[u8]) -> Result<()> {
+    pub fn ecdsa_verify_der(&self, signature: &[u8], digest: &[u8]) -> Result<()> {
         // The `type` argument should be 0 as required in the BoringSSL spec.
         const TYPE: i32 = 0;
 
@@ -163,6 +165,15 @@
         check_int_result(ret, ApiName::ECDSA_verify)
     }
 
+    /// Verifies the NIST-encoded (R | S) ECDSA `signature` of the `digest` with the current
+    /// `EcKey`.
+    ///
+    /// Returns Ok(()) if the verification succeeds, otherwise an error will be returned.
+    pub fn ecdsa_verify_nist(&self, signature: &[u8], digest: &[u8]) -> Result<()> {
+        let signature = ec_cose_signature_to_der(signature)?;
+        self.ecdsa_verify_der(&signature, digest)
+    }
+
     /// Signs the `digest` with the current `EcKey` using ECDSA.
     ///
     /// Returns the DER-encoded ECDSA signature.
@@ -324,6 +335,85 @@
     }
 }
 
+/// Convert a R | S format NIST signature to a DER-encoded form.
+fn ec_cose_signature_to_der(signature: &[u8]) -> Result<Vec<u8>> {
+    let mut ec_sig = EcSignature::new()?;
+    ec_sig.load_from_nist(signature)?;
+    ec_sig.to_der()
+}
+
+/// Wrapper for an `ECDSA_SIG` object representing an EC signature.
+struct EcSignature(NonNull<ECDSA_SIG>);
+
+impl EcSignature {
+    /// Allocate a signature object.
+    fn new() -> Result<Self> {
+        // SAFETY: We take ownership of the returned pointer if it is non-null.
+        let signature = unsafe { ECDSA_SIG_new() };
+
+        let signature =
+            NonNull::new(signature).ok_or_else(|| to_call_failed_error(ApiName::ECDSA_SIG_new))?;
+        Ok(Self(signature))
+    }
+
+    /// Populate the signature parameters from a NIST encoding (R | S).
+    fn load_from_nist(&mut self, signature: &[u8]) -> Result<()> {
+        let coord_bytes = signature.len() / 2;
+        if signature.len() != 2 * coord_bytes {
+            return Err(Error::InternalError);
+        }
+        let mut r = BigNum::from_slice(&signature[..coord_bytes])?;
+        let mut s = BigNum::from_slice(&signature[coord_bytes..])?;
+
+        check_int_result(
+            // SAFETY: The ECDSA_SIG was properly allocated and not yet freed. We have ownership
+            // of the two BigNums and they are not null.
+            unsafe { ECDSA_SIG_set0(self.0.as_mut(), r.as_mut_ptr(), s.as_mut_ptr()) },
+            ApiName::ECDSA_SIG_set0,
+        )?;
+
+        // On success, the ECDSA_SIG has taken ownership of the BigNums.
+        mem::forget(r);
+        mem::forget(s);
+
+        Ok(())
+    }
+
+    /// Return the signature encoded as DER.
+    fn to_der(&self) -> Result<Vec<u8>> {
+        // SAFETY: The ECDSA_SIG was properly allocated and not yet freed. Null is a valid
+        // value for `outp`; no output is written.
+        let len = unsafe { i2d_ECDSA_SIG(self.0.as_ptr(), ptr::null_mut()) };
+        if len < 0 {
+            return Err(to_call_failed_error(ApiName::i2d_ECDSA_SIG));
+        }
+
+        let mut buf = vec![0; len.try_into().map_err(|_| Error::InternalError)?];
+        let outp = &mut buf.as_mut_ptr();
+        // SAFETY: The ECDSA_SIG was properly allocated and not yet freed. `outp` is a non-null
+        // pointer to a mutable buffer of the right size to which the result will be written.
+        let final_len = unsafe { i2d_ECDSA_SIG(self.0.as_ptr(), outp) };
+        if final_len < 0 {
+            return Err(to_call_failed_error(ApiName::i2d_ECDSA_SIG));
+        }
+        // The input hasn't changed, so the length of the output shouldn't have. If it has we
+        // already have potentially undefined behavior so panic.
+        assert_eq!(
+            len, final_len,
+            "i2d_ECDSA_SIG returned inconsistent lengths: {len}, {final_len}"
+        );
+
+        Ok(buf)
+    }
+}
+
+impl Drop for EcSignature {
+    fn drop(&mut self) {
+        // SAFETY: The pointer was allocated by `ECDSA_SIG_new`.
+        unsafe { ECDSA_SIG_free(self.0.as_mut()) };
+    }
+}
+
 /// Wrapper of an `EC_GROUP` reference.
 struct EcGroup<'a>(&'a EC_GROUP);
 
diff --git a/libs/bssl/tests/eckey_test.rs b/libs/bssl/tests/eckey_test.rs
index 3c0e45d..00ed6c5 100644
--- a/libs/bssl/tests/eckey_test.rs
+++ b/libs/bssl/tests/eckey_test.rs
@@ -88,7 +88,7 @@
     assert_eq!(digest, sha256(MESSAGE1)?);
 
     let signature = ec_key.ecdsa_sign(&digest)?;
-    ec_key.ecdsa_verify(&signature, &digest)?;
+    ec_key.ecdsa_verify_der(&signature, &digest)?;
     // Building a `PKey` from a temporary `CoseKey` should work as the lifetime
     // of the `PKey` is not tied to the lifetime of the `CoseKey`.
     let pkey = PKey::from_cose_public_key(&ec_key.cose_public_key()?)?;
@@ -103,7 +103,7 @@
     let digest = digester.digest(MESSAGE1)?;
 
     let signature = ec_key.ecdsa_sign(&digest)?;
-    ec_key.ecdsa_verify(&signature, &digest)?;
+    ec_key.ecdsa_verify_der(&signature, &digest)?;
     let pkey = PKey::from_cose_public_key(&ec_key.cose_public_key()?)?;
     pkey.verify(&signature, MESSAGE1, Some(digester))
 }
@@ -117,7 +117,7 @@
 
     let mut ec_key2 = EcKey::new_p256()?;
     ec_key2.generate_key()?;
-    let err = ec_key2.ecdsa_verify(&signature, &digest).unwrap_err();
+    let err = ec_key2.ecdsa_verify_der(&signature, &digest).unwrap_err();
     let expected_err = Error::CallFailed(ApiName::ECDSA_verify, EcdsaError::BadSignature.into());
     assert_eq!(expected_err, err);
 
@@ -137,7 +137,7 @@
     let signature = ec_key.ecdsa_sign(&digest1)?;
     let digest2 = sha256(MESSAGE2)?;
 
-    let err = ec_key.ecdsa_verify(&signature, &digest2).unwrap_err();
+    let err = ec_key.ecdsa_verify_der(&signature, &digest2).unwrap_err();
     let expected_err = Error::CallFailed(ApiName::ECDSA_verify, EcdsaError::BadSignature.into());
     assert_eq!(expected_err, err);
     Ok(())
diff --git a/rialto/tests/test.rs b/rialto/tests/test.rs
index 9151ce1..0d57301 100644
--- a/rialto/tests/test.rs
+++ b/rialto/tests/test.rs
@@ -228,7 +228,7 @@
     let tbs_cert = cert.tbs_certificate;
     let digest = sha256(&tbs_cert.to_der().unwrap()).unwrap();
     authority_public_key
-        .ecdsa_verify(cert.signature.raw_bytes(), &digest)
+        .ecdsa_verify_der(cert.signature.raw_bytes(), &digest)
         .expect("Failed to verify the certificate signature with the authority public key");
 
     // Checks that the certificate's subject public key is equal to the key in the CSR.
diff --git a/service_vm/requests/src/client_vm.rs b/service_vm/requests/src/client_vm.rs
index d2e674b..3b5d68a 100644
--- a/service_vm/requests/src/client_vm.rs
+++ b/service_vm/requests/src/client_vm.rs
@@ -120,7 +120,7 @@
 fn ecdsa_verify(key: &EcKey, signature: &[u8], message: &[u8]) -> bssl_avf::Result<()> {
     // The message was signed with ECDSA with curve P-256 and SHA-256 at the signature generation.
     let digest = sha256(message)?;
-    key.ecdsa_verify(signature, &digest)
+    key.ecdsa_verify_der(signature, &digest)
 }
 
 fn ecdsa_sign(key: &EcKey, message: &[u8]) -> bssl_avf::Result<Vec<u8>> {
diff --git a/service_vm/requests/src/dice.rs b/service_vm/requests/src/dice.rs
index 1fa02a1..ec05f66 100644
--- a/service_vm/requests/src/dice.rs
+++ b/service_vm/requests/src/dice.rs
@@ -231,6 +231,9 @@
     /// generateCertificateRequestV2.cddl:
     ///
     /// PubKeyEd25519 / PubKeyECDSA256 / PubKeyECDSA384
+    ///
+    /// The signature should be in the format defined by COSE in RFC 9053 section 2 for the
+    /// specifric algorithm.
     pub(crate) fn verify(&self, signature: &[u8], message: &[u8]) -> Result<()> {
         match &self.0.kty {
             KeyType::Assigned(iana::KeyType::EC2) => {
@@ -248,7 +251,7 @@
                     }
                 };
                 let digest = digester.digest(message)?;
-                Ok(public_key.ecdsa_verify(signature, &digest)?)
+                Ok(public_key.ecdsa_verify_nist(signature, &digest)?)
             }
             KeyType::Assigned(iana::KeyType::OKP) => {
                 let curve_type =