source: vbox/trunk/src/VBox/Runtime/common/crypto/key-openssl.cpp@ 96763

/* $Id: key-openssl.cpp 96763 2022-09-16 09:10:51Z vboxsync $ */
/** @file
 * IPRT - Crypto - Cryptographic Keys, OpenSSL glue.
 */

/*
 * Copyright (C) 2006-2022 Oracle and/or its affiliates.
 *
 * This file is part of VirtualBox base platform packages, as
 * available from https://www.virtualbox.org.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation, in version 3 of the
 * License.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <https://www.gnu.org/licenses>.
 *
 * The contents of this file may alternatively be used under the terms
 * of the Common Development and Distribution License Version 1.0
 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
 * in the VirtualBox distribution, in which case the provisions of the
 * CDDL are applicable instead of those of the GPL.
 *
 * You may elect to license modified versions of this file under the
 * terms and conditions of either the GPL or the CDDL or both.
 *
 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
 */


/*********************************************************************************************************************************
*   Header Files                                                                                                                 *
*********************************************************************************************************************************/
#include "internal/iprt.h"
#include <iprt/crypto/key.h>

#include <iprt/err.h>
#include <iprt/string.h>
#include <iprt/crypto/digest.h>


#ifdef IPRT_WITH_OPENSSL
# include "internal/iprt-openssl.h"
# include "internal/magics.h"
# include "internal/openssl-pre.h"
# include <openssl/evp.h>
# include "internal/openssl-post.h"
# ifndef OPENSSL_VERSION_NUMBER
#  error "Missing OPENSSL_VERSION_NUMBER!"
# endif

# include "key-internal.h"


/**
 * Creates an OpenSSL key for the given IPRT one, returning the message digest
 * algorithm if desired.
 *
 * @returns IRPT status code.
 * @param   hKey            The key to convert to an OpenSSL key.
 * @param   fNeedPublic     Set if we need the public side of the key.
 * @param   pszAlgoObjId    Alogrithm stuff we currently need.
 * @param   ppEvpKey        Where to return the pointer to the key structure.
 * @param   ppEvpMdType     Where to optionally return the message digest type.
 * @param   pErrInfo        Where to optionally return more error details.
 */
DECLHIDDEN(int) rtCrKeyToOpenSslKey(RTCRKEY hKey, bool fNeedPublic, void /*EVP_PKEY*/ **ppEvpKey, PRTERRINFO pErrInfo)
{
    *ppEvpKey = NULL;
    AssertReturn(hKey->u32Magic == RTCRKEYINT_MAGIC, VERR_INVALID_HANDLE);
    AssertReturn(fNeedPublic == !(hKey->fFlags & RTCRKEYINT_F_PRIVATE), VERR_WRONG_TYPE);
    AssertReturn(hKey->fFlags & RTCRKEYINT_F_INCLUDE_ENCODED, VERR_WRONG_TYPE); /* build misconfig */

    rtCrOpenSslInit();

    /*
     * Translate the key type from IPRT to EVP speak.
     */
    int         idKeyType;
    switch (hKey->enmType)
    {
        case RTCRKEYTYPE_RSA_PRIVATE:
        case RTCRKEYTYPE_RSA_PUBLIC:
            idKeyType = EVP_PKEY_RSA;
            break;
        default:
            return RTErrInfoSetF(pErrInfo, VERR_NOT_SUPPORTED, "Unsupported key type: %d", hKey->enmType);
    }

    /*
     * Allocate a new key structure and set its type.
     */
    EVP_PKEY *pEvpNewKey = EVP_PKEY_new();
    if (!pEvpNewKey)
        return RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new/%d failed", idKeyType);

    /*
     * Load the key into the structure.
     */
    const unsigned char *puchPublicKey = hKey->pbEncoded;
    EVP_PKEY *pRet;
    if (fNeedPublic)
        *ppEvpKey = pRet = d2i_PublicKey(idKeyType, &pEvpNewKey, &puchPublicKey, hKey->cbEncoded);
    else
        *ppEvpKey = pRet = d2i_PrivateKey(idKeyType, &pEvpNewKey, &puchPublicKey, hKey->cbEncoded);
    if (pRet)
        return VINF_SUCCESS;

    /* Bail out: */
    EVP_PKEY_free(pEvpNewKey);
    return RTErrInfoSet(pErrInfo, VERR_CR_PKIX_OSSL_D2I_PUBLIC_KEY_FAILED,
                        fNeedPublic ? "d2i_PublicKey failed" : "d2i_PrivateKey failed");
}


/**
 * Creates an OpenSSL key for the given IPRT one, returning the message digest
 * algorithm if desired.
 *
 * @returns IRPT status code.
 * @param   hKey            The key to convert to an OpenSSL key.
 * @param   fNeedPublic     Set if we need the public side of the key.
 * @param   pszAlgoObjId    Alogrithm stuff we currently need.
 * @param   ppEvpKey        Where to return the pointer to the key structure.
 * @param   ppEvpMdType     Where to optionally return the message digest type.
 * @param   pErrInfo        Where to optionally return more error details.
 */
DECLHIDDEN(int) rtCrKeyToOpenSslKeyEx(RTCRKEY hKey, bool fNeedPublic, const char *pszAlgoObjId,
                                      void /*EVP_PKEY*/ **ppEvpKey, const void /*EVP_MD*/ **ppEvpMdType, PRTERRINFO pErrInfo)
{
    *ppEvpKey = NULL;
    if (ppEvpMdType)
        *ppEvpMdType = NULL;
    AssertReturn(hKey->u32Magic == RTCRKEYINT_MAGIC, VERR_INVALID_HANDLE);
    AssertReturn(fNeedPublic == !(hKey->fFlags & RTCRKEYINT_F_PRIVATE), VERR_WRONG_TYPE);
    AssertReturn(hKey->fFlags & RTCRKEYINT_F_INCLUDE_ENCODED, VERR_WRONG_TYPE); /* build misconfig */

    rtCrOpenSslInit();

    /*
     * Translate algorithm object ID into stuff that OpenSSL wants.
     */
    int iAlgoNid = OBJ_txt2nid(pszAlgoObjId);
    if (iAlgoNid == NID_undef)
        return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALGO_NOT_KNOWN,
                             "Unknown public key algorithm [OpenSSL]: %s", pszAlgoObjId);
    const char *pszAlgoSn = OBJ_nid2sn(iAlgoNid);

# if OPENSSL_VERSION_NUMBER >= 0x10001000 && !defined(LIBRESSL_VERSION_NUMBER)
    int idAlgoPkey = 0;
    int idAlgoMd = 0;
    if (!OBJ_find_sigid_algs(iAlgoNid, &idAlgoMd, &idAlgoPkey))
        return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALGO_NOT_KNOWN_EVP,
                             "OBJ_find_sigid_algs failed on %u (%s, %s)", iAlgoNid, pszAlgoSn, pszAlgoObjId);
    if (ppEvpMdType)
    {
        const EVP_MD *pEvpMdType = EVP_get_digestbynid(idAlgoMd);
        if (!pEvpMdType)
            return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALGO_NOT_KNOWN_EVP,
                                 "EVP_get_digestbynid failed on %d (%s, %s)", idAlgoMd, pszAlgoSn, pszAlgoObjId);
        *ppEvpMdType = pEvpMdType;
    }
# else
    const EVP_MD *pEvpMdType = EVP_get_digestbyname(pszAlgoSn);
    if (!pEvpMdType)
        return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALGO_NOT_KNOWN_EVP,
                             "EVP_get_digestbyname failed on %s (%s)", pszAlgoSn, pszAlgoObjId);
    if (ppEvpMdType)
        *ppEvpMdType = pEvpMdType;
# endif

    /*
     * Allocate a new key structure and set its type.
     */
    EVP_PKEY *pEvpNewKey = EVP_PKEY_new();
    if (!pEvpNewKey)
        return RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", iAlgoNid);

    int rc;
# if OPENSSL_VERSION_NUMBER >= 0x10001000 && !defined(LIBRESSL_VERSION_NUMBER)
    if (EVP_PKEY_set_type(pEvpNewKey, idAlgoPkey))
    {
        int idKeyType = EVP_PKEY_base_id(pEvpNewKey);
# else
        int idKeyType = pEvpNewKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
# endif
        if (idKeyType != NID_undef)

        {
            /*
             * Load the key into the structure.
             */
            const unsigned char *puchPublicKey = hKey->pbEncoded;
            EVP_PKEY *pRet;
            if (fNeedPublic)
                *ppEvpKey = pRet = d2i_PublicKey(idKeyType, &pEvpNewKey, &puchPublicKey, hKey->cbEncoded);
            else
                *ppEvpKey = pRet = d2i_PrivateKey(idKeyType, &pEvpNewKey, &puchPublicKey, hKey->cbEncoded);
            if (pRet)
                return VINF_SUCCESS;

            /* Bail out: */
            rc = RTErrInfoSet(pErrInfo, VERR_CR_PKIX_OSSL_D2I_PUBLIC_KEY_FAILED,
                              fNeedPublic ? "d2i_PublicKey failed" : "d2i_PrivateKey failed");
        }
        else
# if OPENSSL_VERSION_NUMBER < 0x10001000 || defined(LIBRESSL_VERSION_NUMBER)
            rc = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR, "EVP_PKEY_type() failed");
# else
            rc = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR, "EVP_PKEY_base_id() failed");
    }
    else
        rc = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
                           "EVP_PKEY_set_type(%u) failed (sig algo %s)", idAlgoPkey, pszAlgoSn);
# endif

    EVP_PKEY_free(pEvpNewKey);
    return rc;
}

#endif /* IPRT_WITH_OPENSSL */