Расшифровка файла p7m с использованием CNG

Но эти функции не CNG, они унаследованы. Как мне это сделать в КПГ

все CryptMsg* функции не являются устаревшими или CNG. это вообще другой набор API, и внутренний может использовать оба (устаревший или CNG) для шифрования и дешифрования, подписи и проверки.

обычная последовательность вызовов следующая:

• CryptMsgOpenToDecode
• CryptMsgUpdate
• CryptMsgGetParam(CMSG_RECIPIENT_COUNT_PARAM)
• CryptMsgGetParam(CMSG_RECIPIENT_INFO_PARAM) в цикле
• CertGetSubjectCertificateFromStore
• CryptAcquireCertificatePrivateKey
• CryptMsgControl(CMSG_CTRL_DECRYPT)
• CryptMsgGetParam(CMSG_CONTENT_PARAM)

пример кода:

template <typename T> 
T HR(HRESULT& hr, T t)
{
    hr = t ? NOERROR : GetLastError();
    return t;
}

HRESULT GetMsgType(_In_ HCRYPTMSG hCryptMsg, _Out_ PULONG dwMsgType)
{
    ULONG cb = sizeof(ULONG);
    HRESULT hr;
    HR(hr, CryptMsgGetParam(hCryptMsg, CMSG_TYPE_PARAM, 0, dwMsgType, &cb));
    return hr;
}

HRESULT IsEncryptedMessage(_In_ HCRYPTMSG hCryptMsg)
{
    ULONG dwMsgType;

    HRESULT hr = GetMsgType(hCryptMsg, &dwMsgType);

    if (0 <= hr)
    {
        return dwMsgType == CMSG_ENVELOPED ? S_OK : HRESULT_FROM_NT(STATUS_OBJECT_TYPE_MISMATCH);
    }

    return hr;
}

HRESULT GetKey(_In_ HCERTSTORE hCertStore, 
               _In_ HCRYPTMSG hCryptMsg, 
               _Inout_ PCMSG_CTRL_DECRYPT_PARA pcdp,
               _Out_ BOOL *pfCallerFreeProvOrNCryptKey)
{
    ULONG cb = 0;
    PCERT_INFO pCertId = 0;

    HRESULT hr;
    while (HR(hr, CryptMsgGetParam(hCryptMsg, CMSG_RECIPIENT_INFO_PARAM, pcdp->dwRecipientIndex, pCertId, &cb)))
    {
        if (pCertId)
        {
            if (PCCERT_CONTEXT pCertContext = HR(hr, CertGetSubjectCertificateFromStore(hCertStore, X509_ASN_ENCODING, pCertId)))
            {
                HR(hr, CryptAcquireCertificatePrivateKey(pCertContext, 
                    CRYPT_ACQUIRE_COMPARE_KEY_FLAG|CRYPT_ACQUIRE_PREFER_NCRYPT_KEY_FLAG,
                    0, &pcdp->hNCryptKey, &pcdp->dwKeySpec, pfCallerFreeProvOrNCryptKey));

                CertFreeCertificateContext(pCertContext);
            }

            break;
        }

        pCertId = (PCERT_INFO)alloca(cb);
    }

    return hr;
}

HRESULT Decrypt(_In_ HCERTSTORE hCertStore, _In_ const BYTE* pbEncodedBlob, _In_ ULONG cbEncodedBlob, _Out_ PDATA_BLOB pContent)
{
    HRESULT hr;
    if (HCRYPTMSG hCryptMsg = HR(hr, CryptMsgOpenToDecode(PKCS_7_ASN_ENCODING, 0, 0, 0, 0, 0)))
    {
        if (HR(hr, CryptMsgUpdate(hCryptMsg, pbEncodedBlob, cbEncodedBlob, TRUE)))
        {
            if (S_OK == (hr = IsEncryptedMessage(hCryptMsg)))
            {
                CMSG_CTRL_DECRYPT_PARA cdp = { sizeof(cdp) };

                ULONG cb;

                if (HR(hr, CryptMsgGetParam(hCryptMsg, CMSG_RECIPIENT_COUNT_PARAM, 0, &cdp.dwRecipientIndex, &(cb = sizeof(ULONG)))))
                {
                    hr = HRESULT_FROM_WIN32(ERROR_NOT_FOUND);

                    if (cdp.dwRecipientIndex)
                    {
                        do 
                        {
                            --cdp.dwRecipientIndex;

                            BOOL fCallerFreeProvOrNCryptKey;
                            
                            if (S_OK == (hr = GetKey(hCertStore, hCryptMsg, &cdp, &fCallerFreeProvOrNCryptKey)))
                            {
                                if (HR(hr, CryptMsgControl(hCryptMsg, 0, CMSG_CTRL_DECRYPT, &cdp)))
                                {
                                    cdp.dwRecipientIndex = 0;

                                    PBYTE pb = 0;
                                    cb = 0;

                                    while (HR(hr, CryptMsgGetParam(hCryptMsg, CMSG_CONTENT_PARAM, 0, pb, &cb)))
                                    {
                                        if (pb)
                                        {
                                            if (pContent)
                                            {
                                                pContent->pbData = pb;
                                                pContent->cbData = cb;
                                                pb = 0;
                                            }
                                            break;
                                        }

                                        if (!(pb = new BYTE[cb]))
                                        {
                                            hr = E_OUTOFMEMORY;
                                            break;
                                        }
                                    }

                                    if (pb)
                                    {
                                        delete [] pb;
                                    }

                                    break;
                                }

                                if (fCallerFreeProvOrNCryptKey)
                                {
                                    if (CERT_NCRYPT_KEY_SPEC == cdp.dwKeySpec)
                                    {
                                        NCryptFreeObject(cdp.hNCryptKey);
                                    }
                                    else
                                    {
                                        CryptReleaseContext(cdp.hCryptProv, 0);
                                    }
                                }
                            }

                        } while (cdp.dwRecipientIndex);
                    }
                }
            }
        }

        CryptMsgClose(hCryptMsg);
    }

    return hr;
}

HRESULT Decrypt(_In_ const BYTE* pbEncodedBlob, _In_ ULONG cbEncodedBlob, _Out_ PDATA_BLOB pContent)
{
    HRESULT hr;
    if (HCERTSTORE hCertStore = HR(hr, CertOpenStore(CERT_STORE_PROV_SYSTEM, 0, 0, 
        CERT_SYSTEM_STORE_CURRENT_USER|CERT_STORE_OPEN_EXISTING_FLAG|CERT_STORE_READONLY_FLAG, L"My")))
    {
        hr = Decrypt(hCertStore, pbEncodedBlob, cbEncodedBlob, pContent);
        CertCloseStore(hCertStore, 0);
    }

    return hr;
}

void TestMsgDecrypt(_In_ const BYTE* pbEncodedBlob, _In_ ULONG cbEncodedBlob)
{
    DATA_BLOB db;
    if (S_OK == Decrypt(pbEncodedBlob, cbEncodedBlob, &db))
    {
        delete [] db.pbData;
    }
}

для создания зашифрованного сообщения для тестирования можно использовать этот код:

HRESULT EncryptMsg(_In_ PCCERT_CONTEXT pCertContext, _In_ PBYTE pbMsg, _In_ ULONG cbMsg, _Out_ PBYTE* ppb, _Out_ ULONG* pcb)
{
    HRESULT hr;

    CMSG_ENVELOPED_ENCODE_INFO EnvelopedEncodeInfo = { 
        sizeof(EnvelopedEncodeInfo), 0, { const_cast<PSTR>(szOID_NIST_AES256_CBC) }, 
        0, 1, const_cast<PCERT_INFO*>(&pCertContext->pCertInfo)
    }; 

    if (HCRYPTMSG hCryptMsg = HR(hr, CryptMsgOpenToEncode(
        PKCS_7_ASN_ENCODING, 0, CMSG_ENVELOPED, &EnvelopedEncodeInfo, 0, 0)))
    {
        if (HR(hr, CryptMsgUpdate(hCryptMsg, pbMsg, cbMsg, TRUE)))
        {
            PBYTE pbEncodedBlob = 0;
            ULONG cbEncodedBlob = 0;

            while (HR(hr, CryptMsgGetParam(hCryptMsg, CMSG_CONTENT_PARAM, 0, pbEncodedBlob, &cbEncodedBlob)))
            {
                if (pbEncodedBlob)
                {
                    *ppb = pbEncodedBlob, *pcb = cbEncodedBlob, pbEncodedBlob = 0;
                    break;
                }

                if (!(pbEncodedBlob = new UCHAR[cbEncodedBlob]))
                {
                    hr = E_OUTOFMEMORY;
                    break;
                }
            }

            if (pbEncodedBlob)
            {
                delete [] pbEncodedBlob;
            }
        }

        CryptMsgClose(hCryptMsg);
    }

    return hr;
}

полный тест (кодирование + декодирование)

HRESULT GetCert(_Out_ PCCERT_CONTEXT* ppCertContext, 
              _In_ PCSTR pszCertHash, 
              _In_ ULONG dwFlags = CERT_SYSTEM_STORE_CURRENT_USER|CERT_STORE_OPEN_EXISTING_FLAG|CERT_STORE_READONLY_FLAG)
{
    HRESULT hr;

    UCHAR hash[20];
    DATA_BLOB db = { sizeof(hash), hash };

    if (HR(hr, CryptStringToBinaryA(pszCertHash, 0, CRYPT_STRING_HEX, hash, &db.cbData, 0, 0)))
    {
        if (HCERTSTORE hCertStore = HR(hr, CertOpenStore(CERT_STORE_PROV_SYSTEM, 0, 0, dwFlags, L"My")))
        {
            *ppCertContext = HR(hr, CertFindCertificateInStore(hCertStore, 0, 0, CERT_FIND_HASH, &db, 0));
            CertCloseStore(hCertStore, 0);
        }
    }

    return hr;
}

HRESULT EncH(_In_ PCSTR pszCertHash, _In_ PBYTE pbMsg, _In_ ULONG cbMsg, _Out_ PBYTE* ppb, _Out_ ULONG* pcb)
{
    PCCERT_CONTEXT pCertContext;
    ULONG hr;
    if (NOERROR == (hr = GetCert(&pCertContext, pszCertHash)))
    {
        hr = EncryptMsg(pCertContext, pbMsg, cbMsg, ppb, pcb);
        CertFreeCertificateContext(pCertContext);
    }

    return hr;
}

void HTest(PCSTR pcsz)
{
    PBYTE pb;
    ULONG cb;
    if (NOERROR == EncH("***", (PBYTE)pcsz, (ULONG)strlen(pcsz), &pb, &cb))
    {
        TestMsgDecrypt(pb, cb);
        delete [] pb;
    }
}