C++ OpenSSL RSA encryption ignores non printable characters

C++ ignores non printable bytes when trying to encode a byte array. The encoded result only contains the printable characters.

eg.: bytearray {0x00, 0x01, 0x02, 0x03, 0x41, 0x42, 0x43} would encrypt, and when decrypted (in c++ or c#), resulting bytes are {0x41, 0x42, 0x43}

The solution eludes us.

(note that keys in the code are for testing purposes)

#define _CRT_SECURE_NO_WARNINGS
#define OPENSSL_API_COMPAT 0x00908000L
#include <string>
#include <iostream>
#include <fstream>
#include <openssl/rsa.h>
#include <openssl/pem.h>
#include <openssl/err.h>
#include <openssl/ssl.h>
#include <openssl/evp.h>
#include <openssl/crypto.h>
#include <openssl/applink.c>
#include <openssl/pkcs7.h>


const char* private_key_filepath = "E:\private_key.pem";
const char* public_key_filepath = "E:\public_key.pem";
const char* filepathSha1 = "E:\encrypted_SHA_1.bin";
const char* filepathSha256 = "E:\encrypted_SHA256.bin";
const char* filepathSha512 = "E:\encrypted_SHA512.bin";

RSA * createRSAWithFilename(const char * filename, bool isPublicKey)
{
    FILE * fp = fopen(filename, "r");

    if (fp == NULL)
    {
        printf("Unable to open file %s n", filename);
        return NULL;
    }
    RSA *rsa = RSA_new();

    if (isPublicKey)
    {
        rsa = PEM_read_RSA_PUBKEY(fp, &rsa, NULL, NULL);
    }
    else
    {
        rsa = PEM_read_RSAPrivateKey(fp, &rsa, NULL, NULL);
    }

    fclose(fp);
    return rsa;
}

RSA * createRSA(unsigned char * key, bool isPublic)
{
    RSA *rsa = NULL;
    BIO *keybio;
    keybio = BIO_new_mem_buf(key, -1);
    if (keybio == NULL)
    {
        printf("Failed to create key BIO");
        return 0;
    }
    if (isPublic)
    {
        rsa = PEM_read_bio_RSA_PUBKEY(keybio, &rsa, NULL, NULL);
    }
    else
    {
        rsa = PEM_read_bio_RSAPrivateKey(keybio, &rsa, NULL, NULL);
    }

    return rsa;
}

int public_encrypt(unsigned char * data, int data_len, unsigned char * key, unsigned char *encrypted)
{
    //RSA * rsa = createRSAWithFilename(public_key_filepath, true);
    //https://groups.google.com/g/mailing.openssl.users/c/iv3lgFenxno
    RSA* rsa = createRSA(key, true);
    int result = RSA_public_encrypt(data_len, data, encrypted, rsa, RSA_PKCS1_OAEP_PADDING);
    return result;
}

int private_decrypt(unsigned char * enc_data, int data_len, unsigned char * key, unsigned char *decrypted)
{
    //RSA * rsa = createRSAWithFilename(private_key_filepath, false);
    RSA* rsa = createRSA(key, false);
    int  result = RSA_private_decrypt(data_len, enc_data, decrypted, rsa, RSA_PKCS1_OAEP_PADDING);
    return result;
}

void printLastError(const char *msg)
{
    char err[120];
    ERR_load_crypto_strings();
    ERR_error_string(ERR_get_error(), err);
    printf("%s ERROR: %sn", msg, err);
}

int main()
{
    OpenSSL_add_all_digests();
    OpenSSL_add_all_algorithms();
    //https://groups.google.com/g/mailing.openssl.users/c/iv3lgFenxno
    /* ------------------ PUBLIC KEY ---------------------*/
    std::string strPublicKey = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAthNIG6T4geicFrTBFJkFYvuL7NEVBktXx/g9vORQb0TtiUfa5AzwIykKUfinsvFWxijUEJNxNnsuvTDQGX9ntqr4xKhXkeqSDdQa61rmN76s0OceYBM3ZxQfhvjAwk9HXs/Dnt7XsaH30/SWbhKS16WJPw9aZE2qfJ2JVO0ph89PEkujoCZpo0Q5RiZgbwwqYTghJvP/24Uz9VA/5Ee4thvrs5wyLuP3c3fpkrPZBN9oedcOk9gIs67G3J9ReVwVAdNdwApGLmXuVDD5YVA/A5dqWzrKY0fWw6L41QZRxkZYM8ZhTo4wOxYNqrPXxiAaMgGv6OQmsZKHhNbny5amJQIDAQAB";
    int nPublicKeyLen = strPublicKey.size();      //strPublicKey by base64 Encoded public key string
    for (int i = 64; i < nPublicKeyLen; i += 64)
    {
        if (strPublicKey[i] != 'n')
        {
            strPublicKey.insert(i, "n");
        }
        i++;
    }

    strPublicKey.insert(0, "-----BEGIN PUBLIC KEY-----n");
    strPublicKey.append("n-----END PUBLIC KEY-----n");
    char *chPublicKey = const_cast<char *>(strPublicKey.c_str());

    /* ------------------ PRIVATE KEY ---------------------*/
    std::string strPrivateKey = "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC2E0gbpPiB6JwWtMEUmQVi+4vs0RUGS1fH+D285FBvRO2JR9rkDPAjKQpR+Key8VbGKNQQk3E2ey69MNAZf2e2qvjEqFeR6pIN1BrrWuY3vqzQ5x5gEzdnFB+G+MDCT0dez8Oe3texoffT9JZuEpLXpYk/D1pkTap8nYlU7SmHz08SS6OgJmmjRDlGJmBvDCphOCEm8//bhTP1UD/kR7i2G+uznDIu4/dzd+mSs9kE32h51w6T2Aizrsbcn1F5XBUB013ACkYuZe5UMPlhUD8Dl2pbOspjR9bDovjVBlHGRlgzxmFOjjA7Fg2qs9fGIBoyAa/o5CaxkoeE1ufLlqYlAgMBAAECggEBALAG9C/FznvJ5x4Mop7HUe3gQ+cwPKNZNASgew+jrB/nA781OOpBl+hmRRP60iauLLZnwp5D7bA+Z3GBP69m/V4c7WUzW3Y20f2Mknn355r9PRa6WyOHcIi3a8ORusog219OflGaH+4cKP1B+i1wx4a1A3tz3HAcpjbUF3GD2CLzKDBAYtsY2u6qxqv1qHw/0Ou3ZX5mzSaWpFCHD9IEhhtyWPN3nhis4rwTgTrMPIRcmec0NQbKK+tT3aT6fOsz3bvGFJEIPbUSkZPAa+1yEjoH4H2+LETYSz5bpogMVEmw+Kz+Akswi2JD7+YWP+YQd8xeo7Hi2X5H6/Bo7UTrSgECgYEA4PE1l79xJbrUSVgMsSzeodMJBEoDuyJCiVPblUM2TPFZtgEaciRdkGGOibK7NghcJZEa4mwFsw/LTCvIiRgJGSqiHXyft3p486KgzIXiWxMxcjUWdXIKxRS9B3bjN0aJkPaebilypputCDjlT776yOhwVTlM4w4USamJEAYnGHMCgYEAzzbm6BT1sYSZnNUIRtjFRSWuhqp28ewOpY8t8R8s/f+/J1KBKXpyxMVo3kP9U4iQleZZy7kmbbsphRAo4EbAVFfSzRlosBI+aC3eyV/2bM+2BhcFIbrqxhWogkpHcxreIFR3ZzZ8aoQyYuICGFVuYG1rqkN5vUx5KGzNuRKnWQcCgYEAgYqn4IJBuQbFJo/sj0RV2j/cm3m8eFKcmtWOSYFxjhIYJfawsjc7IuDr42To548c8g2EvAyVo69e1rkIPdt3seSZg/23RUTkzvpDPEWGPAjrmufzzSMmDKWKzveqlrSqmK9OQgbgng5dMbUCalCDwJwLxSjAnKoJCPr3kiDY9v0CgYA0yg/ZYD2Xk+hug/cv1VmsXAUyWYsj5rd0hh12KDMWImlYYzR0lLVYW9JzjW6kMU5kYeLOe8TGf7/8HjnLqTqmQhl6FtcMxBkSAV2Yf9IPtnrw4jx0c2IMRMhBKRs6v8WHBv3EVaUsnbEYb+uIt3r/JeEf50DEcQ8MtWtTkuT/UQKBgDWbR/tBuB3gAkH5SJUKJUMLY4/H0rHyX0OAx3gnuS2MlLxX+Los03+slvhu5Mw//tWMIjE+rVoJ80n2RIAvbI9YrRjZrYQvzOpmEFRRNtnhVo2maQxHoELE5zaJgr49IKzYIwrbrB9fezAd2a4qH+JS3DdeVpKrFG5x9uzyyRYt";
    int nPrivateKeyLen = strPrivateKey.size();
    for (int i = 64; i < nPrivateKeyLen; i += 64)
    {
        if (strPrivateKey[i] != 'n')
        {
            strPrivateKey.insert(i, "n");
        }
        i++;
    }
    strPrivateKey.insert(0, "-----BEGIN RSA PRIVATE KEY-----n");
    strPrivateKey.append("n-----END RSA PRIVATE KEY-----n");
    char *chPrivateKey = const_cast<char *>(strPrivateKey.c_str());
    /* ------------------- [END] KEYS ----------------------*/

    char  encrypted[512] = {};
    unsigned char decrypted[512] = {};
    const char* plainText = "This is RSA test";
    //char plainText[]= {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F};
    SSL_CONF_CTX* ctx = SSL_CONF_CTX_new();
    SSL_CONF_cmd(ctx, "SignatureAlgorithms", "ECDSA+SHA256:RSA+SHA256:DSA+SHA256");

    int encrypted_length = public_encrypt((unsigned char*)plainText, strlen(plainText), (unsigned char*)chPublicKey, (unsigned char*)encrypted);
    if (encrypted_length == -1)
    {
        printLastError("Public Encrypt failed ");
        exit(0);
    }
    printf("Encrypted length =%dn", encrypted_length);

    /* Write to file */
    FILE* fp = fopen(filepathSha256, "wb");
    fwrite(encrypted, encrypted_length, 1, fp);
    fclose(fp);

    //https://stackoverflow.com/questions/59112701/sha512-c-program-using-the-openssl-library
    //https://stackoverflow.com/questions/2262386/generate-sha256-with-openssl-and-c
    //https://www.openssl.org/docs/man1.1.1/man3/SSL_CONF_cmd_value_type.html

    //FILE* fp;
    //Get file size
    std::ifstream in_file(filepathSha256, std::ios::binary);
    in_file.seekg(0, std::ios::end);
    int file_size = in_file.tellg();
    printf("File length: %dn", file_size);

    //read from the file and decrypt it.
    char fileContent[256] = {};
    fp = fopen(filepathSha256, "rb");
    RSA* rsa = createRSA((unsigned char*)chPrivateKey, false);
    fread(fileContent, file_size, 1, fp);

    int decrypted_length = private_decrypt((unsigned char*)fileContent, file_size, (unsigned char*)chPrivateKey, (unsigned char*)decrypted);
    if (decrypted_length == -1)
    {
        printLastError("Private Decrypt failed ");
        exit(0);
    }
    printf("Decrypted Text =%sn", decrypted);
    printf("Decrypted Length =%dn", decrypted_length);


    return 0;

}

Leave a Comment