We Keep Coding

Java - decrypting messages using a private key [duplicate]

I'm writing a program which takes as input from the console - the name of a zip file, name of a zip file to be made containing the (de/en)crypted files generated from the first zip and a file containing the public key. I get the exception when decrypting:
exception Exception in thread "main" javax.crypto.BadPaddingException: Decryption error
at sun.security.rsa.RSAPadding.unpadV15(RSAPadding.java:380)
at sun.security.rsa.RSAPadding.unpad(RSAPadding.java:291)
at com.sun.crypto.provider.RSACipher.doFinal(RSACipher.java:363)
at com.sun.crypto.provider.RSACipher.engineDoFinal(RSACipher.java:389)
at javax.crypto.Cipher.doFinal(Cipher.java:2165)
at com.Main.decrypt(Main.java:67)
at com.Main.main(Main.java:201)
Can't figure out why I get this exception?
Public key:
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCE3pA746UfpC8sFk8ZJp0yupyJqj5jy6cjdxUYoP7mCm7c0mqQDeCcDNBYW2eSozCioPrH/9L+CDQEPLYakoem+jFnUKDH5+pru/0PJTJJF8Xh/ZT9eJlvsYBr1/qSfICf6RTs7kzwq9IuSZBw7/tfNEF9i0A8FVox6HOopXod1QIDAQAB
Private key:
MIICXQIBAAKBgQCE3pA746UfpC8sFk8ZJp0yupyJqj5jy6cjdxUYoP7mCm7c0mqQDeCcDNBYW2eSozCioPrH/9L+CDQEPLYakoem+jFnUKDH5+pru/0PJTJJF8Xh/ZT9eJlvsYBr1/qSfICf6RTs7kzwq9IuSZBw7/tfNEF9i0A8FVox6HOopXod1QIDAQABAoGANOFrYBqK5lvu1koOswDWQZFZqcSSzh8IZyoGwGWa7S0r0EECXlDXmuPSq8e9IfRG8ALHrH+ZlrbnFOSgyVSWHfpj3aH+qknoSX5TW2rMQHih8865xuqheMQ+RTZ7+BRDqNsYkzxB/Z8mqzpoJQSYf+H7nWxdDCgAJVYZzxl3DmUCQQD32iEjnwiwUjii8slcmvCEZl+z84DWNdvJOg6Z38sI4AvrfpKc1WAcDg1rNZCKrRgokh54wpLt08cpFcrD04c3AkEAiTzDmc0bdgfg5wj6xHFZpYlBwiGm/bjOR2PS57P0GNU5PsDllRbFqIuzArITutO5lvZZImzuYz7Lf+cQ73pxUwJBAOdEwmdaneDo17A0m2+to3/nhqWDMVSwLMU3RyiNigZeCMFU+bkd4PBMrHi9IoJDwacZsRU9eZwxYEUV8H2Jg0ECQEEkOqRSm2pXKwX/WSjNtQPCNxhy6NUeV6vDUmTxIjh3XYjP/ynZeVEbnoj1BjB0N2/U11Jj6nPpZqb7gyppMEkCQQCoGdVYDipU+hMMnvxa0zOIyQc/a+HE0lESqn+2ZPafYi9Z1RldRMvUXhP8U7s+OuhRwprdw2ivvOFrnWyz9lL2
The code for the program is bellow . Any help is wellcomed :)
package com;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.security.GeneralSecurityException;
import java.security.KeyFactory;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.util.Base64;
import java.util.Enumeration;
import java.util.Scanner;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import javax.crypto.Cipher;
import org.bouncycastle.asn1.ASN1EncodableVector;
import org.bouncycastle.asn1.ASN1Integer;
import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1Sequence;
import org.bouncycastle.asn1.DERNull;
import org.bouncycastle.asn1.DEROctetString;
import org.bouncycastle.asn1.DERSequence;
import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
public class Main {
public final static int BUFFER_SIZE = 117;
public static void decrypt(String originalZipFileName, String newZipFileName, String privateKeyFileName) throws Exception {
byte[] buffer = new byte[128];
ZipFile originalZipFile = new ZipFile(originalZipFileName);
ZipOutputStream newZipFile = new ZipOutputStream(new FileOutputStream(newZipFileName));
Enumeration<? extends ZipEntry> zipEntries = originalZipFile.entries();
String privateKey = getKeyString(privateKeyFileName);
PrivateKey key = makePrivateKey(privateKey);
Cipher cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.DECRYPT_MODE, key);
while(zipEntries.hasMoreElements()){
ZipEntry entry = zipEntries.nextElement();
ZipEntry copy = new ZipEntry(entry.getName());
newZipFile.putNextEntry(copy);
InputStream inputEntry = originalZipFile.getInputStream(entry);
while(inputEntry.read(buffer) != -1){
newZipFile.write(cipher.doFinal(buffer));
}
newZipFile.closeEntry();
inputEntry.close();
}
newZipFile.close();
originalZipFile.close();
}
public static void encrypt(String originalZipFileName, String newZipFileName, String publicKeyFileName) throws Exception{
byte[] buffer = new byte[BUFFER_SIZE];
ZipFile originalZipFile = new ZipFile(originalZipFileName);
ZipOutputStream newZipFile = new ZipOutputStream(new FileOutputStream(newZipFileName));
Enumeration<? extends ZipEntry> zipEntries = originalZipFile.entries();
String publicKey = getKeyString(publicKeyFileName);
PublicKey key = makePublicKey(publicKey);
Cipher cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.ENCRYPT_MODE, key);
while(zipEntries.hasMoreElements()){
ZipEntry entry = zipEntries.nextElement();
ZipEntry copy = new ZipEntry(entry.getName());
newZipFile.putNextEntry(copy);
InputStream inputEntry = originalZipFile.getInputStream(entry);
while(inputEntry.read(buffer) != -1){
newZipFile.write(cipher.doFinal(buffer));
}
newZipFile.closeEntry();
inputEntry.close();
}
newZipFile.close();
originalZipFile.close();
}
public static String getKeyString(String fileName){
String key = new String();
try {
BufferedReader buf = new BufferedReader(new FileReader(fileName));
key = buf.readLine();
} catch ( IOException e) {
e.printStackTrace();
}
return key.trim();
}
public static PublicKey makePublicKey(String stored) throws GeneralSecurityException {
byte[] data = Base64.getDecoder().decode(stored);
X509EncodedKeySpec spec = new X509EncodedKeySpec(data);
KeyFactory fact = KeyFactory.getInstance("RSA");
return fact.generatePublic(spec);
}
public static PrivateKey makePrivateKey(String stored) throws GeneralSecurityException, Exception {
/*byte[] data = Base64.getDecoder().decode(stored);
PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(data);
KeyFactory fact = KeyFactory.getInstance("RSA");
return fact.generatePrivate(spec);*/
byte[] data = Base64.getDecoder().decode(stored);
ASN1EncodableVector v = new ASN1EncodableVector();
v.add(new ASN1Integer(0));
ASN1EncodableVector v2 = new ASN1EncodableVector();
v2.add(new ASN1ObjectIdentifier(PKCSObjectIdentifiers.rsaEncryption.getId()));
v2.add(DERNull.INSTANCE);
v.add(new DERSequence(v2));
v.add(new DEROctetString(data));
ASN1Sequence seq = new DERSequence(v);
byte[] privKey = seq.getEncoded("DER");
PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(privKey);
KeyFactory fact = KeyFactory.getInstance("RSA");
PrivateKey key = fact.generatePrivate(spec);
return key;
}
public static void main(String[] args) throws Exception {
Scanner scan = new Scanner(System.in);
System.out.println("Enter type of operation:");
String line = scan.nextLine();
if(line.equals("encrypt")){
System.out.println("Enter name of original ZIP file:");
String originalZipFileName = scan.nextLine();
System.out.println("Enter name of new ZIP file:");
String newZipFileName = scan.nextLine();
System.out.println("Enter name of file containg public key:");
String publicKeyFileName = scan.nextLine();
encrypt(originalZipFileName, newZipFileName, publicKeyFileName);
}
if(line.equals("decrypt")){
System.out.println("Enter name of original ZIP file:");
String originalZipFileName = scan.nextLine();
System.out.println("Enter name of new ZIP file:");
String newZipFileName = scan.nextLine();
System.out.println("Enter name of file containg private key:");
String privateKeyFileName = scan.nextLine();
decrypt(originalZipFileName, newZipFileName, privateKeyFileName);
}
}
}
PS: Updated decrypt method. Still gives same error.
public static void decrypt(String originalZipFileName, String newZipFileName, String privateKeyFileName) throws Exception {
byte[] buffer = new byte[128];
ZipFile originalZipFile = new ZipFile(originalZipFileName);
ZipOutputStream newZipFile = new ZipOutputStream(new FileOutputStream(newZipFileName));
Enumeration<? extends ZipEntry> zipEntries = originalZipFile.entries();
String privateKey = getKeyString(privateKeyFileName);
PrivateKey key = makePrivateKey(privateKey);
Cipher cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.DECRYPT_MODE, key);
while(zipEntries.hasMoreElements()){
ZipEntry entry = zipEntries.nextElement();
ZipEntry copy = new ZipEntry(entry.getName());
newZipFile.putNextEntry(copy);
InputStream inputEntry = originalZipFile.getInputStream(entry);
while(inputEntry.read(buffer) != -1){
newZipFile.write(cipher.doFinal(buffer));
}
newZipFile.closeEntry();
inputEntry.close();
}
newZipFile.close();
originalZipFile.close();
}

Jozef is right.
When you create cipher with default parameters, it defaults to "RSA/ECB/PKCS1Padding". You should specify padding explicitly, if you don't like nasty surprises. Because other security providers might have different default parameters. And you never know in advance which security settings each specific JRE has.
So PKCS1 padding adds 11 bytes to your original data increasing it from 117 bytes to 128 bytes. You should take into account that these numbers are specific to 1024 bit RSA keys (which are marginally secure) and will be different for longer keys. Since you are loading the key from a file consider checking its length.
#Test
public void testPadding() throws Exception {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA");
keyGen.initialize(1024, random);
KeyPair keyPair = keyGen.generateKeyPair();
/* constant 117 is a public key size - 11 */
byte[] plaintext = new byte[117];
random.nextBytes(plaintext);
Cipher cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.ENCRYPT_MODE, keyPair.getPublic());
byte[] ciphertext = cipher.doFinal(plaintext);
System.out.println(plaintext.length + " becomes " + ciphertext.length);
}
This prints
117 becomes 128
And finally, consider using AES instead of RSA for file encryption.
So to fix the problem you need to use buffer of size public key length - 11 (117) for encryption and public key size (128) for decryption.
Change
outputFile.write(cipher.doFinal(buffer), 0, read);
to
outputFile.write(cipher.doFinal(buffer));
because buffer read is 117 bytes and size of doFinal result is 128 bytes.
Also you need to buffer input streams. When you are reading from file, it can be slow sometimes and then InputStream will read less data than buffer may contain. By using BufferedInputStream one ensures that there is enough data before read call returns. However, for decryption it's crucial to have the full block of data
InputStream inputEntry = new BufferedInputStream(originalZipFile.getInputStream(entry));

while((read = inputEntry.read(buffer)) != -1){
outputFile.write(cipher.doFinal(buffer), 0, read);
}
You have a problem here. read is the size of the plaintext that was read, not the ciphertext. You should remove the 2nd and 3rd parameters altogether.
It is also a waste of time and space to write the ciphertext to an intermediate file. Just write it straight to the zip stream.

The decrypt method's byte array should be 256 bytes in length as it is the default output size of the algorithm (The extra bytes result in this length). Change byte[] buffer = new byte[128]; to byte[] buffer = new byte[256];.

Java AES Padding Encryption Error While Transferring Video over TCP

I am currently doing a project in Java to transfer a video file over TCP from a server to a client. The idea is the server will continue running and listen to incoming connection. Once there is an incoming connection from a client, the server will automatically send a video file to the client. (As of now the IP and file name is hard coded). The idea is so that the file can be copied and played at the same time
It is used locally and will automatically turn on VLC from the receiving computer to play the file being transferred. I have done the transfer part with no issues. The only problem comes up when I try to encrypt/decrypt the file. The code I have are below
Runnable Thread FileTransfer Server
public class FileTransferServer {
public static void main(String[] args) throws Exception {
//Initialize Sockets
int i = 0;
ServerSocket ssock = new ServerSocket(6012);
while (true){
ClientConnection CC;
CC = new ClientConnection(ssock.accept());
Thread t = new Thread(CC);
t.start();
}
}
}
Server Java File
import java.io.BufferedInputStream;
import javax.crypto.Cipher;
import java.io.InputStream;
import java.io.OutputStream;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
public class ClientConnection implements Runnable
{
private Socket socketPort;
public ClientConnection (Socket socketPort)
{
this.socketPort = socketPort;
}
public void run()
{
try {
DataInputStream input = new DataInputStream(socketPort.getInputStream());
String videoName = input.readUTF();
// automatically get local ip
InetAddress IA = InetAddress.getByName("10.0.0.1");
String key = "Maryhadonecat111";
byte[] keyByte = key.getBytes("UTF-8");
System.out.println(keyByte);
System.out.println(keyByte.toString());
//Specify the file
File file = new File("D:\\Temp\\"+videoName);
FileInputStream fis = new FileInputStream(file);
BufferedInputStream bis = new BufferedInputStream(fis);
//Get socket's output stream
OutputStream os = socketPort.getOutputStream();
//Read File Contents into contents array
byte[] contents;
long fileLength = file.length();
long current = 0;
long start = System.nanoTime();
while(current!=fileLength){
int size = 1000000;
if(fileLength - current >= size)
current += size;
else{
size = (int)(fileLength - current);
current = fileLength;
}
contents = new byte[size];
bis.read(contents, 0, size);
//os.write(contents);
os.write(CryptoTest1.doEncrypt(contents,keyByte));
System.out.print("Sending file to "+ socketPort.getInetAddress().toString() +" " +(current*100)/fileLength+"% complete!\n");
}
os.flush();
//File transfer done. Close the socket connection!
socketPort.close();
// ssock.close();
System.out.println("File sent succesfully!");
} catch (Exception e)
{
System.out.println(e);
}
}
}
Client Java File
import java.io.BufferedOutputStream;
import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.DataOutputStream;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.Socket;
import javax.crypto.Cipher;
import java.io.InputStream;
import java.io.OutputStream;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
public class FileTransferClient {
public static void main(String[] args) throws Exception{
requestFile("10.0.0.1", "papa.avi");
}
public static void requestFile(String IP, String videoName) throws Exception{
//Initialize socket
Socket socket = new Socket(InetAddress.getByName(IP), 6012);
DataOutputStream output = new DataOutputStream( socket.getOutputStream());
output.writeUTF(videoName);
String key = "Maryhadonecat111";
byte[] keyByte = key.getBytes("UTF-8");
byte[] contents = new byte[1000000];
//Initialize the FileOutputStream to the output file's full path.
FileOutputStream fos = new FileOutputStream("D:\\Temp2\\"+videoName);
BufferedOutputStream bos = new BufferedOutputStream(fos);
InputStream is = socket.getInputStream();
System.out.println("Receiving File");
ProcessBuilder pb = new ProcessBuilder("C:\\Program Files (x86)\\VideoLAN\\VLC\\vlc.exe", "D:\\Temp2\\"+videoName);
Process start = pb.start();
//No of bytes read in one read() call
int bytesRead = 0;
while((bytesRead=is.read(contents))!=-1){
System.out.println("Bytes Received: " + bytesRead);
contents = (CryptoTest1.doDecrypt(contents,keyByte));
bos.write(contents, 0, bytesRead);
}
bos.flush();
socket.close();
System.out.println("File saved successfully!");
}
}
CryptoTest1 Java File
public class CryptoTest1
{
public static byte[] doEncrypt(byte[] msg, byte[] key) throws Exception {
//prepare key
SecretKeySpec secretKeySpec = new SecretKeySpec(key, "AES");
//prepare cipher
String cipherALG = "AES/CBC/PKCS5Padding"; // use your preferred algorithm
Cipher cipher = Cipher.getInstance(cipherALG);
String string = cipher.getAlgorithm();
//as iv (Initial Vector) is only required for CBC mode
if (string.contains("CBC")) {
//IvParameterSpec ivParameterSpec = new IvParameterSpec(iv);
IvParameterSpec ivParameterSpec = cipher.getParameters().getParameterSpec(IvParameterSpec.class);
cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec, ivParameterSpec);
} else {
cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec);
}
byte[] encMessage = cipher.doFinal(msg);
return encMessage;
}
public static byte[] doDecrypt(byte[] encMsgtoDec, byte[] key) throws Exception {
//prepare key
SecretKeySpec secretKeySpec = new SecretKeySpec(key, "AES");
//prepare cipher
String cipherALG = "AES/CBC/PKCS5Padding"; // use your preferred algorithm
Cipher cipher = Cipher.getInstance(cipherALG);
String string = cipher.getAlgorithm();
//as iv (Initial Vector) is only required for CBC mode
if (string.contains("CBC")) {
//IvParameterSpec ivParameterSpec = new IvParameterSpec(iv);
IvParameterSpec ivParameterSpec = cipher.getParameters().getParameterSpec(IvParameterSpec.class);
cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParameterSpec);
} else {
cipher.init(Cipher.DECRYPT_MODE, secretKeySpec);
}
byte[] decMsg = cipher.doFinal(encMsgtoDec);
return decMsg;
}
}
Issue
I seem to have no issue encrypting the file and it is being sent over. The issue is decrypting the file. I can't seem to get it working. I've tried a lot of times and most of the errors comes down to "Padding Exception"
I am currently using AES/CBC/PKCS5Padding but I have tried the following
AES/CBC/PKCS5Padding
AES/CBC/NoPadding
AES/ECB/PKCS5Padding
AES/ECB/NoPadding
If I use padding, i will get an exception
javax.crypto.BadPaddingException: Given final block not properly padded
If I do not use padding, I will get an exception of
javax.crypto.IllegalBlockSizeException: Input length not multiple of 16 bytes.
Some other exception I have encountered while I was tinkering
Missing Parameter
java.security.InvalidKeyException: Illegal key size
java.security.InvalidKeyException: Invalid AES key length: 64 bytes
I would like to ask if anyone of you would be willing to point me in a proper direction as to what I am doing wrong. I am still very new to Java, so please assume that I have very little knowledge.
I have searched Stackoverflow for a long time and most encryption questions here are on text files, not on an actual video. If the encryption method I am using is not suitable for video, please let me know if there is a better one.

The IV must be the same for both encryption and decryption. Generally a random IV is created on encryption and this must be provided to the decryption method. One way to handle this is to prefix the encrypted data with with the IV so on decryption it will be available.
Notes:
AES is a block cipher so the input must be an exact multiple of the block size, this is generally accomplished with padding.
A bad padding error generally means that either the key, IV or encrypted data is incorrect, not that the padding is incorrect. The padding is incorrect because the decryption failed.

Determining that decryption has occurred

I am developing a GUI based encryptor/decryptor based on AES-128 bit symmetric encryption.
My problem is that how to determine that decryption has not occurred and show a dialog box "Decryption Failed". The code I have written would always generate a file without a .enc extension regardless of the fact that it is still encrypted !
Hoping to get a answer as always from Stack Overflow's top notch programmers :)
Do note that the decryption process doesn't fail or throws exception ! It's just the fact that it generates a file that's still not decrypted. That we have to stop and that's what I meant !
Code here: (Sorry for bad indentation !)
import java.io.InputStream;
import java.io.OutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.Arrays;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import javax.crypto.spec.SecretKeySpec;
import java.security.SecureRandom;
import java.security.MessageDigest;
public class FileEncryptor{
private String algo;
private String path;
private String password;
public FileEncryptor(String algo,String path, String password) {
this.algo = algo; //setting algo
this.path = path;//setting file path
this.password = password;
}
public void encrypt() throws Exception{
SecureRandom padding = new SecureRandom();
byte[] salt = new byte[16];
padding.nextBytes(salt);
//generating key
byte k[] = password.getBytes();
MessageDigest sha = MessageDigest.getInstance("SHA-1");
k = sha.digest(k);
k = Arrays.copyOf(k, 16);
SecretKeySpec key = new SecretKeySpec(k,algo);
//creating and initialising cipher and cipher streams
Cipher encrypt = Cipher.getInstance(algo);
encrypt.init(Cipher.ENCRYPT_MODE, key);
//opening streams
FileOutputStream fos =new FileOutputStream(path+".enc");
try(FileInputStream fis =new FileInputStream(path)){
try(CipherOutputStream cout=new CipherOutputStream(fos, encrypt)){
copy(fis,cout);
}
}
}
public void decrypt() throws Exception{
SecureRandom padding = new SecureRandom();
byte[] salt = new byte[16];
padding.nextBytes(salt);
//generating same key
byte k[] = password.getBytes();
MessageDigest sha = MessageDigest.getInstance("SHA-1");
k = sha.digest(k);
k = Arrays.copyOf(k, 16);
SecretKeySpec key = new SecretKeySpec(k,algo);
//creating and initialising cipher and cipher streams
Cipher decrypt = Cipher.getInstance(algo);
decrypt.init(Cipher.DECRYPT_MODE, key);
//opening streams
FileInputStream fis = new FileInputStream(path);
try(CipherInputStream cin=new CipherInputStream(fis, decrypt)){
try(FileOutputStream fos =new FileOutputStream(path.substring(0,path.lastIndexOf(".")))){
copy(cin,fos);
}
}
}
private void copy(InputStream is,OutputStream os) throws Exception{
byte buf[] = new byte[4096]; //4K buffer set
int read = 0;
while((read = is.read(buf)) != -1) //reading
os.write(buf,0,read); //writing
}
public static void main (String[] args)throws Exception {
System.out.println("Enter Password: ");
new FileEncryptor("AES","sample.txt",new java.util.Scanner(System.in).nextLine()).encrypt();
new FileEncryptor("AES","sample.txt.enc",new java.util.Scanner(System.in).nextLine()).decrypt();
}
}

Without looking at the API calls, the decrypt methods should throw an exception if an error occurs. In your exception handler, you can set a flag that will allow you to display an error message. You can also delay the decrypted file creation till after successful decryption (or at least till after the first block has been successfully decrypted). If decryption then fails further along the line, you can delete the (essentially temporary) decrypted output file and display the error message.
[edit]
I slightly misunderstood the original post, so some suggestions to check for failed decryption (note that these are higher level than AES, so it might be specific to your application only):
Add a checksum to the plaintext data before encryption
Append other metadata (file size, user, date, etc) to the plaintext, and check for these when decrypting
Usually, a padding exception would occur on decryption - check for these (and any other giveaways)
Use PKI (public key infrastructure) functionality such as signatures (this is outside the scope of this answer, and possibly outside the scope of the problem you're trying to solve)

I suggest appending a constant, rather than a checksum, to your data before encryption, and verifying it after encryption.
And the encryption algorithm should use chaining, that means avoid ECB (see here why: http://bobnalice.wordpress.com/2009/01/28/friends-don%E2%80%99t-let-friends-use-ecb-mode-encryption).
Using a constant with chaining, is nearly as good as a checksum and much simpler.

Decrypt Content Encryption Key in JSON Web Encryption

I am using OpenSSL RSA1_5 for decrypting the CEK (Content Encryption Key).
My aim is to decrypt the JWK (JSON Web Key) by which I will be getting CEK, so by using CEK I can decrypt my ciphertext which is actually the encrypted data.
After using Base64Decode, JWE Header is
{"alg":"RSA1_5","enc":"A128CBC-HS256","typ":"JOSE"}
where "alg" is the algorithm used to decrypt the CEK. Please help me decrypting the CEK first, after this I need to decrypt the Cipher.
My Java class is:
package com.decryption;
import java.io.*;
import java.math.BigInteger;
import java.security.*;
import java.security.spec.*;
import java.security.interfaces.*;
import javax.crypto.*;
import javax.crypto.spec.*;
import javax.crypto.interfaces.*;
public class RSADecrypt
{
public RSADecrypt(String inFileName, String outFileName) {
try {
System.out.println("Inside TRY");
/* Get the encrypted message from file. */
FileInputStream cipherfile = new FileInputStream(inFileName);
byte[] ciphertext = new byte[cipherfile.available()];
cipherfile.read(ciphertext);
cipherfile.close();
System.out.println("Inside 1");
/* Get the private key from file. */
//PrivateKey privatekey = readPrivateKey("D://sso//mmdevnopass.key");
PrivateKey privatekey = readPrivateKey("D://sso//mmdevJWE.key");
System.out.println("Inside 2");
/* Create cipher for decryption. */
Cipher decrypt_cipher = Cipher.getInstance("RSA/ECB/PKCS1Padding");
decrypt_cipher.init(Cipher.DECRYPT_MODE, privatekey);
System.out.println("Inside 3");
/* Reconstruct the plaintext message. */
byte[] plaintext = decrypt_cipher.doFinal(ciphertext);
FileOutputStream plainfile = new FileOutputStream(outFileName);
plainfile.write(plaintext);
plainfile.close();
} catch (Exception e) {
System.out.println("catch1");
e.printStackTrace();
}
}
public static PrivateKey readPrivateKey(String filename) throws Exception {
System.out.println("readPrivateKey()");
FileInputStream file = new FileInputStream(filename);
byte[] bytes = new byte[file.available()];
file.read(bytes);
file.close();
System.out.println("readPrivateKey() 1");
PKCS8EncodedKeySpec privspec = new PKCS8EncodedKeySpec(bytes);
// X509EncodedKeySpec privspec= new X509EncodedKeySpec(bytes);
//RSAPrivateKeySpec privspec = new RSAPrivateKeySpec(modulus, privateExponent)
System.out.println("readPrivateKey() 2");
KeyFactory factory = KeyFactory.getInstance("RSA");
System.out.println("readPrivateKey() 3");
PrivateKey privkey = factory.generatePrivate(privspec);
System.out.println("readPrivateKey() 4");
return privkey;
}
public static void main(String[] arg) {
/*if (arg.length != 2) {
System.err.println("Usage: java RSADecrypt <src file> <dest file>");
} else {*/
System.out.println("Welcome");
String inFileName="D://sso//myJEK.txt";
String outFileName="D://sso//out.txt";
new RSADecrypt(inFileName,outFileName);
// }
}
}
I am getting output as
Welcome
Inside TRY
Inside 1
readPrivateKey()
readPrivateKey() 1
readPrivateKey() 2
readPrivateKey() 3
java.security.spec.InvalidKeySpecException: java.security.InvalidKeyException: invalid key format
at sun.security.rsa.RSAKeyFactory.engineGeneratePrivate(RSAKeyFactory.java:175)
at java.security.KeyFactory.generatePrivate(KeyFactory.java:322)
at com.decryption.RSADecrypt.readPrivateKey(RSADecrypt.java:85)
at com.decryption.RSADecrypt.<init>(RSADecrypt.java:46)
at com.decryption.RSADecrypt.main(RSADecrypt.java:102)
Caused by: java.security.InvalidKeyException: invalid key format
at sun.security.pkcs.PKCS8Key.decode(PKCS8Key.java:324)
at sun.security.pkcs.PKCS8Key.decode(PKCS8Key.java:350)
at sun.security.rsa.RSAPrivateCrtKeyImpl.<init>(RSAPrivateCrtKeyImpl.java:74)
at sun.security.rsa.RSAPrivateCrtKeyImpl.newKey(RSAPrivateCrtKeyImpl.java:58)
at sun.security.rsa.RSAKeyFactory.generatePrivate(RSAKeyFactory.java:274)
at sun.security.rsa.RSAKeyFactory.engineGeneratePrivate(RSAKeyFactory.java:171)
... 4 more
catch1
Please help me to decrypt the CEK and solve this exception.

Your problem is caused by your private key file. Firstly, your method of reading the bytes is error-prone:
FileInputStream file = new FileInputStream(filename);
byte[] bytes = new byte[file.available()];
file.read(bytes);
file.close();
This may not read the entire file. The available() method does not indicate how many bytes are in the file. Please search for a better way of reading this file (perhaps from this question: File to byte[] in Java).
Once this is fixed, you may still have errors unless your file is a DER-encoded PKCS #8 object. A common mistake is to try and use a PEM-encoded file (e.g. containing ----- BEGIN PRIVATE KEY ---- headers and base64-encoded data).

Why is RC4 not able to handle large amount of encrypted data?

I have following code that decrypts a file.
package encryption;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.Security;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
public class Decrypter {
private static final String PASSWORD = "t_9Y#i#eT[h3}-7!";
private static final String KEY_ALGORITHM = "PBEWithMD5AndDES";
private static final String CIPHER_ALGORITHM = "RC4"; //Using Salsa20 or HC256 solves the problem
private static final String PROVIDER = "BC";
public static void main(String[] args) throws Exception {
Security.addProvider(new BouncyCastleProvider());
File inputFile = new File(args[0]);
File outputFile = new File(args[1]);
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(KEY_ALGORITHM);
SecretKey key = keyFactory.generateSecret(new PBEKeySpec(PASSWORD.toCharArray()));
Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM);
cipher.init(Cipher.DECRYPT_MODE, key);
InputStream inputStream = new FileInputStream(inputFile);
OutputStream outputStream = new FileOutputStream(outputFile);
CipherInputStream cipherInputStream = new CipherInputStream(inputStream, cipher);
byte []byteBuffer = new byte[(int)inputFile.length()];
cipherInputStream.read(byteBuffer);
outputStream.write(byteBuffer); //Only 512bytes of decrypted data is written to file, the rest becomes null
outputStream.close();
}
}
My question is what am I doing wrong? Why doesn't RC4 decrypt a block of size more than 512 bytes.

RC4 is a stream cipher, so it can decode any amount of data. Your issue is that InputStreams are not read in massive chunks. Normally you loop around the read call until there is no more data left to read and use a small buffer. See the documentation of read().
This could be implemented as
while(true) {
int numRead = cipherInputStream.read(byteBuffer);
if(numRead == -1)
break;
outputStream.write(byteBuffer, 0, numRead);
}

#Michael Lowman has the correct answer, but I'd thought I'd show another way just to advertise a feature of the DataInputStream class.
You can the read-it-all-in-one-go behavior, like a perl slurp, by using the DataInputStream.readFully() method. In your example, you can read in the bytes with this method, then write them out and decrypt by using as CipherOutputStream instead of a CipherInputStream.
Consider the following fragment as an example:
byte[] byteBuffer = new byte[(int) inputFile.length()];
DataInputStream dis = new DataInputStream(inputStream);
dis.readFully(byteBuffer);
dis.close();
CipherOutputStream cos = new CipherOutputStream(outputStream, cipher);
cos.write(byteBuffer);
cos.close();

InputStream.read only returns a certain amount of data, you are supposed to loop until the stream is empty. However I suggest you use commons-io's org.apache.commons.io.FileUtils.copyInputStreamToFile(InputStream, File) to copy the streams rather than rolling your own...