Suppose,
I am splitting 2590400 KB (approx 2.5 GB) file in 30 parts.
It will produce 30 files with size of 86347 KB.
Which seems correct, 2590400/30 = 86346.66666667
Now if I merge all the parts (30) again it is producing the file of 3453873 KB file, which should be 2590410 KB.
Can anyone help me why this difference is there? I am using below code for merge and split files.
SplitFile.java
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
/**
* #author vishal.zanzrukia
*
*/
public class SplitFile {
public static final String INPUT_FILE = "D:\\me\\projects\\input\\file\\path.txt";
public static final int NUMBER_OF_OUTPUT_FILES = 30;
public static final String FILE_SUFFIX = ".txt";
/**
* split file
*
* #throws Exception
*/
static void splitFile() throws Exception{
File inputFile = new File(INPUT_FILE + "_Splits");
inputFile.mkdir();
RandomAccessFile raf = new RandomAccessFile(INPUT_FILE, "r");
long sourceSize = raf.length();
long bytesPerSplit = sourceSize / NUMBER_OF_OUTPUT_FILES;
long remainingBytes = sourceSize % NUMBER_OF_OUTPUT_FILES;
int maxReadBufferSize = 8 * 1024; // 8KB
for (int destIx = 1; destIx <= NUMBER_OF_OUTPUT_FILES; destIx++) {
BufferedOutputStream bw = new BufferedOutputStream(new FileOutputStream(INPUT_FILE + "_Splits\\split." + destIx + FILE_SUFFIX));
if (bytesPerSplit > maxReadBufferSize) {
long numReads = bytesPerSplit / maxReadBufferSize;
long numRemainingRead = bytesPerSplit % maxReadBufferSize;
for (int i = 0; i < numReads; i++) {
readWrite(raf, bw, maxReadBufferSize);
}
if (numRemainingRead > 0) {
readWrite(raf, bw, numRemainingRead);
}
} else {
readWrite(raf, bw, bytesPerSplit);
}
bw.close();
}
if (remainingBytes > 0) {
BufferedOutputStream bw = new BufferedOutputStream(new FileOutputStream("split." + NUMBER_OF_OUTPUT_FILES + 1));
readWrite(raf, bw, remainingBytes);
bw.close();
}
raf.close();
}
/**
* join file
*
* #throws Exception
*/
static void joinFiles() throws Exception{
int maxReadBufferSize = 8 * 1024;
BufferedOutputStream bw = new BufferedOutputStream(new FileOutputStream(INPUT_FILE + "_Splits\\fullJoin" + FILE_SUFFIX));
File inputFileDir = new File(INPUT_FILE + "_Splits");
RandomAccessFile raf = null;
if(inputFileDir.isDirectory()){
for(File file : inputFileDir.listFiles()){
raf = new RandomAccessFile(file, "r");
long numReads = raf.length() / maxReadBufferSize;
long numRemainingRead = raf.length() % maxReadBufferSize;
for (int i = 0; i < numReads; i++) {
readWrite(raf, bw, maxReadBufferSize);
}
if (numRemainingRead > 0) {
readWrite(raf, bw, numRemainingRead);
}
raf.close();
}
}
bw.close();
}
public static void mergeFiles() {
File[] files = new File[NUMBER_OF_OUTPUT_FILES];
for(int i=1;i<=NUMBER_OF_OUTPUT_FILES;i++){
files[i-1] = new File(INPUT_FILE + "_Splits\\split."+i+FILE_SUFFIX);
}
String mergedFilePath = INPUT_FILE + "_Splits\\fullJoin" + FILE_SUFFIX;
File mergedFile = new File(mergedFilePath);
mergeFiles(files, mergedFile);
}
public static void mergeFiles(File[] files, File mergedFile) {
FileWriter fstream = null;
BufferedWriter out = null;
try {
fstream = new FileWriter(mergedFile, true);
out = new BufferedWriter(fstream);
} catch (IOException e1) {
e1.printStackTrace();
}
for (File f : files) {
System.out.println("merging: " + f.getName());
FileInputStream fis;
try {
fis = new FileInputStream(f);
BufferedReader in = new BufferedReader(new InputStreamReader(fis));
String aLine;
while ((aLine = in.readLine()) != null) {
out.write(aLine);
out.newLine();
}
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
// splitFile();
mergeFiles();
}
static void readWrite(RandomAccessFile raf, BufferedOutputStream bw, long numBytes) throws IOException {
byte[] buf = new byte[(int) numBytes];
int val = raf.read(buf);
if (val != -1) {
bw.write(buf);
}
}
}
Use your joinFiles method: don't try to read a file by line-by-line using a Reader if you want to keep it exactly like it was, because line endings may differ by platform.
Instead read them as a binary file using an InputStream or RandomAccessFile and write using an OutputStream.
The only problem in your joinFiles method is that it used File.listFiles() which makes no guarantees about the order in which the files are returned.
I combined your mergeFiles() code with joinFiles() to make this work (remember to invoke joinFiles() instead of mergeFiles() from your main method)
static void joinFiles(File[] files) throws Exception {
int maxReadBufferSize = 8 * 1024;
BufferedOutputStream bw = new BufferedOutputStream(new FileOutputStream(INPUT_FILE + "_Splits\\fullJoin"
+ FILE_SUFFIX));
RandomAccessFile raf = null;
for (File file : files) {
raf = new RandomAccessFile(file, "r");
long numReads = raf.length() / maxReadBufferSize;
long numRemainingRead = raf.length() % maxReadBufferSize;
for (int i = 0; i < numReads; i++) {
readWrite(raf, bw, maxReadBufferSize);
}
if (numRemainingRead > 0) {
readWrite(raf, bw, numRemainingRead);
}
raf.close();
}
bw.close();
}
public static void joinFiles() throws Exception {
File[] files = new File[NUMBER_OF_OUTPUT_FILES];
for (int i = 1; i <= NUMBER_OF_OUTPUT_FILES; i++) {
files[i - 1] = new File(INPUT_FILE + "_Splits\\split." + i + FILE_SUFFIX);
}
joinFiles(files);
}
The problem is the very last line of code:
static void readWrite(RandomAccessFile raf, BufferedOutputStream bw, long numBytes) throws IOException {
byte[] buf = new byte[(int) numBytes];
int val = raf.read(buf);
if (val != -1) {
bw.write(buf);
}
}
When you write, you write back numBytes of data, but the read function has usefully returned:
the total number of bytes read into the buffer, or -1 if there is no more data because the end of this file has been reached.
Therefore, your fix is to use a different write:
bw.write(buf, 0 val);
I am completing a lab assignment for school and get this error when I compile. The program runs fine, bit would like to fix what is causing the error. The program code and the complete error is below. Thanks as always!
Note: Recompile with -Xlint:unchecked for details.
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package ie.moguntia.webcrawler;
import java.net.*;
import java.io.*;
import java.util.*;
/**
*
* #author Cong
*/
public class SaveURL
{
/**
* Opens a buffered stream on the url and copies the contents to writer
*/
public static void saveURL(URL url, Writer writer)
throws IOException {
BufferedInputStream in = new BufferedInputStream(url.openStream());
for (int c = in.read(); c != -1; c = in.read()) {
writer.write(c);
}
}
/**
* Opens a buffered stream on the url and copies the contents to OutputStream
*/
public static void saveURL(URL url, OutputStream os)
throws IOException {
InputStream is = url.openStream();
byte[] buf = new byte[1048576];
int n = is.read(buf);
while (n != -1) {
os.write(buf, 0, n);
n = is.read(buf);
}
}
/**
* Writes the contents of the url to a string by calling saveURL with a
* string writer as argument
*/
public static String getURL(URL url)
throws IOException {
StringWriter sw = new StringWriter();
saveURL(url, sw);
return sw.toString();
}
/**
* Writes the contents of the url to a new file by calling saveURL with
* a file writer as argument
*/
public static void writeURLtoFile(URL url, String filename)
throws IOException {
// FileWriter writer = new FileWriter(filename);
// saveURL(url, writer);
// writer.close();
FileOutputStream os = new FileOutputStream(filename);
saveURL(url, os);
os.close();
}
/**
* Extract links directly from a URL by calling extractLinks(getURL())
*/
public static Vector extractLinks(URL url)
throws IOException {
return extractLinks(getURL(url));
}
public static Map extractLinksWithText(URL url)
throws IOException {
return extractLinksWithText(getURL(url));
}
/**
* Extract links from a html page given as a raw and a lower case string
* In order to avoid the possible double conversion from mixed to lower case
* a second method is provided, where the conversion is done externally.
*/
public static Vector extractLinks(String rawPage, String page) {
int index = 0;
Vector links = new Vector();
while ((index = page.indexOf("<a ", index)) != -1)
{
if ((index = page.indexOf("href", index)) == -1) break;
if ((index = page.indexOf("=", index)) == -1) break;
String remaining = rawPage.substring(++index);
StringTokenizer st
= new StringTokenizer(remaining, "\t\n\r\"'>#");
String strLink = st.nextToken();
if (! links.contains(strLink)) links.add(strLink);
}
return links;
}
/**
* Extract links (key) with link text (value)
* Note that due to the nature of a Map only one link text is returned per
* URL, even if a link occurs multiple times with different texts.
*/
public static Map extractLinksWithText(String rawPage, String page) {
int index = 0;
Map links = new HashMap();
while ((index = page.indexOf("<a ", index)) != -1)
{
int tagEnd = page.indexOf(">", index);
if ((index = page.indexOf("href", index)) == -1) break;
if ((index = page.indexOf("=", index)) == -1) break;
int endTag = page.indexOf("</a", index);
String remaining = rawPage.substring(++index);
StringTokenizer st
= new StringTokenizer(remaining, "\t\n\r\"'>#");
String strLink = st.nextToken();
String strText = "";
if (tagEnd != -1 && tagEnd + 1 <= endTag) {
strText = rawPage.substring(tagEnd + 1, endTag);
}
strText = strText.replaceAll("\\s+", " ");
links.put(strLink, strText);
}
return links;
}
/**
* Extract links from a html page given as a String
* The return value is a vector of strings. This method does neither check
* the validity of its results nor does it care about html comments, so
* links that are commented out are also retrieved.
*/
public static Vector extractLinks(String rawPage) {
return extractLinks(rawPage, rawPage.toLowerCase().replaceAll("\\s", " "));
}
public static Map extractLinksWithText(String rawPage) {
return extractLinksWithText(rawPage, rawPage.toLowerCase().replaceAll("\\s", " "));
}
/**
* As a standalone program this class is capable of copying a url to a file
*/
public static void main(String[] args) {
try {
if (args.length == 1) {
URL url = new URL(args[0]);
System.out.println("Content-Type: " +
url.openConnection().getContentType());
// Vector links = extractLinks(url);
// for (int n = 0; n < links.size(); n++) {
// System.out.println((String) links.elementAt(n));
// }
Set links = extractLinksWithText(url).entrySet();
Iterator it = links.iterator();
while (it.hasNext()) {
Map.Entry en = (Map.Entry) it.next();
String strLink = (String) en.getKey();
String strText = (String) en.getValue();
System.out.println(strLink + " \"" + strText + "\" ");
}
return;
} else if (args.length == 2) {
writeURLtoFile(new URL(args[0]), args[1]);
return;
}
} catch (Exception e) {
System.err.println("An error occured: ");
e.printStackTrace();
// System.err.println(e.toString());
}
// Display usage information
// (If the program had done anything sensible, we wouldn't be here.)
System.err.println("Usage: java SaveURL <url> [<file>]");
System.err.println("Saves a URL to a file.");
System.err.println("If no file is given, extracts hyperlinks on url to console.");
}
}
You are using the raw (i.e. non-generic) forms of several classes that have generic type parameters, including
Map
HashMap
Vector
Iterator
Set
Map.Entry
Use the generic forms of these classes by supplying appropriate type parameters.
So i have error when i'm trying to unpack my zip file on pc.
7zip error is - An attempt to move the file pointer before the beginning of the file.
UPD - error occurs after sending to ftp, before sending the archive is all right!
for work with ftp i'm use ClientFTP lib.
at first , on my device, i create a text file with gps coordinates, then i'm try to zip it and upload to ftp
then download to pc and unpack.
I have tried many ways to create an archive
code below to create a text file
File file;
try{
file = new File(path, "locations.lft");
if (file.exists())
file.delete();
file.createNewFile();
}catch (Exception e){
try{ftp.quit();}catch(Exception E){}
return;
}
BufferedWriter BW = null;
try{
BW = new BufferedWriter(new FileWriter(file));
}catch(Exception e){
try{ftp.quit();}catch(Exception E){}
return;
}
Cursor row = db.Query("locations",null,"date_time BETWEEN " + ToBeggingOfDay(date).getTime() + " AND " +
ToEndOfDay(date).getTime());
try{
while (row.moveToNext()){
String line = "" + row.getString(row.getColumnIndex("latitude")) + "~" + row.getString(row.getColumnIndex("longtitude")) + "~" + DateFormat.format("yyyy-MM-dd-kk-mm-ss", new Date( row.getLong(row.getColumnIndex("date_time")) )) + "~" + "0~~~~"+row.getString(row.getColumnIndex("speed"));
BW.write(line);
BW.newLine();
}
}catch (Exception e){
SendMsg("ERROR", "Can't write in file (" + e.getLocalizedMessage() + ")");
try{ftp.quit();}catch(Exception E){}
try{BW.close();}catch(Exception E){}
return;
}finally{
row.close();
try{BW.close();}catch(Exception E){}
}
code for pack zip file
public class ZipUtil {
/**
* A constants for buffer size used to read/write data
*/
private static final int BUFFER_SIZE = 4096;
/**
* Compresses a collection of files to a destination zip file
* #param listFiles A collection of files and directories
* #param destZipFile The path of the destination zip file
* #throws FileNotFoundException
* #throws IOException
*/
public void compressFiles(List<File> listFiles, String destZipFile) throws FileNotFoundException, IOException {
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(destZipFile));
for (File file : listFiles) {
if (file.isDirectory()) {
addFolderToZip(file, file.getName(), zos);
} else {
addFileToZip(file, zos);
}
}
zos.flush();
zos.close();
}
/**
* Adds a directory to the current zip output stream
* #param folder the directory to be added
* #param parentFolder the path of parent directory
* #param zos the current zip output stream
* #throws FileNotFoundException
* #throws IOException
*/
private void addFolderToZip(File folder, String parentFolder,
ZipOutputStream zos) throws FileNotFoundException, IOException {
for (File file : folder.listFiles()) {
if (file.isDirectory()) {
addFolderToZip(file, parentFolder + "/" + file.getName(), zos);
continue;
}
zos.putNextEntry(new ZipEntry(parentFolder + "/" + file.getName()));
BufferedInputStream bis = new BufferedInputStream(
new FileInputStream(file));
long bytesRead = 0;
byte[] bytesIn = new byte[BUFFER_SIZE];
int read = 0;
while ((read = bis.read(bytesIn)) != -1) {
zos.write(bytesIn, 0, read);
bytesRead += read;
}
zos.closeEntry();
}
}
/**
* Adds a file to the current zip output stream
* #param file the file to be added
* #param zos the current zip output stream
* #throws FileNotFoundException
* #throws IOException
*/
private void addFileToZip(File file, ZipOutputStream zos)
throws FileNotFoundException, IOException {
zos.putNextEntry(new ZipEntry(file.getName()));
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(
file));
long bytesRead = 0;
byte[] bytesIn = new byte[BUFFER_SIZE];
int read = 0;
while ((read = bis.read(bytesIn)) != -1) {
zos.write(bytesIn, 0, read);
bytesRead += read;
}
zos.closeEntry();
}
}
at the end of the code sent to the FTP
BufferedInputStream bis = null;
try{
bis = new BufferedInputStream(new FileInputStream(file));
if (!ftp.storeFile(FTPFileName, bis)){
SendMsg("ERROR", "Can't write in file");
return;
}
}catch (Exception e) {
SendMsg("ERROR", "Can't write in file");
return;
}finally{
try{bis.close();}catch (Exception E) {}
file.delete();
}
Example
try {
ZipUtil zipper = new ZipUtil();
File directoryToZip = new File(path+"locations.lft");
String zipFilePath = path + "1.zip";
List<File> listFiles = new ArrayList<File>(1);
listFiles.add(directoryToZip);
zipper.compressFiles(listFiles, zipFilePath);
}catch(Exception e){
e.printStackTrace();
}
file = new File(path, "1.zip");
Sorry for my bad english , hope you have some ideas why i get this error
I am using the following code on the client side to upload to the server
public class UploaderExample{
private static final String Boundary = "--7d021a37605f0";
public void upload(URL url, List<File> files) throws Exception
{
HttpURLConnection theUrlConnection = (HttpURLConnection) url.openConnection();
theUrlConnection.setDoOutput(true);
theUrlConnection.setDoInput(true);
theUrlConnection.setUseCaches(false);
theUrlConnection.setChunkedStreamingMode(1024);
theUrlConnection.setRequestProperty("Content-Type", "multipart/form-data; boundary="
+ Boundary);
DataOutputStream httpOut = new DataOutputStream(theUrlConnection.getOutputStream());
for (int i = 0; i < files.size(); i++)
{
File f = files.get(i);
String str = "--" + Boundary + "\r\n"
+ "Content-Disposition: form-data;name=\"file" + i + "\"; filename=\"" + f.getName() + "\"\r\n"
+ "Content-Type: image/png\r\n"
+ "\r\n";
httpOut.write(str.getBytes());
FileInputStream uploadFileReader = new FileInputStream(f);
int numBytesToRead = 1024;
int availableBytesToRead;
while ((availableBytesToRead = uploadFileReader.available()) > 0)
{
byte[] bufferBytesRead;
bufferBytesRead = availableBytesToRead >= numBytesToRead ? new byte[numBytesToRead]
: new byte[availableBytesToRead];
uploadFileReader.read(bufferBytesRead);
httpOut.write(bufferBytesRead);
httpOut.flush();
}
httpOut.write(("--" + Boundary + "--\r\n").getBytes());
}
httpOut.write(("--" + Boundary + "--\r\n").getBytes());
httpOut.flush();
httpOut.close();
// read & parse the response
InputStream is = theUrlConnection.getInputStream();
StringBuilder response = new StringBuilder();
byte[] respBuffer = new byte[4096];
while (is.read(respBuffer) >= 0)
{
response.append(new String(respBuffer).trim());
}
is.close();
System.out.println(response.toString());
}
public static void main(String[] args) throws Exception
{
List<File> list = new ArrayList<File>();
list.add(new File("C:\\square.png"));
list.add(new File("C:\\narrow.png"));
UploaderExample uploader = new UploaderExample();
uploader.upload(new URL("http://systemout.com/upload.php"), list);
}
}
I have tried writing the servlet that receives the image file and saves it to a folder on the server....but have failed miserably...This is part of an academic project i need to submit as part of my degree....Please Help!!!
I want help ...can someone guide me on how the servlet will be written....
I tried the following:
response.setContentType("text/html;charset=UTF-8");
PrintWriter out = response.getWriter();
try {
InputStream input = null;
OutputStream output = null;
try {
input = request.getInputStream();
output = new FileOutputStream("C:\\temp\\file.png");
byte[] buffer = new byte[10240];
for (int length = 0; (length = input.read(buffer)) > 0 ; ) {
output.write(buffer, 0, length);
}
}
catch(Exception e){
out.println(e.getMessage());
}
finally {
if (output != null) {
output.close();
}
if (input != null) {
input.close();
}
}
out.println("Success");
}
catch(Exception e){
out.println(e.getMessage());
}
finally {
out.close();
}
}
I went ahead and tried the fileupload from apache.org....and wrote the following servlet code:
protected void processRequest(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
response.setContentType("text/html;charset=UTF-8");
PrintWriter out = response.getWriter();
try {
out.println(1);
boolean isMultipart = ServletFileUpload.isMultipartContent(request);
if (isMultipart) {
// Create a factory for disk-based file items
FileItemFactory factory = new DiskFileItemFactory();
// Create a new file upload handler
ServletFileUpload upload = new ServletFileUpload(factory);
// Parse the request
List /* FileItem */ items = upload.parseRequest(request);
// Process the uploaded items
Iterator iter = items.iterator();
while (iter.hasNext()) {
FileItem item = (FileItem) iter.next();
if (item.isFormField()) {
//processFormField(item);
} else {
//processUploadedFile(item);
String fieldName = item.getFieldName();
String fileName = item.getName();
String contentType = item.getContentType();
boolean isInMemory = item.isInMemory();
long sizeInBytes = item.getSize();
//write to file
File uploadedFile = new File("C:\\temp\\image.png");
item.write(uploadedFile);
out.println("Sucess!");
}
}
} else {
out.println("Invalid Content!");
}
} catch (Exception e) {
out.println(e.getMessage());
} finally {
out.close();
}
}
However i am still confused on how to write the multipart code on the client side...the one i posted above is not working with my servlet implementation.....help please....some links where i can learn writing posting multipart form from java desktop app would be useful
So here's my recommendation: don't write this code yourself! Use http://commons.apache.org/fileupload/ instead. It will save you a lot of headaches, and you'll be up and running quite quickly. I'm pretty sure that problem is that the InputStream contains the multi-part boundaries, and is thus not a valid image.
Here's another observation: since you're not doing any transformations on the image, there's no need to read and write the image bytes using ImageIO. You're better off writing the bytes straight from the InputStream to the file.
I have an InputStream from which I'm reading characters. I would like multiple readers to access this InputStream. It seems that a reasonable way to achieve this is to write incoming data to a StringBuffer or StringBuilder, and have the multiple readers read that. Unfortunately, StringBufferInputStream is deprecated. StringReader reads a string, not a mutable object that's continuously being updated. What are my options? Write my own?
Note: My other answer is more general (and better in my opinion).
As noted by #dimo414, the answer below requires the first reader to always be ahead of the second reader. If this is indeed the case for you, then this answer might still be preferable since it builds upon standard classes.
To create two readers that read independently from the same source, you'll have to make sure they don't consume data from the same stream.
This can be achieved by combining TeeInputStream from Apache Commons and a PipedInputStream and PipedOutputStream as follows:
import java.io.*;
import org.apache.commons.io.input.TeeInputStream;
class Test {
public static void main(String[] args) throws IOException {
// Create the source input stream.
InputStream is = new FileInputStream("filename.txt");
// Create a piped input stream for one of the readers.
PipedInputStream in = new PipedInputStream();
// Create a tee-splitter for the other reader.
TeeInputStream tee = new TeeInputStream(is, new PipedOutputStream(in));
// Create the two buffered readers.
BufferedReader br1 = new BufferedReader(new InputStreamReader(tee));
BufferedReader br2 = new BufferedReader(new InputStreamReader(in));
// Do some interleaved reads from them.
System.out.println("One line from br1:");
System.out.println(br1.readLine());
System.out.println();
System.out.println("Two lines from br2:");
System.out.println(br2.readLine());
System.out.println(br2.readLine());
System.out.println();
System.out.println("One line from br1:");
System.out.println(br1.readLine());
System.out.println();
}
}
Output:
One line from br1:
Line1: Lorem ipsum dolor sit amet, <-- reading from start
Two lines from br2:
Line1: Lorem ipsum dolor sit amet, <-- reading from start
Line2: consectetur adipisicing elit,
One line from br1:
Line2: consectetur adipisicing elit, <-- resumes on line 2
As you've probably noted, once you've read a byte from an input stream, it's gone forever (unless you've saved it somewhere yourself).
The solution below does save the bytes until all subscribing input streams have read it.
It works as follows:
// Create a SplittableInputStream from the originalStream
SplittableInputStream is = new SplittableInputStream(originalStream);
// Fork this to get more input streams reading independently from originalStream
SplittableInputStream is2 = is.split();
SplittableInputStream is3 = is.split();
Each time is is split() it will yield a new InputStream that will read the bytes from the point where is was split.
The SplittableInputStream looks as follows (copy'n'paste away!):
class SplittableInputStream extends InputStream {
// Almost an input stream: The read-method takes an id.
static class MultiplexedSource {
static int MIN_BUF = 4096;
// Underlying source
private InputStream source;
// Read positions of each SplittableInputStream
private List<Integer> readPositions = new ArrayList<>();
// Data to be read by the SplittableInputStreams
int[] buffer = new int[MIN_BUF];
// Last valid position in buffer
int writePosition = 0;
public MultiplexedSource(InputStream source) {
this.source = source;
}
// Add a multiplexed reader. Return new reader id.
int addSource(int splitId) {
readPositions.add(splitId == -1 ? 0 : readPositions.get(splitId));
return readPositions.size() - 1;
}
// Make room for more data (and drop data that has been read by
// all readers)
private void readjustBuffer() {
int from = Collections.min(readPositions);
int to = Collections.max(readPositions);
int newLength = Math.max((to - from) * 2, MIN_BUF);
int[] newBuf = new int[newLength];
System.arraycopy(buffer, from, newBuf, 0, to - from);
for (int i = 0; i < readPositions.size(); i++)
readPositions.set(i, readPositions.get(i) - from);
writePosition -= from;
buffer = newBuf;
}
// Read and advance position for given reader
public int read(int readerId) throws IOException {
// Enough data in buffer?
if (readPositions.get(readerId) >= writePosition) {
readjustBuffer();
buffer[writePosition++] = source.read();
}
int pos = readPositions.get(readerId);
int b = buffer[pos];
if (b != -1)
readPositions.set(readerId, pos + 1);
return b;
}
}
// Non-root fields
MultiplexedSource multiSource;
int myId;
// Public constructor: Used for first SplittableInputStream
public SplittableInputStream(InputStream source) {
multiSource = new MultiplexedSource(source);
myId = multiSource.addSource(-1);
}
// Private constructor: Used in split()
private SplittableInputStream(MultiplexedSource multiSource, int splitId) {
this.multiSource = multiSource;
myId = multiSource.addSource(splitId);
}
// Returns a new InputStream that will read bytes from this position
// onwards.
public SplittableInputStream split() {
return new SplittableInputStream(multiSource, myId);
}
#Override
public int read() throws IOException {
return multiSource.read(myId);
}
}
Finally, a demo:
String str = "Lorem ipsum\ndolor sit\namet\n";
InputStream is = new ByteArrayInputStream(str.getBytes("UTF-8"));
// Create the two buffered readers.
SplittableInputStream is1 = new SplittableInputStream(is);
SplittableInputStream is2 = is1.split();
BufferedReader br1 = new BufferedReader(new InputStreamReader(is1));
BufferedReader br2 = new BufferedReader(new InputStreamReader(is2));
// Do some interleaved reads from them.
System.out.println("One line from br1:");
System.out.println(br1.readLine());
System.out.println();
System.out.println("Two lines from br2:");
System.out.println(br2.readLine());
System.out.println(br2.readLine());
System.out.println();
System.out.println("One line from br1:");
System.out.println(br1.readLine());
System.out.println();
Output:
One line from br1:
Lorem ipsum
Two lines from br2:
Lorem ipsum
dolor sit
One line from br1:
dolor sit
Use TeeInputStream to copy all the bytes read from InputStream to secondary OutputStream, e.g. ByteArrayOutputStream.
Input stream work like this: once you read a portion from it, it's gone forever. You can't go back and re-read it. what you could do is something like this:
class InputStreamSplitter {
InputStreamSplitter(InputStream toReadFrom) {
this.reader = new InputStreamReader(toReadFrom);
}
void addListener(Listener l) {
this.listeners.add(l);
}
void work() {
String line = this.reader.readLine();
while(line != null) {
for(Listener l : this.listeners) {
l.processLine(line);
}
}
}
}
interface Listener {
processLine(String line);
}
have all interested parties implement Listener and add them to InputStreamSplitter
Instead of using StringWriter/StringBufferInputStream, write your original InputStream to a ByteArrayOutputStream. Once you've finished reading from the original InputStream, pass the byte array returned from ByteArrayOutputStream.toByteArray to a ByteArrayInputStream. Use this InputStream as the InputStream of choice for passing around other things that need to read from it.
Essentially, all you'd be doing here is storing the contents of the original InputStream into a byte[] cache in memory as you tried to do originally with StringWriter/StringBufferInputStream.
Here's another way to read from two streams independently, without presuming one is ahead of the other, but with standard classes. It does, however, eagerly read from the underlying input stream in the background, which may be undesirable, depending on your application.
public static void main(String[] args) throws IOException {
// Create the source input stream.
InputStream is = new ByteArrayInputStream("line1\nline2\nline3".getBytes());
// Create a piped input stream for each reader;
PipedInputStream in1 = new PipedInputStream();
PipedInputStream in2 = new PipedInputStream();
// Start copying the input stream to both piped input streams.
startCopy(is, new TeeOutputStream(
new PipedOutputStream(in1), new PipedOutputStream(in2)));
// Create the two buffered readers.
BufferedReader br1 = new BufferedReader(new InputStreamReader(in1));
BufferedReader br2 = new BufferedReader(new InputStreamReader(in2));
// Do some interleaved reads from them.
// ...
}
private static void startCopy(InputStream in, OutputStream out) {
(new Thread() {
public void run() {
try {
IOUtils.copy(in, out);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}).start();
}
Looking for a possible way to have an outputstream sending bytes to two or more different Inputstream, I found this forum.
Unfortunately, the exact solution was directing to PipedOutputStream and PipedInputStream.
So, I was declined to write a PipeOutputStream extension.
Here it is. The example is written in the PipedOutputStream's "main" method.
/**
* Extensao de {#link PipedOutputStream}, onde eh possivel conectar mais de um {#link PipedInputStream}
*/
public class PipedOutputStreamEx extends PipedOutputStream {
/**
*
*/
public PipedOutputStreamEx() {
// TODO Auto-generated constructor stub
}
/* REMIND: identification of the read and write sides needs to be
more sophisticated. Either using thread groups (but what about
pipes within a thread?) or using finalization (but it may be a
long time until the next GC). */
private PipedInputStreamEx[] sinks=null;
public synchronized void connect(PipedInputStreamEx... pIns) throws IOException {
for (PipedInputStreamEx snk : pIns) {
if (snk == null) {
throw new NullPointerException();
} else if (sinks != null || snk.connected) {
throw new IOException("Already connected");
}
snk.in = -1;
snk.out = 0;
snk.connected = true;
}
this.sinks = pIns;
}
/**
* Writes the specified <code>byte</code> to the piped output stream.
* <p>
* Implements the <code>write</code> method of <code>OutputStream</code>.
*
* #param b the <code>byte</code> to be written.
* #exception IOException if the pipe is <a href=#BROKEN> broken</a>,
* {#link #connect(java.io.PipedInputStream) unconnected},
* closed, or if an I/O error occurs.
*/
public void write(int b) throws IOException {
if (this.sinks == null) {
throw new IOException("Pipe(s) not connected");
}
for (PipedInputStreamEx sink : this.sinks) {
sink.receive(b);
}
}
/**
* Writes <code>len</code> bytes from the specified byte array
* starting at offset <code>off</code> to this piped output stream.
* This method blocks until all the bytes are written to the output
* stream.
*
* #param b the data.
* #param off the start offset in the data.
* #param len the number of bytes to write.
* #exception IOException if the pipe is <a href=#BROKEN> broken</a>,
* {#link #connect(java.io.PipedInputStream) unconnected},
* closed, or if an I/O error occurs.
*/
public void write(byte b[], int off, int len) throws IOException {
if (sinks == null) {
throw new IOException("Pipe not connected");
} else if (b == null) {
throw new NullPointerException();
} else if ((off < 0) || (off > b.length) || (len < 0) ||
((off + len) > b.length) || ((off + len) < 0)) {
throw new IndexOutOfBoundsException();
} else if (len == 0) {
return;
}
for (PipedInputStreamEx sink : this.sinks) {
sink.receive(b, off, len);
}
}
/**
* Flushes this output stream and forces any buffered output bytes
* to be written out.
* This will notify any readers that bytes are waiting in the pipe.
*
* #exception IOException if an I/O error occurs.
*/
public synchronized void flush() throws IOException {
if (sinks != null) {
for (PipedInputStreamEx sink : this.sinks) {
synchronized (sink) {
sink.notifyAll();
}
}
}
}
/**
* Closes this piped output stream and releases any system resources
* associated with this stream. This stream may no longer be used for
* writing bytes.
*
* #exception IOException if an I/O error occurs.
*/
public void close() throws IOException {
if (sinks != null) {
for (PipedInputStreamEx sink : this.sinks) {
sink.receivedLast();
}
}
}
/**
* Teste desta extensao de {#link PipedOutputStream}
* #param args
* #throws InterruptedException
* #throws IOException
*/
public static void main(String[] args) throws InterruptedException, IOException {
final PipedOutputStreamEx pOut = new PipedOutputStreamEx();
final PipedInputStreamEx pInHash = new PipedInputStreamEx();
final PipedInputStreamEx pInConsole = new PipedInputStreamEx();
pOut.connect(pInHash, pInConsole);
Thread escreve = new Thread("Escrevendo") {
#Override
public void run() {
String[] paraGravar = new String[]{
"linha1 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha2 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha3 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha4 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha5 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha6 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha7 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha8 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha9 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha10 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha11 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha12 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha13 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha14 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha15 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha16 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha17 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha18 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha19 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
, "linha20 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n"
};
for (String s :paraGravar) {
try {
pOut.write(s.getBytes("ISO-8859-1") );
Thread.sleep(100);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
try {
pOut.close();
} catch (IOException e) {
e.printStackTrace();
}
}
};
Thread le1 = new Thread("Le1 - hash") {
#Override
public void run() {
try {
System.out.println("HASH: "+HashUtil.getHashCRC(pInHash,true));
} catch (Exception e) {
e.printStackTrace();
}
}
};
Thread le2 = new Thread("Le2 - escreve no console") {
#Override
public void run() {
BufferedReader bIn = new BufferedReader(new InputStreamReader(pInConsole));
String s;
try {
while ( (s=bIn.readLine())!=null) {
Thread.sleep(700); //teste simulando o um leitor lento...
System.out.println(s);
}
} catch (IOException e) {
throw new RuntimeException(e);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
};
escreve.start();
le1.start();
le2.start();
escreve.join();
le1.join();
le2.join();
pInHash.close();
pInConsole.close();
}
}
Here is the PipedInputStreamEx code. Unfortunately, I had to copy all JDK code, to have access to "connected", "in" and "out" properties.
/**
* Extensao de {#link PipedInputStream}, que permite conetar mais de um destes no {#link PipedOutputStream}
* Como a classe ancestral possui propriedades 'package friend', tivemos que copiar o codigo herdado :/
*/
public class PipedInputStreamEx extends PipedInputStream {
#Override
public void connect(PipedOutputStream src) throws IOException {
throw new IOException("conecte usando PipedOutputStream.connect()");
}
//----------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------
//--------- INICIO codigo da classe herdada (alguns metodos comentados...)----------------------------------
//----------------------------------------------------------------------------------------------------------
boolean closedByWriter = false;
volatile boolean closedByReader = false;
boolean connected = false;
/* REMIND: identification of the read and write sides needs to be
more sophisticated. Either using thread groups (but what about
pipes within a thread?) or using finalization (but it may be a
long time until the next GC). */
Thread readSide;
Thread writeSide;
private static final int DEFAULT_PIPE_SIZE = 1024;
/**
* The default size of the pipe's circular input buffer.
* #since JDK1.1
*/
// This used to be a constant before the pipe size was allowed
// to change. This field will continue to be maintained
// for backward compatibility.
protected static final int PIPE_SIZE = DEFAULT_PIPE_SIZE;
/**
* The circular buffer into which incoming data is placed.
* #since JDK1.1
*/
protected byte buffer[];
/**
* The index of the position in the circular buffer at which the
* next byte of data will be stored when received from the connected
* piped output stream. <code>in<0</code> implies the buffer is empty,
* <code>in==out</code> implies the buffer is full
* #since JDK1.1
*/
protected int in = -1;
/**
* The index of the position in the circular buffer at which the next
* byte of data will be read by this piped input stream.
* #since JDK1.1
*/
protected int out = 0;
// /**
// * Creates a <code>PipedInputStream</code> so
// * that it is connected to the piped output
// * stream <code>src</code>. Data bytes written
// * to <code>src</code> will then be available
// * as input from this stream.
// *
// * #param src the stream to connect to.
// * #exception IOException if an I/O error occurs.
// */
// public PipedInputStream(PipedOutputStream src) throws IOException {
// this(src, DEFAULT_PIPE_SIZE);
// }
//
// /**
// * Creates a <code>PipedInputStream</code> so that it is
// * connected to the piped output stream
// * <code>src</code> and uses the specified pipe size for
// * the pipe's buffer.
// * Data bytes written to <code>src</code> will then
// * be available as input from this stream.
// *
// * #param src the stream to connect to.
// * #param pipeSize the size of the pipe's buffer.
// * #exception IOException if an I/O error occurs.
// * #exception IllegalArgumentException if <code>pipeSize <= 0</code>.
// * #since 1.6
// */
// public PipedInputStream(PipedOutputStream src, int pipeSize)
// throws IOException {
// initPipe(pipeSize);
// connect(src);
// }
/**
* Creates a <code>PipedInputStream</code> so
* that it is not yet {#linkplain #connect(java.io.PipedOutputStream)
* connected}.
* It must be {#linkplain java.io.PipedOutputStream#connect(
* java.io.PipedInputStream) connected} to a
* <code>PipedOutputStream</code> before being used.
*/
public PipedInputStreamEx() {
initPipe(DEFAULT_PIPE_SIZE);
}
/**
* Creates a <code>PipedInputStream</code> so that it is not yet
* {#linkplain #connect(java.io.PipedOutputStream) connected} and
* uses the specified pipe size for the pipe's buffer.
* It must be {#linkplain java.io.PipedOutputStream#connect(
* java.io.PipedInputStream)
* connected} to a <code>PipedOutputStream</code> before being used.
*
* #param pipeSize the size of the pipe's buffer.
* #exception IllegalArgumentException if <code>pipeSize <= 0</code>.
* #since 1.6
*/
public PipedInputStreamEx(int pipeSize) {
initPipe(pipeSize);
}
private void initPipe(int pipeSize) {
if (pipeSize <= 0) {
throw new IllegalArgumentException("Pipe Size <= 0");
}
buffer = new byte[pipeSize];
}
// /**
// * Causes this piped input stream to be connected
// * to the piped output stream <code>src</code>.
// * If this object is already connected to some
// * other piped output stream, an <code>IOException</code>
// * is thrown.
// * <p>
// * If <code>src</code> is an
// * unconnected piped output stream and <code>snk</code>
// * is an unconnected piped input stream, they
// * may be connected by either the call:
// * <p>
// * <pre><code>snk.connect(src)</code> </pre>
// * <p>
// * or the call:
// * <p>
// * <pre><code>src.connect(snk)</code> </pre>
// * <p>
// * The two
// * calls have the same effect.
// *
// * #param src The piped output stream to connect to.
// * #exception IOException if an I/O error occurs.
// */
// public void connect(PipedOutputStream src) throws IOException {
// src.connect(this);
// }
/**
* Receives a byte of data. This method will block if no input is
* available.
* #param b the byte being received
* #exception IOException If the pipe is <a href=#BROKEN> <code>broken</code></a>,
* {#link #connect(java.io.PipedOutputStream) unconnected},
* closed, or if an I/O error occurs.
* #since JDK1.1
*/
protected synchronized void receive(int b) throws IOException {
checkStateForReceive();
writeSide = Thread.currentThread();
if (in == out)
awaitSpace();
if (in < 0) {
in = 0;
out = 0;
}
buffer[in++] = (byte)(b & 0xFF);
if (in >= buffer.length) {
in = 0;
}
}
/**
* Receives data into an array of bytes. This method will
* block until some input is available.
* #param b the buffer into which the data is received
* #param off the start offset of the data
* #param len the maximum number of bytes received
* #exception IOException If the pipe is <a href=#BROKEN> broken</a>,
* {#link #connect(java.io.PipedOutputStream) unconnected},
* closed,or if an I/O error occurs.
*/
synchronized void receive(byte b[], int off, int len) throws IOException {
checkStateForReceive();
writeSide = Thread.currentThread();
int bytesToTransfer = len;
while (bytesToTransfer > 0) {
if (in == out)
awaitSpace();
int nextTransferAmount = 0;
if (out < in) {
nextTransferAmount = buffer.length - in;
} else if (in < out) {
if (in == -1) {
in = out = 0;
nextTransferAmount = buffer.length - in;
} else {
nextTransferAmount = out - in;
}
}
if (nextTransferAmount > bytesToTransfer)
nextTransferAmount = bytesToTransfer;
assert(nextTransferAmount > 0);
System.arraycopy(b, off, buffer, in, nextTransferAmount);
bytesToTransfer -= nextTransferAmount;
off += nextTransferAmount;
in += nextTransferAmount;
if (in >= buffer.length) {
in = 0;
}
}
}
private void checkStateForReceive() throws IOException {
if (!connected) {
throw new IOException("Pipe not connected");
} else if (closedByWriter || closedByReader) {
throw new IOException("Pipe closed");
} else if (readSide != null && !readSide.isAlive()) {
throw new IOException("Read end dead");
}
}
private void awaitSpace() throws IOException {
while (in == out) {
checkStateForReceive();
/* full: kick any waiting readers */
notifyAll();
try {
wait(1000);
} catch (InterruptedException ex) {
throw new java.io.InterruptedIOException();
}
}
}
/**
* Notifies all waiting threads that the last byte of data has been
* received.
*/
synchronized void receivedLast() {
closedByWriter = true;
notifyAll();
}
/**
* Reads the next byte of data from this piped input stream. The
* value byte is returned as an <code>int</code> in the range
* <code>0</code> to <code>255</code>.
* This method blocks until input data is available, the end of the
* stream is detected, or an exception is thrown.
*
* #return the next byte of data, or <code>-1</code> if the end of the
* stream is reached.
* #exception IOException if the pipe is
* {#link #connect(java.io.PipedOutputStream) unconnected},
* <a href=#BROKEN> <code>broken</code></a>, closed,
* or if an I/O error occurs.
*/
public synchronized int read() throws IOException {
if (!connected) {
throw new IOException("Pipe not connected");
} else if (closedByReader) {
throw new IOException("Pipe closed");
} else if (writeSide != null && !writeSide.isAlive()
&& !closedByWriter && (in < 0)) {
throw new IOException("Write end dead");
}
readSide = Thread.currentThread();
int trials = 2;
while (in < 0) {
if (closedByWriter) {
/* closed by writer, return EOF */
return -1;
}
if ((writeSide != null) && (!writeSide.isAlive()) && (--trials < 0)) {
throw new IOException("Pipe broken");
}
/* might be a writer waiting */
notifyAll();
try {
wait(1000);
} catch (InterruptedException ex) {
throw new java.io.InterruptedIOException();
}
}
int ret = buffer[out++] & 0xFF;
if (out >= buffer.length) {
out = 0;
}
if (in == out) {
/* now empty */
in = -1;
}
return ret;
}
/**
* Reads up to <code>len</code> bytes of data from this piped input
* stream into an array of bytes. Less than <code>len</code> bytes
* will be read if the end of the data stream is reached or if
* <code>len</code> exceeds the pipe's buffer size.
* If <code>len </code> is zero, then no bytes are read and 0 is returned;
* otherwise, the method blocks until at least 1 byte of input is
* available, end of the stream has been detected, or an exception is
* thrown.
*
* #param b the buffer into which the data is read.
* #param off the start offset in the destination array <code>b</code>
* #param len the maximum number of bytes read.
* #return the total number of bytes read into the buffer, or
* <code>-1</code> if there is no more data because the end of
* the stream has been reached.
* #exception NullPointerException If <code>b</code> is <code>null</code>.
* #exception IndexOutOfBoundsException If <code>off</code> is negative,
* <code>len</code> is negative, or <code>len</code> is greater than
* <code>b.length - off</code>
* #exception IOException if the pipe is <a href=#BROKEN> <code>broken</code></a>,
* {#link #connect(java.io.PipedOutputStream) unconnected},
* closed, or if an I/O error occurs.
*/
public synchronized int read(byte b[], int off, int len) throws IOException {
if (b == null) {
throw new NullPointerException();
} else if (off < 0 || len < 0 || len > b.length - off) {
throw new IndexOutOfBoundsException();
} else if (len == 0) {
return 0;
}
/* possibly wait on the first character */
int c = read();
if (c < 0) {
return -1;
}
b[off] = (byte) c;
int rlen = 1;
while ((in >= 0) && (len > 1)) {
int available;
if (in > out) {
available = Math.min((buffer.length - out), (in - out));
} else {
available = buffer.length - out;
}
// A byte is read beforehand outside the loop
if (available > (len - 1)) {
available = len - 1;
}
System.arraycopy(buffer, out, b, off + rlen, available);
out += available;
rlen += available;
len -= available;
if (out >= buffer.length) {
out = 0;
}
if (in == out) {
/* now empty */
in = -1;
}
}
return rlen;
}
/**
* Returns the number of bytes that can be read from this input
* stream without blocking.
*
* #return the number of bytes that can be read from this input stream
* without blocking, or {#code 0} if this input stream has been
* closed by invoking its {#link #close()} method, or if the pipe
* is {#link #connect(java.io.PipedOutputStream) unconnected}, or
* <a href=#BROKEN> <code>broken</code></a>.
*
* #exception IOException if an I/O error occurs.
* #since JDK1.0.2
*/
public synchronized int available() throws IOException {
if(in < 0)
return 0;
else if(in == out)
return buffer.length;
else if (in > out)
return in - out;
else
return in + buffer.length - out;
}
/**
* Closes this piped input stream and releases any system resources
* associated with the stream.
*
* #exception IOException if an I/O error occurs.
*/
public void close() throws IOException {
closedByReader = true;
synchronized (this) {
in = -1;
}
}
//----------------------------------------------------------------------------------------------------------
//--------- FIM codigo da classe herdada -------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------
}