We Keep Coding

Can Apache Flink write to files that are named based on a key?

Is it possible in Apache Flink to write to multiple text files depending on a key? For instance, I have some data like this.
key1, foo, bar
key2, baz, foo
key3, etc, etc
The value of the key isn’t known at compile time; new keys would come in and I’d like to write the results for that key to a separate file to those of the other keys.
I'd expect to see 3 files, named 'key1.txt', 'key2.txt' and 'key3.txt'.
Is this something Flink can do out of the box?

You can try the following sink's implementation, which can be used with KeyedStream :
KeyedStream<Tuple2<String, String>, Tuple> keyedDataStream = dataStream.keyBy(0);
StreamKeyPartitionerSink<Tuple2<String, SynopsesEvent>> sinkFunction = new StreamKeyPartitionerSink<Tuple2<String, SynopsesEvent>>(
"../data/key_grouping", "f0"); // f0 is the key field name
keyedDataStream.addSink(sinkFunction);
For more info about state managament in Flink : https://ci.apache.org/projects/flink/flink-docs-release-1.3/dev/stream/state.html#keyed-state since I used it for managing state per key.
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
/**
* * Flink sink writes tuples to files partitioned by their keys, which also writes the records as
* batches.
*
* #param <IN> Input tuple type
*
* #author ehabqadah
*/
public class StreamKeyPartitionerSink<IN> extends RichSinkFunction<IN> {
private transient ValueState<String> outputFilePath;
private transient ValueState<List<IN>> inputTupleList;
/**
* Number of rcords to be hold before writing.
*/
private int writeBatchSize;
/**
* The output directory path
*/
private String outputDirPath;
/**
* The name of the input tuple key
*/
private String keyFieldName;
public StreamKeyPartitionerSink(String outputDirPath, String keyFieldName) {
this(outputDirPath, keyFieldName, 1);
}
/**
*
* #param outputDirPath- writeBatchSize the size of on hold batch before write
* #param writeBatchSize - output directory
*/
public StreamKeyPartitionerSink(String outputDirPath, String keyFieldName, int writeBatchSize) {
this.writeBatchSize = writeBatchSize;
this.outputDirPath = outputDirPath;
this.keyFieldName = keyFieldName;
}
#Override
public void open(Configuration config) {
// initialize state holders
`//for more info about state management check `//
ValueStateDescriptor<String> outputFilePathDesc =
new ValueStateDescriptor<String>("outputFilePathDesc",
TypeInformation.of(new TypeHint<String>() {}));
ValueStateDescriptor<List<IN>> inputTupleListDesc =
new ValueStateDescriptor<List<IN>>("inputTupleListDesc",
TypeInformation.of(new TypeHint<List<IN>>() {}));
outputFilePath = getRuntimeContext().getState(outputFilePathDesc);
inputTupleList = getRuntimeContext().getState(inputTupleListDesc);
}
#Override
public void invoke(IN value) throws Exception {
List<IN> inputTuples =
inputTupleList.value() == null ? new ArrayList<IN>() : inputTupleList.value();
inputTuples.add(value);
if (inputTuples.size() == writeBatchSize) {
writeInputList(inputTuples);
inputTuples = new ArrayList<IN>();
}
// update the state
inputTupleList.update(inputTuples);
}
/**
* Write the tuple list, each record in separate line
*
* #param tupleList
* #throws Exception
*/
public void writeInputList(List<IN> tupleList) {
String path = getOrInitFilePath(tupleList);
try (PrintWriter outStream = new PrintWriter(new BufferedWriter(new FileWriter(path, true)))) {
for (IN tupleToWrite : tupleList) {
outStream.println(tupleToWrite);
}
} catch (IOException e) {
throw new RuntimeException("Exception occured while writing file " + path, e);
}
}
private String getOrInitFilePath(List<IN> tupleList) {
IN firstInstance = tupleList.get(0);
String path = null;
try {
path = outputFilePath.value();
if (path == null) {
Field keyField = firstInstance.getClass().getField(keyFieldName);
String keyValue = keyField.get(firstInstance).toString();
path = Paths.get(outputDirPath, keyValue + ".txt").toString();
setUpOutputFilePathPath(outputDirPath, path);
// save the computed path for this key
outputFilePath.update(path);
}
} catch (IOException | NoSuchFieldException | SecurityException | IllegalArgumentException
| IllegalAccessException e) {
throw new RuntimeException(
"ExceptionsetUpOutputFilePathPath occured while fetching the value of key field " + path,
e);
}
return path;
}
private void setUpOutputFilePathPath(String outputDirPath, String path) throws IOException {
if (!Files.exists(Paths.get(outputDirPath))) {
Files.createDirectories(Paths.get(outputDirPath));
}
// create the file if it does not exist and delete its content
Files.write(Paths.get(path), "".getBytes(), StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING);
}
}

That is not possible ouf-of-the-box. However, you can implement an own output format and use it via result.out(...) (for Batch API); see https://ci.apache.org/projects/flink/flink-docs-release-1.1/apis/batch/index.html#data-sinks
For Streaming API, it would be stream.addSink(...); see https://ci.apache.org/projects/flink/flink-docs-release-1.1/apis/streaming/index.html#data-sinks

Spring batch job with FlatFileItemReader hangs when big csv file is used

I am triggering a spring batch job from my web application. CSV file to be processed will be requested by user. I am using FlatFileItemReader as reader. My job configuration for reader is as below
#Bean
#Scope("job")
public CustomFlatFileItemReader<ProducerMessage> csvReader(#Value("#{jobParameters[fileName]}") String fileName, #Value("#{jobParameters[s3SourceFolderPrefix]}") String s3SourceFolderPrefix, #Value("#{jobParameters[timeStamp]}") long timeStamp, ConfigurationService mongoConfigurationService) {
CustomFlatFileItemReader faltFileReader = new CustomFlatFileItemReader();
DefaultLineMapper<ProducerMessage> lineMapper = new DefaultLineMapper<ProducerMessage>();
lineMapper.setLineTokenizer(new DelimitedLineTokenizer());
CSVFieldMapper csvFieldMapper = new CSVFieldMapper(fileName, s3SourceFolderPrefix, timeStamp, mongoConfigurationService);
lineMapper.setFieldSetMapper(csvFieldMapper);
faltFileReader.setLineMapper(lineMapper);
faltFileReader.setResource(new FileSystemResource(fileName));
return faltFileReader;
}
I am using resource as FileSystemResource does this load the whole file in memory ?
When the CSV file is of big size ( more than 1 Gb ) step execution goes to an never ending loop. I could see the job in repository as STARTED status.
Please help.
My step configuration is :
#Bean(name = "csvFileStep")
public Step csvFileStep(StepBuilderFactory stepBuilderFactory,
ItemReader csvReader, ItemWriter writer,
#Value("${reader.chunkSize}")
int chunkSize) {
LOGGER.info("Step configuration loaded with chunk size {}", chunkSize);
return stepBuilderFactory.get("step1")
.chunk(chunkSize).reader(csvReader)
.faultTolerant()
.skip(CSVFieldMappingException.class)
//Setting the skip limit to maxValue of Int so as to skip as many mapping exceptions.
.skipLimit(Int.MaxValue())
.writer(writer)
.build();
}
My writer class :
package com.fastretailing.catalogPlatformSCMProducer.producerjob.writer.rds;
import com.fastretailing.catalogPlatformSCMProducer.model.Configuration;
import com.fastretailing.catalogPlatformSCMProducer.model.NotificationBean;
import com.fastretailing.catalogPlatformSCMProducer.model.ProducerMessage;
import com.fastretailing.catalogPlatformSCMProducer.notification.JobStatus;
import com.fastretailing.catalogPlatformSCMProducer.util.ProducerUtil;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.configuration.annotation.JobScope;
import org.springframework.batch.item.support.AbstractItemStreamItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
/**
* Producer Job writer for writing directly to RDS.
*/
public class RdsWriter extends AbstractItemStreamItemWriter<ProducerMessage> {
#Autowired
#Qualifier("rdsJdbcTemplate")
JdbcTemplate rdsJdbcTemplate;
#Autowired
Configuration configuration;
private final Logger LOGGER = LoggerFactory.getLogger(this.getClass());
#Override
public void write(List<? extends ProducerMessage> list) throws Exception {
for(ProducerMessage message : list){
handleRecord(message);
}
}
#Transactional
public void handleRecord(ProducerMessage message) {
try {
String pkCheckQuery = ProducerUtil.generatePrimaryKeyCheckQuery(message);
Long returnValue = null;
try {
returnValue = rdsJdbcTemplate.queryForObject(pkCheckQuery, Long.class);
} catch (EmptyResultDataAccessException e) {
LOGGER.debug("Primary key not exists in RDS table. This will insert new row");
}
if(null == returnValue || returnValue < message.getTimeStamp()){
String query = ProducerUtil.getRdsWriteQuery(message);
LOGGER.debug("Executing Query {}", query);
rdsJdbcTemplate.update(query);
JobStatus.addRowsWritten(1);
}else{
JobStatus.addRowsSkippedWriting(1);
LOGGER.debug("Skipped row due to timestamp check failure for feedName {}", message.getFeedConfigName());
}
} catch (Exception e) {
JobStatus.changeStatus(ProducerUtil.SNS_NOTIFICATION_EVENT_IN_COMPLETE);
JobStatus.addRowsSkippedWriting(1);
JobStatus.addExceptionInLogWriter(ExceptionUtils.getStackTrace(e));
LOGGER.error("Exception while writing records to RDS table. These records will be skipped from writing.", e);
}
}
}
My custom reader : ( This is same as FlatFileItemReader apart from custom exception catching in doRead method)
/*
* Copyright 2006-2007 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.fastretailing.catalogPlatformSCMProducer.producerjob.reader;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.Charset;
import com.fastretailing.catalogPlatformSCMProducer.exception.CSVFieldMappingException;
import com.fastretailing.catalogPlatformSCMProducer.notification.JobStatus;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ReaderNotOpenException;
import org.springframework.batch.item.file.*;
import org.springframework.batch.item.file.separator.RecordSeparatorPolicy;
import org.springframework.batch.item.file.separator.SimpleRecordSeparatorPolicy;
import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
import org.springframework.util.ClassUtils;
import org.springframework.util.StringUtils;
/**
* Reusing Spring's FlatFileItemReader from https://github.com/spring-projects/spring-batch/blob/master/spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/FlatFileItemReader.java
*to change exception handling policy in case of mapping exceptions.
*
*/
public class CustomFlatFileItemReader<T> extends AbstractItemCountingItemStreamItemReader<T> implements
ResourceAwareItemReaderItemStream<T>, InitializingBean {
private static final Log logger = LogFactory.getLog(CustomFlatFileItemReader.class);
// default encoding for input files
public static final String DEFAULT_CHARSET = Charset.defaultCharset().name();
private RecordSeparatorPolicy recordSeparatorPolicy = new SimpleRecordSeparatorPolicy();
private Resource resource;
private BufferedReader reader;
private int lineCount = 0;
private String[] comments = new String[] { "#" };
private boolean noInput = false;
private String encoding = DEFAULT_CHARSET;
private LineMapper<T> lineMapper;
private int linesToSkip = 0;
private LineCallbackHandler skippedLinesCallback;
private boolean strict = true;
private BufferedReaderFactory bufferedReaderFactory = new DefaultBufferedReaderFactory();
public CustomFlatFileItemReader() {
setName(ClassUtils.getShortName(CustomFlatFileItemReader.class));
}
/**
* In strict mode the reader will throw an exception on
* {#link #open(org.springframework.batch.item.ExecutionContext)} if the input resource does not exist.
* #param strict <code>true</code> by default
*/
public void setStrict(boolean strict) {
this.strict = strict;
}
/**
* #param skippedLinesCallback will be called for each one of the initial skipped lines before any items are read.
*/
public void setSkippedLinesCallback(LineCallbackHandler skippedLinesCallback) {
this.skippedLinesCallback = skippedLinesCallback;
}
/**
* Public setter for the number of lines to skip at the start of a file. Can be used if the file contains a header
* without useful (column name) information, and without a comment delimiter at the beginning of the lines.
*
* #param linesToSkip the number of lines to skip
*/
public void setLinesToSkip(int linesToSkip) {
this.linesToSkip = linesToSkip;
}
/**
* Setter for line mapper. This property is required to be set.
* #param lineMapper maps line to item
*/
public void setLineMapper(LineMapper<T> lineMapper) {
this.lineMapper = lineMapper;
}
/**
* Setter for the encoding for this input source. Default value is {#link #DEFAULT_CHARSET}.
*
* #param encoding a properties object which possibly contains the encoding for this input file;
*/
public void setEncoding(String encoding) {
this.encoding = encoding;
}
/**
* Factory for the {#link BufferedReader} that will be used to extract lines from the file. The default is fine for
* plain text files, but this is a useful strategy for binary files where the standard BufferedReaader from java.io
* is limiting.
*
* #param bufferedReaderFactory the bufferedReaderFactory to set
*/
public void setBufferedReaderFactory(BufferedReaderFactory bufferedReaderFactory) {
this.bufferedReaderFactory = bufferedReaderFactory;
}
/**
* Setter for comment prefixes. Can be used to ignore header lines as well by using e.g. the first couple of column
* names as a prefix.
*
* #param comments an array of comment line prefixes.
*/
public void setComments(String[] comments) {
this.comments = new String[comments.length];
System.arraycopy(comments, 0, this.comments, 0, comments.length);
}
/**
* Public setter for the input resource.
*/
#Override
public void setResource(Resource resource) {
this.resource = resource;
}
/**
* Public setter for the recordSeparatorPolicy. Used to determine where the line endings are and do things like
* continue over a line ending if inside a quoted string.
*
* #param recordSeparatorPolicy the recordSeparatorPolicy to set
*/
public void setRecordSeparatorPolicy(RecordSeparatorPolicy recordSeparatorPolicy) {
this.recordSeparatorPolicy = recordSeparatorPolicy;
}
/**
* #return string corresponding to logical record according to
* {#link #setRecordSeparatorPolicy(RecordSeparatorPolicy)} (might span multiple lines in file).
*/
#Override
protected T doRead() throws Exception {
if (noInput) {
return null;
}
String line = readLine();
if (line == null) {
return null;
}
else {
try {
return lineMapper.mapLine(line, lineCount);
}catch (CSVFieldMappingException e){
String message = "Parsing error at line: " + lineCount + " in resource=["
+ resource.getDescription() + "], input=[" + line + "]";
logger.error(message,e);
Exception customException = new CSVFieldMappingException(message, e);
JobStatus.addExceptionInLogWriter(ExceptionUtils.getStackTrace(customException));
throw customException;
}
catch (Exception ex) {
Exception exNew = new FlatFileParseException("Parsing error at line: " + lineCount + " in resource=["
+ resource.getDescription() + "], input=[" + line + "]", ex, line, lineCount);
JobStatus.addExceptionInLogWriter(ExceptionUtils.getStackTrace(exNew));
throw exNew;
}
}
}
/**
* #return next line (skip comments).getCurrentResource
*/
private String readLine() {
if (reader == null) {
throw new ReaderNotOpenException("Reader must be open before it can be read.");
}
String line = null;
try {
line = this.reader.readLine();
if (line == null) {
return null;
}
lineCount++;
while (isComment(line)) {
line = reader.readLine();
if (line == null) {
return null;
}
lineCount++;
}
line = applyRecordSeparatorPolicy(line);
}
catch (IOException e) {
// Prevent IOException from recurring indefinitely
// if client keeps catching and re-calling
noInput = true;
throw new NonTransientFlatFileException("Unable to read from resource: [" + resource + "]", e, line,
lineCount);
}
return line;
}
private boolean isComment(String line) {
for (String prefix : comments) {
if (line.startsWith(prefix)) {
return true;
}
}
return false;
}
#Override
protected void doClose() throws Exception {
lineCount = 0;
if (reader != null) {
reader.close();
}
}
#Override
protected void doOpen() throws Exception {
Assert.notNull(resource, "Input resource must be set");
Assert.notNull(recordSeparatorPolicy, "RecordSeparatorPolicy must be set");
noInput = true;
if (!resource.exists()) {
if (strict) {
throw new IllegalStateException("Input resource must exist (reader is in 'strict' mode): " + resource);
}
logger.warn("Input resource does not exist " + resource.getDescription());
return;
}
if (!resource.isReadable()) {
if (strict) {
throw new IllegalStateException("Input resource must be readable (reader is in 'strict' mode): "
+ resource);
}
logger.warn("Input resource is not readable " + resource.getDescription());
return;
}
reader = bufferedReaderFactory.create(resource, encoding);
for (int i = 0; i < linesToSkip; i++) {
String line = readLine();
if (skippedLinesCallback != null) {
skippedLinesCallback.handleLine(line);
}
}
noInput = false;
}
#Override
public void afterPropertiesSet() throws Exception {
Assert.notNull(lineMapper, "LineMapper is required");
}
#Override
protected void jumpToItem(int itemIndex) throws Exception {
for (int i = 0; i < itemIndex; i++) {
readLine();
}
}
private String applyRecordSeparatorPolicy(String line) throws IOException {
String record = line;
while (line != null && !recordSeparatorPolicy.isEndOfRecord(record)) {
line = this.reader.readLine();
if (line == null) {
if (StringUtils.hasText(record)) {
// A record was partially complete since it hasn't ended but
// the line is null
throw new FlatFileParseException("Unexpected end of file before record complete", record, lineCount);
}
else {
// Record has no text but it might still be post processed
// to something (skipping preProcess since that was already
// done)
break;
}
}
else {
lineCount++;
}
record = recordSeparatorPolicy.preProcess(record) + line;
}
return recordSeparatorPolicy.postProcess(record);
}
}
I am configuring the jdbcTemplate like this in my jobConfiguration class.
#Primary
#Bean(name = "mysqlDs")
#ConfigurationProperties(prefix = "datasource.sql.jobMetaDataDb")
public DataSource sqlDataSource() {
return DataSourceBuilder.create().build();
}
#Bean(name = "mysql")
#Autowired
public JdbcTemplate slaveJdbcTemplate(#Qualifier("mysqlDs") DataSource mysqlDs) {
return new JdbcTemplate(mysqlDs);
}
#Bean(name = "rdsDataSource")
#ConfigurationProperties(prefix = "datasource.sql.rdsWriterDb")
public DataSource rdsDataSource() {
return DataSourceBuilder.create().build();
}
#Bean(name = "rdsJdbcTemplate")
#Autowired
public JdbcTemplate rdsJdbcTemplate(#Qualifier("rdsDataSource") DataSource rdsDataSource) {
return new JdbcTemplate(rdsDataSource);
}

How can I store a String with escape characters without turning the escape characters into their substitutes?

I'm writing a Java-based program that downloads a JavaScript file from GitHub, parses it into a String, and then pushes it to a MediaWiki site. The problem is that the JS file has fairly complicated regex in it, and Java is parsing it into escape characters (e.g. [^\S\r\n] getting parsed into /\s). Is there a way I can store a String without these escape characters being activated?
EDIT: The code:
package org.wikipedia.afchbuddy;
import java.io.IOException;
import static java.lang.System.*;
import java.nio.file.*;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.channels.*;
import java.net.URL;
import java.io.FileOutputStream;
import javax.security.auth.login.LoginException;
/**
* This is a bot script designed to allow sysops to push releases for AFCH
* on-wiki.
*
* #author Nathan2055
*/
public class AFCHBuddy {
private static String username;
private static String password;
private static String location;
private static String buffer;
private static String error = "An error occured trying to access and write to Wikipedia. If you report this error, please provide the following debug info:";
private static String basescript = "https://raw.github.com/WPAFC/afch/develop/src/";
private static String basescript_stable = "https://raw.github.com/WPAFC/afch/master/src/";
private static String mediawikilocation = "MediaWiki:Gadget-afchelper.js";
/**
* The main bot script.
*
* #param args This program accepts three command line arguments, 0 is the
* push location (stable, beta, or testwiki), 1 is your username, and 2 is
* your password.
*/
public static void main(String[] args) {
if (args.length != 3) {
err.println("Incorrect number of arguments. Please see the README for more information.");
System.exit(1);
}
location = args[0];
username = args[1];
password = args[2];
switch (location) {
case "testwiki":
build("testwiki");
break;
case "beta":
build("beta");
break;
case "stable":
build("stable");
break;
default:
err.println("Error: Unknown build location.");
System.exit(1);
}
}
/**
* This method does the actual work of performing the edits.
*
* #param destarg The destination wiki.
*/
private static void build(String destarg) {
Wiki wiki;
try {
if ("testwiki".equals(destarg)) {
wiki = new Wiki("test.wikipedia.org");
} else {
wiki = new Wiki();
}
wiki.login(username, password);
downloadFile(basescript + "MediaWiki:Gadget-afchelper.js", "afch.js");
buffer = readFile("afch.js");
wiki.edit(mediawikilocation, buffer, "Update afch.js using AFCHBuddy");
downloadFile(basescript + "MediaWiki:Gadget-afchelper.css", "afch.css");
buffer = readFile("afch.css");
wiki.edit("MediaWiki:Gadget-afchelper.css", buffer, "Update afch.css using AFCHBuddy");
downloadFile(basescript + "core.js", "core.js");
buffer = readFile("core.js");
wiki.edit(mediawikilocation + "/core.js", buffer, "Update core.js using AFCHBuddy");
downloadFile(basescript + "redirects.js", "redirects.js");
buffer = readFile("redirects.js");
wiki.edit(mediawikilocation + "/redirects.js", buffer, "Update redirects.js using AFCHBuddy");
downloadFile(basescript + "submissions.js", "submissions.js");
buffer = readFile("submissions.js");
wiki.edit(mediawikilocation + "/submissions.js", buffer, "Update submissions.js using AFCHBuddy");
downloadFile(basescript + "ffu.js", "ffu.js");
buffer = readFile("ffu.js");
wiki.edit(mediawikilocation + "/ffu.js", buffer, "Update ffu.js using AFCHBuddy");
wiki.logout();
err.println("Files syncronized successfully.");
System.exit(0);
} catch (IOException | LoginException e) {
err.println(error);
e.printStackTrace(err);
System.exit(1);
}
}
/**
* Converts a file to a String.
*
* #param path This accepts the path of the file to be processed in String
* form.
* #return The output String.
* #throws IOException The exception if something goes wrong while reading
* the file.
*/
private static String readFile(String path) throws IOException {
byte[] encoded = Files.readAllBytes(Paths.get(path));
return Charset.defaultCharset().decode(ByteBuffer.wrap(encoded)).toString();
}
/**
* This downloads the input URL and saves it to dest.
*
* #param URL The file's location on the internet.
* #param dest The file's soon to be location on your computer.
* #throws IOException The exception if something goes wrong while
* processing.
*/
private static void downloadFile(String URL, String dest) throws IOException {
URL website = new URL(URL);
ReadableByteChannel rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(dest);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
}
}

Retrieving email attachment filename with mime4j

I'm trying to use mime4j to parse emails, all is working fine, however I'm not able to get the file name of the attachment. Unfortunately the BodyDescriptor doesn't include this information in the content disposition, or content type fields.
I have read that the MaximalBodyDescriptor will include the filename, however I don't know how to tell the parser to return a MaximalBodyDescriptor object.
My handler is implementing the ContentHandler interface. I can't see an alternate interface which would work.
Any advice appreciated.

Here is a helper class that we use successfully to parse e-mails with their attachments.
In this approach attach.getFileName() has the filename.
package com.bitplan.smartCRM;
import java.awt.Toolkit;
import java.awt.datatransfer.Clipboard;
import java.awt.datatransfer.ClipboardOwner;
import java.awt.datatransfer.DataFlavor;
import java.awt.datatransfer.StringSelection;
import java.awt.datatransfer.Transferable;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.List;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.mail.Flags;
import javax.mail.Folder;
import javax.mail.MessagingException;
import javax.mail.Session;
import javax.mail.Store;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import org.apache.commons.io.IOUtils;
import org.apache.james.mime4j.MimeException;
import org.apache.james.mime4j.dom.Body;
import org.apache.james.mime4j.dom.Entity;
import org.apache.james.mime4j.dom.Message;
import org.apache.james.mime4j.dom.MessageBuilder;
import org.apache.james.mime4j.dom.MessageServiceFactory;
import org.apache.james.mime4j.dom.Multipart;
import org.apache.james.mime4j.dom.SingleBody;
import org.apache.james.mime4j.dom.TextBody;
import org.apache.james.mime4j.dom.address.MailboxList;
import org.apache.james.mime4j.message.MessageImpl;
import org.apache.james.mime4j.stream.Field;
import com.bitplan.restinterface.Configuration;
/**
* EMail Helper class
*
* #author wf
* #author Denis Lunev <den#mozgoweb.com>
* #see http
* ://www.mozgoweb.com/posts/how-to-parse-mime-message-using-mime4j-library
* /
*/
public class EMailHelper implements ClipboardOwner {
public static boolean debug = true;
public static Logger LOGGER = Logger
.getLogger("com.bitplan.common.EMailHelper");
private StringBuffer txtBody;
private StringBuffer htmlBody;
private ArrayList<Entity> attachments;
/**
* get a String from an input Stream
*
* #param inputStream
* #return
* #throws IOException
*/
public String fromInputStream(InputStream inputStream) throws IOException {
String result = IOUtils.toString(inputStream);
// System.out.println(result);
return result;
}
/**
* get the full Mail from a message
*
* #param message
* #return
* #throws MessagingException
* #throws IOException
*/
public String fullMail(javax.mail.Message message) throws MessagingException,
IOException {
StringBuffer sBuf = new StringBuffer();
#SuppressWarnings("unchecked")
Enumeration<javax.mail.Header> headers = message.getAllHeaders();
while (headers.hasMoreElements()) {
javax.mail.Header header = headers.nextElement();
sBuf.append(header.getName() + ": " + header.getValue() + "\n");
}
sBuf.append(fromInputStream(message.getInputStream()));
return sBuf.toString();
}
/**
* Authentication
*/
public static class Authentication {
enum AuthenticationType {
pop3, smtp
};
String host;
String user;
String password;
AuthenticationType authenticationType;
Transport mTransport;
/**
* create an Authentication from the configuration
*
* #param configuration
* #param pAuthType
*/
public Authentication(Configuration configuration,
AuthenticationType pAuthType) {
authenticationType = pAuthType;
String prefix = pAuthType.name() + ".";
// use prefix e.g. pop3.host / smtp.host
host = (String) configuration.toMap().get(prefix + "host");
user = (String) configuration.toMap().get(prefix + "user");
password = (String) configuration.toMap().get(prefix + "password");
}
/**
* authenticate for sending / receiving e-mail
*
* #throws MessagingException
*/
public Transport authenticate() throws MessagingException {
Properties lProps = new Properties();
Session session = Session.getDefaultInstance(lProps);
switch (authenticationType) {
case pop3:
Store store = session.getStore("pop3");
store.connect(host, user, password);
store.close();
return null;
case smtp:
// http://javamail.kenai.com/nonav/javadocs/com/sun/mail/smtp/package-summary.html
mTransport = session.getTransport("smtp");
mTransport.connect(host, user, password);
return mTransport;
}
return null;
}
}
/**
* send the given e-mail
*
* #param email
* #throws MessagingException
*/
public void send(EMail email, Configuration configuration)
throws MessagingException {
Authentication lAuth = new Authentication(configuration,
Authentication.AuthenticationType.pop3);
Properties lProps = System.getProperties();
lProps.put("mail.smtp.host", lAuth.host);
// WF 2004-09-18: make sure full qualified domain name is used for localhost
// the default InetAddress.getLocalHost().getHostName() might not work ...
// lProps.put("mail.smtp.localhost",java.net.InetAddress.getLocalHost().getCanonicalHostName());
Session lSession = Session.getInstance(lProps);
MimeMessage lMsg = new MimeMessage(lSession);
lMsg.setFrom(new InternetAddress(email.getFromAdr()));
lMsg.setRecipients(javax.mail.Message.RecipientType.TO,
InternetAddress.parse(email.getToAdr()));
if (email.getCC() != null)
lMsg.setRecipients(javax.mail.Message.RecipientType.CC,
InternetAddress.parse(email.getCC()));
/*
* if (bcc()!=null) lMsg.setRecipients(Message.RecipientType.BCC,
* InternetAddress.parse(bcc())); lMsg.setHeader("X-Mailer", "JavaMail");
* lMsg.setSentDate(new Date()); lMsg.setSubject(subject());
* lMsg.setText(content()); lMsg.saveChanges(); Transport
* lTransport=lAuth.authenticate(); if (lTransport!=null)
* lTransport.sendMessage(lMsg,lMsg.getAllRecipients()); } else {
* Transport.send(lMsg); }
*/
}
/**
* retrieve pop3 mail from the given host
*
* #param pop3host
* #param user
* #param password
* #throws Exception
*/
public List<EMail> retrievePop3Mail(EMailManager eMailmanager,
Configuration configuration) throws Exception {
List<EMail> result = new ArrayList<EMail>();
Properties lProps = new Properties();
Session session = Session.getDefaultInstance(lProps);
Store store = session.getStore("pop3");
File attachmentDirectory = (File) configuration.toMap().get(
"attachmentDirectory");
// get a pop3 authentication
Authentication auth = new Authentication(configuration,
Authentication.AuthenticationType.pop3);
store.connect(auth.host, auth.user, auth.password);
Folder remoteInbox = store.getFolder("INBOX");
remoteInbox.open(Folder.READ_WRITE);
javax.mail.Message message[] = remoteInbox.getMessages();
if (message.length > 0) {
// get all messages
LOGGER.log(Level.INFO, "Getting " + message.length
+ " messages from POP3 Server '" + store.getURLName() + "'");
for (int i = 0; i < message.length; i++) {
if (!message[i].isSet(Flags.Flag.DELETED)) {
EMail email = eMailmanager.create();
String mailInput = this.fullMail(message[i]);
// System.out.print(mailInput);
ByteArrayInputStream mailStream = new ByteArrayInputStream(
mailInput.getBytes());
this.parseMessage(email, mailStream, attachmentDirectory);
result.add(email);
message[i].setFlag(Flags.Flag.DELETED, true);
}
} // for
} // if
remoteInbox.close(true);
store.close();
return result;
}
/**
* parse the Message into the given EMail
*
* #param email
* #param fileName
* #param attachmentDirectory
*
* #throws Exception
*/
public void parseMessage(EMail email, String fileName,
String attachmentDirectory) throws Exception {
parseMessage(email, new File(fileName), new File(attachmentDirectory));
}
/**
* strip the brackets
*
* #param addressList
* #return
*/
public String stripBrackets(MailboxList addressList) {
String result = null;
if (addressList != null) {
result = addressList.toString();
if (result.startsWith("[") && result.endsWith("]")) {
result = result.substring(1, result.length() - 1);
}
}
return result;
}
/**
* parse the Message from the given file into the given e-mail using the given
* attachmentDirectory
*
* #param email
* #param file
* #param attachmentDirectory
* #throws Exception
*/
public void parseMessage(EMail email, File file, File attachmentDirectory)
throws Exception {
if (!file.canRead() || (!file.isFile()))
throw new IllegalArgumentException(file.getCanonicalPath()
+ " is not a readable file");
// Get stream from file
FileInputStream fis = new FileInputStream(file);
parseMessage(email, fis, attachmentDirectory);
}
/**
* parse the Message from the given file into the given e-mail using the given
* attachmentDirectory
*
* #param email
* #param emailInputStream
* #param attachmentDirectory
* #throws Exception
*/
public void parseMessage(EMail email, InputStream eMailInputStream,
File attachmentDirectory) throws Exception {
Message mimeMsg = null;
if (!attachmentDirectory.isDirectory())
throw new IllegalArgumentException(attachmentDirectory.getCanonicalPath()
+ " is not a directory");
txtBody = new StringBuffer();
htmlBody = new StringBuffer();
attachments = new ArrayList<Entity>();
Exception ex = null;
try {
// Create message with stream from file
// If you want to parse String, you can use:
// Message mimeMsg = new Message(new
// ByteArrayInputStream(mimeSource.getBytes()));
MessageServiceFactory factory = MessageServiceFactory.newInstance();
MessageBuilder msgBuilder = factory.newMessageBuilder();
try {
mimeMsg = msgBuilder.parseMessage(eMailInputStream);
} catch (Throwable th) {
LOGGER.log(Level.SEVERE,th.getClass().getName());
LOGGER.log(Level.SEVERE,th.getMessage());
}
if (mimeMsg == null) {
LOGGER.log(Level.SEVERE, "could not read mime msg:\n",
this.fromInputStream(eMailInputStream));
return;
}
// Get some standard headers
if (mimeMsg.getTo() != null)
email.setToAdr(stripBrackets(mimeMsg.getTo().flatten()));
email.setFromAdr(stripBrackets(mimeMsg.getFrom()));
email.setSubject(mimeMsg.getSubject());
email.setSendDate(mimeMsg.getDate());
email.setEMailId(mimeMsg.getMessageId());
LOGGER.log(Level.INFO, "To: " + email.getToAdr());
LOGGER.log(Level.INFO, "From: " + email.getFromAdr());
LOGGER.log(Level.INFO, "Subject: " + mimeMsg.getSubject());
// Get custom header by name
Field priorityFld = mimeMsg.getHeader().getField("X-Priority");
// If header doesn't found it returns null
if (priorityFld != null) {
// Print header value
LOGGER.log(Level.FINEST, "Priority: " + priorityFld.getBody());
}
// If message contains many parts - parse all parts
if (mimeMsg.isMultipart()) {
Multipart multipart = (Multipart) mimeMsg.getBody();
parseBodyParts(multipart);
// fix mime4j 0.7.2 behaviour to have no separate text/plain part
if (txtBody.length() == 0) {
txtBody.append(multipart.getPreamble());
}
} else {
// If it's single part message, just get text body
String text = getTxtPart(mimeMsg);
txtBody.append(text);
}
email.setContent(txtBody.toString());
// Print text and HTML bodies
if (debug) {
LOGGER.log(Level.FINEST, "Text body: " + txtBody.toString());
LOGGER.log(Level.FINEST, "Html body: " + htmlBody.toString());
}
// loop over attachments
for (Entity attach : attachments) {
writeAttachment(attach, attachmentDirectory);
}
} catch (Exception cex) {
ex = cex;
} finally {
if (eMailInputStream != null) {
try {
eMailInputStream.close();
} catch (IOException ioex2) {
ioex2.printStackTrace();
}
}
}
if (ex != null) {
throw ex;
}
}
/**
* write the given Attachment
*
* #param attach
* #param attachmentDirectory
* #throws IOException
*/
public void writeAttachment(Entity attach, File attachmentDirectory)
throws IOException {
String attName = attach.getFilename();
// Create file with specified name
if (attName == null) {
LOGGER.log(Level.WARNING, "attachment has no file name using 'attachment"
+ attach.hashCode() + "' instead");
attName = "attachment" + attach.hashCode();
}
FileOutputStream fos = new FileOutputStream(new File(attachmentDirectory,
attName));
try {
writeBody(fos, attach.getBody());
} finally {
fos.close();
}
}
/**
* write the given body to the given fileoutput stream
*
* #param fos
* #param body
* #throws IOException
*/
public void writeBody(FileOutputStream fos, Body body) throws IOException {
if (body instanceof SingleBody) {
((SingleBody) body).writeTo(fos);
} else if (body instanceof MessageImpl) {
writeBody(fos, ((MessageImpl) body).getBody());
} else {
LOGGER.log(Level.WARNING, "can't handle body of type "
+ body.getClass().getSimpleName());
}
}
/**
* This method classifies bodyPart as text, html or attached file
*
* #param multipart
* #throws IOException
*/
private void parseBodyParts(Multipart multipart) throws IOException {
// loop over the parts
for (Entity part : multipart.getBodyParts()) {
String mimeType = part.getMimeType();
if (mimeType.equals("text/plain")) {
String txt = getTxtPart(part);
txtBody.append(txt);
} else if (mimeType.equals("text/html")) {
String html = getTxtPart(part);
htmlBody.append(html);
} else if (part.getDispositionType() != null
&& !part.getDispositionType().equals("")) {
// If DispositionType is null or empty, it means that it's multipart,
// not attached file
attachments.add(part);
}
// If current part contains other, parse it again by recursion
if (part.isMultipart()) {
parseBodyParts((Multipart) part.getBody());
}
}
}
/**
*
* #param part
* #return
* #throws IOException
*/
private String getTxtPart(Entity part) throws IOException {
// Get content from body
TextBody tb = (TextBody) part.getBody();
return this.fromInputStream(tb.getInputStream());
}
/**
* Place a String on the clipboard, and make this class the owner of the
* Clipboard's contents.
*
* #param aString
*/
public void setClipboardContents(String aString) {
StringSelection stringSelection = new StringSelection(aString);
Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
clipboard.setContents(stringSelection, this);
}
/**
* get text from the clipboard
*
* #return
* #throws Exception
*/
public String getClipboardText() throws Exception {
Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
String text = (String) clipboard.getData(DataFlavor.stringFlavor);
return text;
}
/**
* get Mail from clipboard
*
* #param email
* #param attachmentDirectory
* #return
* #throws Exception
*/
public boolean getMailFromClipboard(EMail email, File attachmentDirectory)
throws Exception {
String mailText = getClipboardText();
if (mailText == null)
return false;
this.parseMessage(email,
new ByteArrayInputStream(mailText.getBytes("UTF-8")),
attachmentDirectory);
return true;
}
/*
* (non-Javadoc)
*
* #see
* java.awt.datatransfer.ClipboardOwner#lostOwnership(java.awt.datatransfer
* .Clipboard, java.awt.datatransfer.Transferable)
*/
#Override
public void lostOwnership(Clipboard clipboard, Transferable contents) {
}
}

I recommand you to use Token streams.
It is quite straight forward with it.
You can locate attachment by using headers of your multipart section :
Content-Disposition:attachment; filename="toto.txt"
You must be carefull when parsing a header with it ... It can be mail headers or multipart section header....

How to send PDF file from backend Java code?

I have a JSP file, there backend helper class to it. From the back end helper I need to send PDF file to the JSP as an attachment. How can I achieve that?

I would suggest you to use Apache Commons File Upload component. That's probably the best way rather than reinventing the wheel. ;)

Since you haven't told us if you're using any MVC framework or just plain Servlet, I'll go for the basics.
If you want to upload a file, Apache Commons File Upload is the best library that will translate your multipart/form-data encoded HttpServletRequest message and provide you with files uploaded (in InputStream format, mostly preferred).
It's up to you, the developer, to write the data back to a persistent storage of your choice.
The reverse, it's to take the file, get the appropriate MIME-Type, Content-Length (file size), and file data (InputStream, if possible) and render it back to the HttpServletResponse).
This code (fully functional and written by me) does put the file as attachment/inline.
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletResponse;
/**
* #author Buhake Sindi (The Elite Gentleman)
* #since 01 September 2011
*/
public class FileServletRenderer implements ServletRenderer {
private static final int DEFAULT_BUFFER_SIZE = 10240; // 10KB
private static final String OCTECT_STREAM_MIME_TYPE = "application/octect-stream";
private String contentType = OCTECT_STREAM_MIME_TYPE;
private long contentLength;
private String contentDisposition = "inline";
private String fileName;
private InputStream inputStream;
/**
* #return the contentType
*/
public String getContentType() {
return contentType;
}
/**
* #param contentType
* the contentType to set
*/
public void setContentType(String contentType) {
this.contentType = contentType;
}
/**
* #return the contentLength
*/
public long getContentLength() {
return contentLength;
}
/**
* #param contentLength
* the contentLength to set
*/
public void setContentLength(long contentLength) {
this.contentLength = contentLength;
}
/**
* #return the contentDisposition
*/
public String getContentDisposition() {
return contentDisposition;
}
/**
* #param contentDisposition
* the contentDisposition to set
*/
public void setContentDisposition(String contentDisposition) {
this.contentDisposition = contentDisposition;
}
/**
* #return the fileName
*/
public String getFileName() {
return fileName;
}
/**
* #param fileName
* the fileName to set
*/
public void setFileName(String fileName) {
this.fileName = fileName;
}
/**
* #return the inputStream
*/
public InputStream getInputStream() {
return inputStream;
}
/**
* #param inputStream
* the inputStream to set
*/
public void setInputStream(InputStream inputStream) {
this.inputStream = inputStream;
}
public void setFile(File file) throws IOException {
if (file == null) {
throw new IOException("file is null.");
}
setInputStream(new BufferedInputStream(new FileInputStream(file)));
setContentLength(file.length());
}
/*
* (non-Javadoc)
*
* #see org.bfs.bayweb.util.renderer.ServletViewRenderer#render(javax.servlet.
* ServletRequest, javax.servlet.ServletResponse)
*/
public void render(ServletRequest request, ServletResponse response) throws IOException {
// TODO Auto-generated method stub
BufferedOutputStream bos = new BufferedOutputStream(response.getOutputStream());
try {
byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
int inputStreamLength = 0;
int length = 0;
if (contentType == null) {
contentType = request.getServletContext().getMimeType(getFileName());
}
//We couldn't determine Content-Type
if (contentType == null) {
contentType = OCTECT_STREAM_MIME_TYPE;
}
while ((length = getInputStream().read(buffer)) > 0) {
inputStreamLength += length;
bos.write(buffer, 0, length);
}
if (inputStreamLength != getContentLength()) {
setContentLength(inputStreamLength);
}
if (response instanceof HttpServletResponse) {
HttpServletResponse httpResponse = (HttpServletResponse) response;
httpResponse.reset();
httpResponse.setHeader("Content-Type", getContentType());
httpResponse.setHeader("Content-Length", String.valueOf(getContentLength()));
httpResponse.setHeader("Content-Disposition", "\"" + getContentDisposition() + "\""
+ ((getFileName() != null && !getFileName().isEmpty()) ? "; filename=\"" + getFileName() + "\"" : ""));
httpResponse.setHeader("Content-Type", getContentType());
}
// finally
bos.flush();
// clear
} finally {
// TODO Auto-generated catch block
close(bos);
close(getInputStream());
}
}
private void close(Closeable resource) throws IOException {
if (resource != null) {
resource.close();
}
}
}
The most important method is render(HttpServletRequest, HttpServletResponse).