We Keep Coding

How to return Custom RepeatStatus from spring Batch step

Step Class: GenerateReferenceNumber
package com.npst.imps.action;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import com.npst.imps.utils.TransactionResponseData;
public class GenerateReferenceNumber implements Tasklet {
#Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
double rrn= Math.random();
chunkContext.getStepContext().getStepExecution().getJobExecution().getExecutionContext().put("rrn", rrn);
double tid= (double) chunkContext.getStepContext().getStepExecution().getJobExecution().getExecutionContext().get("tid");
chunkContext.getStepContext().getStepExecution().getJobExecution().getExecutionContext().put("trnsactionstatus", "RRN generated for Tid::"+tid+" is "+rrn);
TransactionResponseData transactionResponseData =(TransactionResponseData) chunkContext.getStepContext().getStepExecution().getJobExecution().getExecutionContext().get("transactionResponseData");
transactionResponseData.setRrn(rrn+"");
chunkContext.getStepContext().getStepExecution().getJobExecution().getExecutionContext().put("transactionResponseData", transactionResponseData);
return RepeatStatus.FINISHED;
}
}
Instead of Repeatstatus.FINISHED , how can I return my own defined status and based on them the next step will be decided. Custom status like success, fail, partial etc.
batchjob.xml
<beans:beans xmlns="http://www.springframework.org/schema/batch"
xmlns:beans="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/batch
http://www.springframework.org/schema/batch/spring-batch-3.0.xsd">
<job id="MBSFT">
<step id="PrepareTid" allow-start-if-complete="true" next="PrepareRRN">
<tasklet ref="PrepareTransactionId" />
</step>
<step id="PrepareRRN" allow-start-if-complete="true">
<tasklet ref="GenerateReferenceNumber" />
<next on="COMPLETED" to="IdentifyImpsService" />
</step>
<step id="IdentifyImpsService" allow-start-if-complete="true">
<tasklet ref="IdentifyIMPSRequestType" />
<next on="COMPLETED" to="FetchNBIN" />
</step>
<step id="FetchNBIN" allow-start-if-complete="true">
<tasklet ref="FetchNBINFromIFSC" />
</step>
</job>
</beans:beans>

I suppose this is not possible.
You may put your custom return status into StepExecution, use an ExecutionContextPromotionListener to move property from step to job execution context and than use a JobExecutionDecider to redirect flow.

You Can easily send custom status from StepExecutionListener.
ExitStatus is simple enum. You can pass any string as Exit status.
Decide your batch flow based on exist status.
Below is sample java config code.
//Return status from step listener
#Override
public ExitStatus afterStep(StepExecution stepExecution) {
if(condition1)
return new ExitStatus("CUSTOM_STATUS");
if(condition2)
return new ExitStatus("CUSTOM_STATUS_XYZ");
if(condition3)
return new ExitStatus.COMPLETED
}
#Bean
public Job myJob(JobBuilderFactory jobs) throws Exception {
return jobs.get("myJob")
.start(Step1())
.next(Step2()).on("CUSTOM_STATUS").to(step3())
.next(Step2()).on("CUSTOM_STATUS_XYZ").to(step4())
.next(Step2())).on("COMPLETED").to(step4())
.build()
.listener(listener)
.preventRestart()
.build();
}

Job in spring batch is getting executed multiple times and not stopping

I am trying to read data from cassandra using spring batch, where I have implemented ItemReader, ItemProcessor, and ItemWriter. I am able to read the data , process it and write back the data to the same table. I am creating xml file to execute the job:
xml:
<job id="LoadStatusIndicator" job-repository="jobRepository" restartable="false">
<step id="LoadStatus" next="">
<tasklet>
<chunk reader="StatusReader" processor="ItemProcessor" writer="ItemWriter"
commit-interval="10" />
</tasklet>
</step>
</job>
<beans:bean id="ItemWriter" scope="step"
class="com.batch.writer.ItemWriter">
</beans:bean>
<beans:bean id="ItemProcessor" scope="step"
class="com.batch.processor.ItemProcessor">
</beans:bean>
<beans:bean id="Reader" scope="step"
class="com.reader.ItemReader">
<beans:property name="dataSource" ref="CassandraSource" />
</beans:bean>
applicationcontext.xml:
<beans:bean id="CassandraSource" parent="DataSourceParent">
<beans:property name="url" value="jdbc:cassandra://${cassandra.hostName}:${cassandra.port}/${cassandra.keyspace}" />
<beans:property name="driverClassName" value="org.apache.cassandra.cql.jdbc.CassandraDriver" />
</beans:bean>
reader class:
public static final String query = "SELECT * FROM test_1 allow filtering;";
#Override
public List<Item> read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException
{
List<Item> results = new ArrayList<Item>();
try {
results = cassandraTemplate.select(query,Item.class);
} catch (Exception e) {
e.printStackTrace();
}
return results;
}
writer classs:
#Override
public void write(List<? extends Item> item) throws Exception {
try {
cassandraTemplate.insert(item);
}catch(Exception e){e.printStackTrace();}
But the problem is the whole job is getting executed multiple times , infact it is not stopping at all. I have to force stop the job execution. I have only 2 rows in the table.
I think it is because of the commit-interval defined in xml, but having commit-interval = 10, job executes more than 10 times
According to my understanding, when I run the xml file that means I am running the job only one time, it calls the reader once keeps the data in the run time memory (job repository), calls item processor once (I use list ) and the whole list is inserted at once

SOLVED
In reader class I wrote:
if (results.size!=0)
return results;
else
return null;

spring batch getting data from file and passing to procedure

For a Spring batch project, I need to get the date from a file and then I need to pass this date to a procedure and then run the procedure.
Then the result of the procedure must be written to a csv file.
I tried using listeners but couldn't do this.
Can anyone please tell how this can be achieved or if possible can you share any sample code in github.

First of all, you will need to get the date from your file and store it in the JobExecutionContext. One of the most simple solution would be to create a custom Tasklet to read the text file and store the result String in the context via a StepExecutionListener
This tasklet takes a file parameter and stores the result string with the key file.date :
public class CustomTasklet implements Tasklet, StepExecutionListener {
private String date;
private String file;
#Override
public void beforeStep(StepExecution stepExecution) {}
#Override
public ExitStatus afterStep(StepExecution stepExecution) {
stepExecution.getJobExecution().getExecutionContext().put("file.date", date);
}
#Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
// Read from file using FileUtils (Apache)
date = FileUtils.readFileToString(file);
}
public void setFile(String file) {
this.file = file;
}
}
Use it this way :
<batch:step>
<batch:tasklet>
<bean class="xx.xx.xx.CustomTasklet">
<property name="file" value="${file.path}"></property>
</bean>
</batch:tasklet>
</batch:step>
Then you will use a Chunk with late binding to retrieve the previously stored value (i.e. using #{jobExecutionContext['file.date']}).
The reader will be a StoredProcedureItemReader :
<bean class="org.springframework.batch.item.database.StoredProcedureItemReader">
<property name="dataSource" ref="dataSource" />
<property name="procedureName" value="${procedureName}" />
<property name="fetchSize" value="50" />
<property name="parameters">
<list>
<bean class="org.springframework.jdbc.core.SqlParameter">
<constructor-arg index="0" value="#{jobExecutionContext['file.date']}" />
</bean>
</list>
</property>
<property name="rowMapper" ref="rowMapper" />
<property name="preparedStatementSetter" ref="preparedStatementSetter" />
</bean>
The writer will be a FlatFileItemWriter :
<bean class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
<property name="resource" value="${file.dest.path}" />
<property name="lineAggregator" ref="lineAggregator" />
</bean>

SpringBatch local partitioning restart problems

I am having issues with restart of local partitioning batch. I am throwing RuntimeException on 101st processed item. The job fails, but something is going wrong, because on restart, the job continues from 150th item (and not from the 100th item that it should).
Here is the xml-conf:
<bean id="taskExecutor" class="org.springframework.scheduling.commonj.WorkManagerTaskExecutor" >
<property name="workManagerName" value="springWorkManagers" />
</bean>
<bean id="transactionManager" class="org.springframework.transaction.jta.WebSphereUowTransactionManager"/>
<batch:job id="LocalPartitioningJob">
<batch:step id="masterStep">
<batch:partition step="slaveStep" partitioner="splitPartitioner">
<batch:handler grid-size="5" task-executor="taskExecutor" />
</batch:partition>
</batch:step>
</batch:job>
<batch:step id="slaveStep">
<batch:tasklet transaction-manager="transactionManager">
<batch:chunk reader="partitionReader" processor="compositeItemProcessor" writer="sqlWriter" commit-interval="50" />
<batch:transaction-attributes isolation="SERIALIZABLE" propagation="REQUIRE" timeout="600" />
<batch:listeners>
<batch:listener ref="Processor1" />
<batch:listener ref="Processor2" />
<batch:listener ref="Processor3" />
</batch:listeners>
</batch:tasklet>
</batch:step>
<bean id="jobRepository" class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="transactionManager" ref="transactionManager" />
<property name="tablePrefix" value="${sb.db.tableprefix}" />
<property name="dataSource" ref="ds" />
<property name="maxVarCharLength" value="1000"/>
</bean>
<bean id="transactionManager" class="org.springframework.transaction.jta.WebSphereUowTransactionManager"/>
<jee:jndi-lookup id="ds" jndi-name="${sb.db.jndi}" cache="true" expected-type="javax.sql.DataSource" />
The splitPartitioner implements Partitioner and splits the initial data and saves it to the executionContexts as lists. The processors call remote ejb's to fetch additional data and the sqlWriter is just a org.spring...JdbcBatchItemWriter. PartitionReader code below:
public class PartitionReader implements ItemStreamReader<TransferObjectTO> {
private List<TransferObjectTO> partitionItems;
public PartitionReader() {
}
public synchronized TransferObjectTO read() {
if(partitionItems.size() > 0) {
return partitionItems.remove(0);
} else {
return null;
}
}
#SuppressWarnings("unchecked")
#Override
public void open(ExecutionContext executionContext) throws ItemStreamException {
partitionItems = (List<TransferObjectTO>) executionContext.get("partitionItems");
}
#Override
public void update(ExecutionContext executionContext) throws ItemStreamException {
executionContext.put("partitionItems", partitionItems);
}
#Override
public void close() throws ItemStreamException {
}
}

It seems that I had few misunderstandings of SpringBatch and my buggy code. The first misunderstanding was that I thought that the readCount would be rolled back on RuntimeException. Now I see that this is not the case, but when SpringBatch is incrementing this value and upon step failure, the value is committed.
Related to above, I thought that the update method on ItemStreamReader would be always called, but the executionContext update to database would just be committed or rolled back. But it seems that the update is called only if no errors occur and the executionContext update is always committed.
The third misunderstanding was that the partitioning "master step" would not be re-executed on restart, but only slave steps are re-executed. But actually "master step" is re-executed if "master step"'s slave step would fail. So I guess that master and slave steps are actually somehow handled as a single step.
And then there was my buggy code in the PartitionReader, which was supposed to save db-server disk space. Maybe the partitionItems should not be edited on next()? (Related to the above statements) Anyhow here is the code for the working PartitionReader:
public class PartitionReader implements ItemStreamReader<TransferObjectTO> {
private List<TransferObjectTO> partitionItems;
private int index;
public PartitionReader() {
}
public synchronized TransferObjectTO read() {
if(partitionItems.size() > index) {
return partitionItems.get(index++);
} else {
return null;
}
}
#SuppressWarnings("unchecked")
#Override
public void open(ExecutionContext executionContext) throws ItemStreamException {
partitionItems = (List<TransferObjectTO>) executionContext.get("partitionItems");
index = executionContext.getInt("partitionIndex", 0);
}
#Override
public void update(ExecutionContext executionContext) throws ItemStreamException {
executionContext.put("partitionIndex", index);
}
#Override
public void close() throws ItemStreamException {
}
}

Spring batch integration

I am looking for a guidance/solution with Spring batch integration. I have a directory to which external application will send xml files. My application should read the file content and move the file to another directory.
The application should be able to process the files in parallel.
Thanks in advance.

You can use Spring Integration ftp / sftp combined with Spring Batch:
1.Spring Integration Ftp Configuration :
<bean id="ftpClientFactory"
class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="${host.name}" />
<property name="port" value="${host.port}" />
<property name="username" value="${host.username}" />
<property name="password" value="${host.password}" />
<property name="bufferSize" value="100000"/>
</bean>
<int:channel id="ftpChannel" />
<int-ftp:outbound-channel-adapter id="ftpOutbound"
channel="ftpChannel" remote-directory="/yourremotedirectory/" session-factory="ftpClientFactory" use-temporary-file-name="false" />
2.Create your reader and autowire a service to provide your items if needed :
#Scope("step")
public class MajorItemReader implements InitializingBean{
private List<YourItem> yourItems= null;
#Autowired
private MyService provider;
public YourItem read() {
if ((yourItems!= null) && (yourItems.size() != 0)) {
return yourItems.remove(0);
}
return null;
}
//Reading Items from Service
private void reloadItems() {
this.yourItems= new ArrayList<YourItem>();
// use the service to provide your Items
if (yourItems.isEmpty()) {
yourItems= null;
}
}
public MyService getProvider() {
return provider;
}
public void setProvider(MyService provider) {
this.provider = provider;
}
#Override
public void afterPropertiesSet() throws Exception {
reloadItems();
}
}
3. Create Your Own Item Processor
public class MyProcessor implements
ItemProcessor<YourItem, YourItem> {
#Override
public YourItem process(YourItem arg0) throws Exception {
// Apply any logic to your Item before transferring it to the writer
return arg0;
}
}
4. Create Your Own Writer :
public class MyWriter{
#Autowired
#Qualifier("ftpChannel")
private MessageChannel messageChannel;
public void write(YourItem pack) throws IOException {
//create your file and from your Item
File file = new File("the_created_file");
// Sending the file via Spring Integration Ftp Channel
Message<File> message = MessageBuilder.withPayload(file).build();
messageChannel.send(message);
}
5.Batch Configuration :
<bean id="dataSourcee"
class="org.springframework.jdbc.datasource.DriverManagerDataSource">
<property name="driverClassName" value="" />
<property name="url" value="" />
<property name="username" value="" />
<property name="password" value="" />
</bean>
<bean id="jobRepository"
class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="dataSource" ref="dataSourcee" />
<property name="transactionManager" ref="transactionManagerrr" />
<property name="databaseType" value="" />
</bean>
<bean id="transactionManagerrr"
class="org.springframework.batch.support.transaction.ResourcelessTransactionManager" />
<bean id="jobLauncher"
class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
</bean>
6.Another ApplicationContext file to Configure Your Job :
<context:annotation-config />
<bean id="provider" class="mypackage.MyService" />
<context:component-scan base-package="mypackage" />
<bean id="myReader" class="mypackage.MyReader"
<property name="provider" ref="provider" />
</bean>
<bean id="myWriter" class="mypackage.MyWriter" />
<bean id="myProcessor" class="mypackage.MyProcessor" />
<bean id="mReader"
class="org.springframework.batch.item.adapter.ItemReaderAdapter">
<property name="targetObject" ref="myReader" />
<property name="targetMethod" value="read" />
</bean>
<bean id="mProcessor"
class="org.springframework.batch.item.adapter.ItemProcessorAdapter">
<property name="targetObject" ref="myProcessor" />
<property name="targetMethod" value="process" />
</bean>
<bean id="mWriter"
class="org.springframework.batch.item.adapter.ItemWriterAdapter">
<property name="targetObject" ref="myWriter" />
<property name="targetMethod" value="write" />
</bean>
<batch:job id="myJob">
<batch:step id="step01">
<batch:tasklet>
<batch:chunk reader="mReader" writer="mWriter"
processor="mProcessor" commit-interval="1">
</batch:chunk>
</batch:tasklet>
</batch:step>
</batch:job>
<bean id="myRunScheduler" class="mypackage.MyJobLauncher" />
<task:scheduled-tasks>
<task:scheduled ref="myJobLauncher" method="run"
cron="0 0/5 * * * ?" />
<!-- this will maker the job runs every 5 minutes -->
</task:scheduled-tasks>
7.Finally Configure A launcher to launch your job :
public class MyJobLauncher {
#Autowired
private JobLauncher jobLauncher;
#Autowired
#Qualifier("myJob")
private Job job;
public void run() {
try {
String dateParam = new Date().toString();
JobParameters param = new JobParametersBuilder().addString("date",
dateParam).toJobParameters();
JobExecution execution = jobLauncher.run(job, param);
execution.stop();
} catch (Exception e) {
e.printStackTrace();
}
}