I currently have a Spring Integration-JDBC implementation up and running that polls a db table for records and then sends valid records to be processed by Spring Batch. I'm in the process of adding an additional table monitor to the project, and an additional batch job, but I'm uncertain what nuts and bolts of Batch need to be unique to the other task, and what can/should be reused?
Spring Batch Job Setup:
<bean id="jobOperator" class="org.springframework.batch.core.launch.support.SimpleJobOperator">
<property name="jobExplorer">
<bean class="org.springframework.batch.core.explore.support.JobExplorerFactoryBean">
<property name="dataSource" ref="dataSource" />
</bean>
</property>
<property name="jobRepository" ref="jobRepository" />
<property name="jobRegistry" ref="jobRegistry" />
<property name="jobLauncher" ref="jobLauncher" />
</bean>
<bean id="jobRegistry" class="org.springframework.batch.core.configuration.support.MapJobRegistry"/>
<bean id="jobRepository" class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
<property name="transactionManager" ref="transactionManager"/>
</bean>
<bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
</bean>
Should I be making a jobOperator2, JobLauncher2, ... for all of these?
No, those are fine and should be as single instance per application and per job store.
All you need a new job definition for that new task.
See more information in the Reference Manual.
Related
I'm new in Spring. Recently I do try to make spring batch and spring integration work together. I want to have JobListener which will listen for message comes to specific channel and launch Spring Batch Job.
I found example on github(https://github.com/chrisjs/spring-batch-scaling/tree/master/message-job-launch) and I tried to configure some way copy Spring Batch and Spring Integration together and this looks like:
<!--Incomming channel OneToOne-->
<int:channel id="requests-channel"/>
<!--For multiple consumers OneToMany-->
<int:publish-subscribe-channel id="reply-channel"/>
<!--Channel for file adapter-->
<int:channel id="file-adapter-reply-channel"/>
<int:channel id="statuses">
<int:queue capacity="10"/>
</int:channel>
<int:channel id="jobLaunchReplyChannel"/>
<!--Intercept request-->
<int-http:inbound-gateway request-channel="requests-channel"
supported-methods="PUT"
path="/testData/setProfileDescription"
reply-timeout="30000"
reply-channel="reply-channel">
</int-http:inbound-gateway>
<!--Sending HTTP response back to user OR either 'no reply received within timeout'-->
<bean id="profileDescriptionActivator"
class="ru.tcsbank.service.integrations.activators.ProfileDescriptionActivator"/>
<int:service-activator ref="profileDescriptionActivator"
input-channel="requests-channel"
output-channel="reply-channel"
method="httpMessageActivator"/>
<!--Write profile description to file-->
<bean id="custom-file-name-generator"
class="ru.tcsbank.service.integrations.transformers_generators.ProfilesFileAdapterNameGenerator"/>
<file:outbound-channel-adapter channel="file-adapter-reply-channel"
directory="file:out"
filename-generator="custom-file-name-generator"/>
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource" lazy-init="true" destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver"/>
<property name="url" value="jdbc:mysql://localhost:3306/testdb"/>
<property name="username" value="test_user"/>
<property name="password" value="qwerty123"/>
</bean>
<bean id="stepScope" class="org.springframework.batch.core.scope.StepScope">
<property name="autoProxy" value="true"/>
</bean>
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
<property name="dataSource" ref="dataSource"/>
</bean>
<bean id="jobRepositoryInDB" class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="transactionManager" ref="transactionManager"/>
</bean>
<bean id="itemProcessor" class="ru.tcsbank.service.batch_processing.CustomItemProcessor"/>
<bean id="itemReader" class="ru.tcsbank.service.batch_processing.CustomReader" scope="step">
<property name="resource" value="classpath:fileOut/*.csv" />
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<property name="lineTokenizer">
<bean class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="delimiter" value=","/>
<property name="names" value="id,firstName,lastName"/>
</bean>
</property>
<property name="fieldSetMapper">
<bean class="ru.tcsbank.service.batch_processing.ProfileDescriptionLineMapper"/>
</property>
</bean>
</property>
</bean>
<bean id="itemWriter" class="ru.tcsbank.service.batch_processing.CustomWriter"/>
<batch:job id="helloWorldJob" job-repository="jobRepositoryInDB">
<batch:listeners>
<batch:listener ref="jobListener"/>
</batch:listeners>
<batch:step id="step1">
<batch:tasklet>
<batch:chunk reader="itemReader" writer="itemWriter" processor="itemProcessor" commit-interval="10"/>
</batch:tasklet>
</batch:step>
</batch:job>
<int:transformer input-channel="reply-channel" output-channel="file-adapter-reply-channel">
<bean class="ru.tcsbank.service.batch_processing.FileMessageToJobRequest">
<property name="job" ref="helloWorldJob"/>
<property name="fileParameterName" value="input.file.name"/>
</bean>
</int:transformer>
<bean id="jobListener" class="ru.tcsbank.service.batch_processing.CustomJobExecutionListener">
<constructor-arg index="0" ref="notificationSender"/>
</bean>
<batch-int:job-launching-gateway request-channel="reply-channel"
reply-channel="file-adapter-reply-channel"/>
<int:logging-channel-adapter channel="jobLaunchReplyChannel"/>
<int:channel id="notificationsChannel"/>
<int:gateway id="notificationSender"
service-interface="ru.tcsbank.service.batch_processing.NotificationSender"
default-request-channel="notificationsChannel"/>
I expect my helloWorldJob run when(as I understand correctly) my jobListener receives message from notificationsChannel. But it do not work(do not receives message from notificationsChannel) Beyond then it throws error like:
Dispatcher has no subscribers for channel
'application.notificationsChannel'.; nested exception is >org.springframework.integration.MessageDispatchingException: Dispatcher >has no subscribers, failedMessage=GenericMessage [payload=TEST. >Image processing job ran for: 0 minutes, 0 seconds.
It's hard to understand what you would like to achieve with all this custom code, but what I can say, that there is no subscribers for that notificationsChannel in your configuration. You indeed send messages to it via notificationSender gateway, but you don't provide any endpoint to consume that notificationsChannel.
In the sample you mention in the link we have something like this:
<int-jms:outbound-channel-adapter id="notifications" destination-name="notifications"
channel="notificationsChannel"/>
So, messages sent to the notificationsChannel are landed in the notifications queue on JMS broker. Your sample are leaking such a subscriber. Therefore I only can explain a reason of the exception, but definitely can't tell you what do to.
UPDATE
You may not use notificationSender in your solution. Looks like it just a result of the CustomJobExecutionListener. So, if you don't need to listen for job process, just remove that CustomJobExecutionListener and, therefore, this notificationSender declaration together with the notificationsChannel definition.
Everything else you are asking in comments is out of the scope of this SO question. Please, consider to raise those concerns in the separate SO thread.
I am trying to implement Spring batch job for database cleanup.It just delete entry from a table in scheduled way.
First we fetch 10 rows from table.(ItemReader)
Removing these 10 entries from table(ItemWriter)
I have scheduled the batch at 15 minute interval.
When we launch the batch,surprisingly 10 thread tries to read the data from table.
Below is the configuration.
<!-- spring batch context -->
<bean id="jobRepository"
class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
<property name="transactionManager" ref="batchTransactionManager" />
</bean>
<bean id="batchTransactionManager"
class="org.springframework.batch.support.transaction.ResourcelessTransactionManager" />
<bean id="jobLauncher"
class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
<!--<property name="taskExecutor">
<bean class="org.springframework.core.task.SimpleAsyncTaskExecutor" />
</property>-->
</bean>
<bean
class="org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor">
<property name="jobRegistry" ref="jobRegistry" />
</bean>
<bean id="jobRegistry"
class="org.springframework.batch.core.configuration.support.MapJobRegistry" />
<!-- spring batch context -->
<!--<bean id="completionPolicy" class="org.springframework.batch.repeat.policy.DefaultResultCompletionPolicy"/>-->
<batch:job id="csrfTokenCleanUpBatchJob">
<batch:step id="step">
<tasklet>
<chunk reader="csrfTokenReader" writer="csrfTokenWriter" commit-interval="10"></chunk>
</tasklet>
</batch:step>
</batch:job>
<!-- run every 10 seconds -->
<bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
<property name="triggers">
<bean id="cronTrigger" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="jobDetail" ref="jobDetail" />
<property name="cronExpression" value="* 0/15 * * * ?" />
</bean>
</property>
</bean>
<bean id="jobDetail" class="org.springframework.scheduling.quartz.JobDetailBean">
<property name="jobClass" value="com.test.oauth.batch.job.CSRFTokenJobLauncher" />
<property name="group" value="quartz-batch" />
<property name="jobDataAsMap">
<map>
<entry key="jobName" value="csrfTokenCleanUpBatchJob" />
<entry key="jobLocator" value-ref="jobRegistry" />
<entry key="jobLauncher" value-ref="jobLauncher" />
</map>
</property>
</bean>
</beans>
It is all by design you want to process each record. The ItemWriter gets as many records as you want but is bound by the commit-interval. Yours is 1 which means each record is individually committed, I suggest you set it to 50. The processor processes each record by it self until the commit interval is reached then the writer is called. As mentioned yours is 1.
Also, make the read method of ItemReader as synchronised.
I have a Batch Job Configured to read messages from JMS Destination and write to a XML file using Chuck Tasklet. The JMS reader is custom implemented which inturn invokes JMSTemplate's receive method. I am using webMethods Broker as JMS Broker. During Batch run, we observed that the Consumer session created while reading the messages from Broker Destination are not being destroyed up on completion of the batch. They are only destroyed after I shutdown the JVM. I have provided more details below,
JMS Spring XML Configuration :
<bean id="JMS.SourceQueue.JndiTemplate" class="org.springframework.jndi.JndiTemplate">
<property name="environment">
<map>
<entry key="java.naming.provider.url" value="wmjmsnaming://Broker #1#X.X.X.X:6849" />
<entry key="java.naming.factory.initial" value="com.webmethods.jms.naming.WmJmsNamingCtxFactory" />
<entry key="com.webmethods.jms.naming.clientgroup" value="IS-JMS" />
</map>
</property>
</bean>
<bean id="JMS.SourceQueue.JmsConnectionFactory" class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiTemplate" ref="JMS.SourceQueue.JndiTemplate" />
<property name="jndiName" value="SmartBatchConnectionFactory" />
<property name="lookupOnStartup" value="true" />
<property name="cache" value="false" />
<property name="proxyInterface" value="javax.jms.ConnectionFactory" />
</bean>
<bean id="JMS.SourceQueue.ConnectionFactory"
class="org.springframework.jms.connection.CachingConnectionFactory">
<constructor-arg ref="JMS.SourceQueue.JmsConnectionFactory" />
</bean>
<bean id="JMS.SourceQueue.DefaultDestination" class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiTemplate" ref="JMS.SourceQueue.JndiTemplate" />
<property name="jndiName" value="SourceQueue" />
</bean>
<bean id="JMS.SourceQueue.MessageTemplate" class="org.springframework.jms.core.JmsTemplate">
<property name="connectionFactory" ref="JMS.SourceQueue.ConnectionFactory" />
<property name="receiveTimeout" value="10000" />
<property name="sessionTransacted" value="true" />
<property name="defaultDestination" ref="JMS.SourceQueue.DefaultDestination" />
</bean>
Any help on pointing out the actual issue would be great.
Call destroy() on the JMS.SourceQueue.ConnectionFactory at the end of the last step (or otherwise when the job is completed).
Using the caching connection factory is recommended to avoid creating connections/consumers for each message, but it needs to be told when to physically release the resources.
I am using Spring and trying to setup a global transaction spanning over two MS SQL Server DBs. The app is running inside Tomcat 6.
I have these definitions:
<bean id="dataSource1" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
....
</bean>
<bean id="sessionFactory1"
class="org.springframework.orm.hibernate3.annotation.AnnotationSessionFactoryBean">
<property name="dataSource" ref="dataSource1"/>
....
</bean>
<bean id="hibernateTransactionManager1"
class="org.springframework.orm.hibernate3.HibernateTransactionManager">
<property name="sessionFactory">
<ref local="sessionFactory1"/>
</property>
</bean>
<bean id="dataSource2" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
....
</bean>
<bean id="sessionFactory2"
class="org.springframework.orm.hibernate3.annotation.AnnotationSessionFactoryBean">
<property name="dataSource" ref="dataSource2"/>
....
</bean>
<bean id="hibernateTransactionManager2"
class="org.springframework.orm.hibernate3.HibernateTransactionManager">
<property name="sessionFactory">
<ref local="sessionFactory2"/>
</property>
</bean>
Then also, each DAO is linked either to sessionFactory1 or to sessionFactory2.
<bean name="stateHibernateDao" class="com.project.dao.StateHibernateDao">
<property name="sessionFactory" ref="sessionFactory1"/>
</bean>
Also, I recently added these two.
<bean id="atomikosTransactionManager" class="com.atomikos.icatch.jta.UserTransactionManager" init-method="init" destroy-method="close">
<property name="forceShutdown" value="false" />
<property name="transactionTimeout" value="300" />
</bean>
<bean id="atomikosUserTransaction" class="com.atomikos.icatch.jta.UserTransactionImp">
<property name="transactionTimeout" value="300" />
</bean>
I am trying to programmatically manage the global transaction
(this is some old legacy code and I don't want to change it too
much so I prefer keeping this managed programmatically).
So now I have this UserTransaction ut (injected from Spring), so I call ut.begin(), do some DB/DAO operations to the two DBs through the DAOs, then I call ut.commit().
The thing is that even before the ut.commit() call, I can see the data is already committed to the DBs?!
I don't think Atomikos is aware of my two DBs, their data sources, session factories, etc. I don't think it starts any transactions on them. Looks like they are not enlisted at all in the global transaction.
To me it seems that each DB/DAO operation goes to the SQL Server on its own, so SQL Server creates an implicit transaction for just that DAO/DB operation, applies the operation and commits the implicit the transaction.
But 1) and 2) are just guesses of mine.
My questions:
Do I need to start the two DB transactions myself (but OK, this is what I am currently doing and I am trying to get rid of; that's why I am trying to use Atomikos to start with)?
How I can configure all this correctly so that when I call ut.begin() it begins a global transaction to the two DBs and when I call ut.commit() it commits it?
I haven't played with JTA recently so seems to me I am missing something quite basic here. What is it?
Edit 1
<bean id="globalTransactionManager" class="org.springframework.transaction.jta.JtaTransactionManager">
<property name="userTransaction" ref="atomikosUserTransaction"/>
<property name="transactionManager" ref="atomikosTransactionManager" />
<property name="allowCustomIsolationLevels" value="true" />
<property name="transactionSynchronization" value="2" />
</bean>
I'm using WebLogic 10.3.3 with Oracle 11g and face a weird problem with Spring Batch as soon as I'm switching from Spring ResourcelessTransactionManager (which is mainly for testing) to the productive DataSourceTransactionManager. First I used WebLogics default driver oracle.jdbc.xa.client.OracleXADataSource but this one fails because Spring can't set the isolation level - this is also documented here.
I'm fine with that since I don't need global transactions anyway so I switched to oracle.jdbc.driver.OracleDriver. Now I'm getting the error message
ORA-01453: SET TRANSACTION must be first statement of transaction
I don't find a lot of information on this, there was a bug but that should have been fixed in Oracle 7 long time ago. It looks like a transaction is started before (?) the actual job gets added to the JobRepository and is not closed properly or something like that.
JobI was able to solve this by setting the Isolation level for all transactions to READ_COMMITTED. By default, Spring sets that to SERIALIZABLE which is very strict (but perfectly fine). This didn't work on my machine although Oracle should support it:
http://www.oracle.com/technetwork/issue-archive/2005/05-nov/o65asktom-082389.html
Here's my code - first for the configuration:
<bean id="jobRepository" class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="transactionManager" ref="transactionManager" />
<property name="dataSource" ref="dataSource" />
<property name="isolationLevelForCreate" value="ISOLATION_READ_COMMITTED" />
</bean>
...and this is for the job itself (simplified):
public class MyFancyBatchJob {
#Transactional(isolation=Isolation.READ_COMMITTED)
public void addJob() {
JobParameters params = new JobParametersBuilder().toJobParameters();
Job job = jobRegistry.getJob("myFancyJob");
JobExecution execution = jobLauncher.run(job, params);
}
}
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource" >
<property name="driverClassName" value="oracle.jdbc.driver.OracleDriver"></property>
<property name="url" value="jdbc:oracle:thin:<username>/<password>#<host>:1521:<sid>" />
</bean>
<jdbc:initialize-database data-source="dataSource">
<jdbc:script location="org/springframework/batch/core/schema-drop-oracle10g.sql" />
<jdbc:script location="org/springframework/batch/core/schema-oracle10g.sql" />
</jdbc:initialize-database>
<bean id="jobRepository"
class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="dataSource" ref="dataSource" />
<property name="transactionManager" ref="transactionManager" />
<property name="databaseType" value="oracle" />
<property name="tablePrefix" value="BATCH_"/>
<property name="isolationLevelForCreate" value="ISOLATION_DEFAULT"/>
</bean>
<bean id="jobLauncher"
class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
</bean>
/*for spring batch with oracle 10g and 11g
*/