I am trying to create Endpoint for Importing csv file to database with the help of spring batch. But i am not able to do. Can someone help me to do with this. It will be great thankful. I am new learner.
My source code is given below please try to help me to sort this issue.
#RestController
public class MyImportController {
private static final Logger logger = LoggerFactory.getLogger(MyImportController.class);
#Autowired
private JobLauncher jobLauncher;
#Autowired
private Job importUserJob;
#GetMapping(value = "/import/file")
public String uploadFile() {
return "/uploadFile";
}
#PostMapping(value="/import/file", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public String create(#RequestParam("file") MultipartFile multipartFile) throws IOException, JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException {
String paths = "/Users/dilipverma/Desktop/CSV_FILE_READER/src/main/resources/temp/";
File fileToImport = new File(path + multipartFile.getOriginalFilename());
System.out.println(" ::::::::::::::::::: File name is ::---------------------------"+fileToImport);
OutputStream outputStream = new FileOutputStream(fileToImport);
IOUtils.copy(multipartFile.getInputStream(), outputStream);
outputStream.flush();
outputStream.close();
JobExecution jobExecution = jobLauncher.run(importUserJob, new JobParametersBuilder()
.addString("fullPathFileName", fileToImport.getAbsolutePath())
.toJobParameters());
logger.info(" :::::::::::::::::::::: Job Status is :::::::::::::::::: "+jobExecution.getStatus());
return "Done";
}
}
Above is my Controller to accept the CSV File.
//Batch File
#Configuration
#EnableBatchProcessing
public class BatchConfig {
#Bean
public ResourcelessTransactionManager batchTransactionManager(){
return new ResourcelessTransactionManager();
}
#Bean
protected JobRepository jobRepository(ResourcelessTransactionManager batchTransactionManager) throws Exception{
MapJobRepositoryFactoryBean jobRepository = new MapJobRepositoryFactoryBean();
jobRepository.setTransactionManager(batchTransactionManager);
return jobRepository.getObject();
}
#Bean
public JobLauncher jobLauncher(JobRepository jobRepository){
SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
jobLauncher.setJobRepository(jobRepository);
return jobLauncher;
}
}
#Configuration
public class ImportJobConfig {
#Autowired
private JobBuilderFactory jobBuilderFactory;
#Autowired
private StepBuilderFactory stepBuilderFactory;
#Bean
#Scope(value = "step", proxyMode = ScopedProxyMode.TARGET_CLASS)
public FlatFileItemReader<Contact> importReader(#Value("#{jobParameters[fullPathFileName]}") String pathToFile) {
FlatFileItemReader<Contact> reader = new FlatFileItemReader<>();
reader.setResource(new FileSystemResource(pathToFile));
reader.setLineMapper(lineMapper());
return reader;
}
#Bean
public LineMapper<Contact> lineMapper() {
DefaultLineMapper<Contact> defaultLineMapper = new DefaultLineMapper<>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
lineTokenizer.setDelimiter(",");
lineTokenizer.setStrict(false);
lineTokenizer.setNames( new String[] {"email", "contactno"});
BeanWrapperFieldSetMapper<Contact> fieldSetMapper = new BeanWrapperFieldSetMapper<>();
fieldSetMapper.setTargetType(Contact.class);
defaultLineMapper.setFieldSetMapper(fieldSetMapper);
defaultLineMapper.setLineTokenizer(lineTokenizer);
return defaultLineMapper;
}
#Bean
public APSUploadFileItemProcessor processor() {
return new APSUploadFileItemProcessor();
}
#Bean
public Job importUserJob(ItemReader<Contact> importReader,
ItemWriter<Contact> itemWriter) {
return jobBuilderFactory.get("importUserJob")
.incrementer(new RunIdIncrementer())
.flow(step1(importReader,itemWriter))
.end()
.build();
}
#Bean
public Step step1(ItemReader<Contact> importReader,
ItemWriter<Contact> itemWriter) {
return stepBuilderFactory.get("step1")
.<Contact, Contact>chunk(10)
.reader(importReader)
.processor(processor())
.writer(itemWriter)
.build();
}
}
//Processor for Job.
#Component
public class APSUploadFileItemProcessor implements ItemProcessor<Contact, Contact> {
#Override
public Contact process(Contact apsUploadFile){
return apsUploadFile;
}
}
//Storing into database
#Component
public class DBWriter implements ItemWriter<Contact> {
#Autowired
private ContactRepository contactRepository;
#Override
public void write(List<? extends Contact> contacts) throws Exception {
contactRepository.save(contacts);
}
}
Here done below is my Error which i am getting while passing my Csv file.
2020-01-20 15:18:50.859 ERROR 15841 --- [nio-8090-exec-2] o.s.batch.core.step.AbstractStep : Encountered an error executing step step1 in job importUserJob
org.springframework.beans.NotReadablePropertyException: Invalid property 'Id' of bean class [java.util.Collections$UnmodifiableRandomAccessList]: Could not find field for property during fallback access!
at org.springframework.data.util.DirectFieldAccessFallbackBeanWrapper.getPropertyValue(DirectFieldAccessFallbackBeanWrapper.java:58) ~[spring-data-commons-2.1.10.RELEASE.jar:2.1.10.RELEASE]
at org.springframework.data.jpa.repository.support.JpaMetamodelEntityInformation.getId(JpaMetamodelEntityInformation.java:152) ~[spring-data-jpa-2.1.10.RELEASE.jar:2.1.10.RELEASE]
at org.springframework.data.repository.core.support.AbstractEntityInformation.isNew(AbstractEntityInformation.java:42) ~[spring-data-commons-2.1.10.RELEASE.jar:2.1.10.RELEASE]
at org.springframework.data.jpa.repository.support.JpaMetamodelEntityInformation.isNew(JpaMetamodelEntityInformation.java:231) ~[spring-data-jpa-2.1.10.RELEASE.jar:2.1.10.RELEASE]
at org.springframework.data.jpa.repository.support.SimpleJpaRepository.save(SimpleJpaRepository.java:534) ~[spring-data-jpa-2.1.10.RELEASE.jar:2.1.10.RELEASE]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_191]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_191]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_191]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_191]
at org.springframework.data.repository.core.support.RepositoryComposition$RepositoryFragments.invoke(RepositoryComposition.java:359) ~[spring-data-commons-2.1.10.RELEASE.jar:2.1.10.RELEASE]
One way to resolve this issue is save contacts by using contactRepository.saveAll(contacts).
#Override
public void write(List<? extends Contact> contacts) throws Exception {
personRepository.saveAll(contacts);
}
By using this, I got the output.
Related
I have an issue regarding running batch process through a request instead of run it automatically when the application runs.
Normally, when I start the application, batch process automatically is handled and it allows to save all values into database. I don't want to do that.
After I want to make a request defined in controller, batch process will starts. It is what I really want to do.
How can I do that?
Here is the project link : Link
Here is the BatchConfiguration class shown below.
#Configuration // Informs Spring that this class contains configurations
#EnableBatchProcessing // Enables batch processing for the application
#RequiredArgsConstructor
public class BatchConfiguration {
private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
private final UserRepository userRepository;
#Bean
public FlatFileItemReader<UserInput> reader() {
FlatFileItemReader<UserInput> itemReader = new FlatFileItemReader<>();
itemReader.setResource(new FileSystemResource("src/main/resources/MOCK_DATA.csv"));
itemReader.setName("csvReader");
itemReader.setLinesToSkip(1);
itemReader.setLineMapper(lineMapper());
return itemReader;
}
private LineMapper<UserInput> lineMapper() {
DefaultLineMapper<UserInput> lineMapper = new DefaultLineMapper<>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
lineTokenizer.setDelimiter(",");
lineTokenizer.setStrict(false);
lineTokenizer.setNames(
"personId","firstName","lastName","email","gender","birthday","country"
);
BeanWrapperFieldSetMapper<UserInput> fieldSetMapper = new BeanWrapperFieldSetMapper<>();
fieldSetMapper.setTargetType(UserInput.class);
lineMapper.setLineTokenizer(lineTokenizer);
lineMapper.setFieldSetMapper(fieldSetMapper);
return lineMapper;
}
#Bean
public UserProcessor processor() {
return new UserProcessor();
}
#Bean
public RepositoryItemWriter<User> writer() {
RepositoryItemWriter<User> writer = new RepositoryItemWriter<>();
writer.setRepository(userRepository);
writer.setMethodName("save");
return writer;
}
#Bean
public Step step1() {
return stepBuilderFactory.get("csv-step").<UserInput, User>chunk(10)
.reader(reader())
.processor(processor())
.writer(writer())
.listener(stepExecutionListener())
.taskExecutor(taskExecutor())
.build();
}
#Bean
public Job runJob() {
return jobBuilderFactory.get("importuserjob")
.listener(jobExecutionListener())
.flow(step1()).end().build();
}
#Bean
public TaskExecutor taskExecutor() {
SimpleAsyncTaskExecutor asyncTaskExecutor = new SimpleAsyncTaskExecutor();
asyncTaskExecutor.setConcurrencyLimit(10);
return asyncTaskExecutor;
}
#Bean
public UserJobExecutionNotificationListener stepExecutionListener() {
return new UserJobExecutionNotificationListener(userRepository);
}
#Bean
public UserStepCompleteNotificationListener jobExecutionListener() {
return new UserStepCompleteNotificationListener();
}
}
Here is the controller class shown below.
public class BatchController {
private final JobLauncher jobLauncher;
private final Job job;
#PostMapping("/importuserjob")
public ResponseEntity<String> importCsvToDBJob() {
log.info("BatchController | importCsvToDBJob is called");
JobParameters jobParameters = new JobParametersBuilder()
.addLong("startAt", System.currentTimeMillis()).toJobParameters();
try {
jobLauncher.run(job, jobParameters);
} catch (JobExecutionAlreadyRunningException | JobRestartException |
JobInstanceAlreadyCompleteException | JobParametersInvalidException e) {
log.info("BatchController | importCsvToDBJob | error : " + e.getMessage());
e.printStackTrace();
}
return new ResponseEntity<>("Batch Process started!!", HttpStatus.OK);
}
}
To stop the job running automatically set the following config flag
spring:
batch:
job:
enabled: false
you can always set the value at launch if you are running a jar
java -Dspring.batch.job.enabled=false -jar myapp.jar
Not sure if it's just a cut and paste issue, but you are missing a constructor in your controller which is required to set final variables, joblauncher and job. Also I assume your controller has #Controller & #Slf4j annotations
I am trying to store 6 CSV file data in the database. Here I am converting CSV data into another object and going to save it.
To Create the object, I need another field from the rest endpoint as a request body and pass it as a JobParameter.
I need to access that parameter in my processor class. I tried different methods. But I am getting the following errors. Any solution would appreciate.
EL1008E: Property or field 'jobParameters' cannot be found on object of type 'org.springframework.beans.factory.config.BeanExpressionContext' - maybe not public or not valid?
This is my RestEnd Point:
#PostMapping
public void save(#RequestBody DownloadFileRequestDto downloadFileRequestDto) throws IOException {
JobParameters JobParameters = new JobParametersBuilder()
.addString("myParam", downloadFileRequestDto.getMyParam())
.toJobParameters();
try {
JobExecution run = jobLauncher.run(job, JobParameters);
} catch (JobExecutionAlreadyRunningException e) {
throw new RuntimeException(e);
} catch (JobRestartException e) {
throw new RuntimeException(e);
} catch (JobInstanceAlreadyCompleteException e) {
throw new RuntimeException(e);
} catch (JobParametersInvalidException e) {
throw new RuntimeException(e);
}
}
This is my Spring Batch Configuration class:
#Configuration
#EnableBatchProcessing
#AllArgsConstructor
public class SpringBatchConfig {
private JobBuilderFactory jobBuilderFactory;
private StepBuilderFactory stepBuilderFactory;
private UserRepository userRepository;
#Bean
public FlatFileItemReader<InputUser> reader() {
FlatFileItemReader<InputUser> itemReader = new FlatFileItemReader<>();
itemReader.setResource(new FileSystemResource("src/main/resources/users.csv"));
itemReader.setName("csvReader");
itemReader.setLinesToSkip(1);
itemReader.setLineMapper(lineMapper());
return itemReader;
}
private LineMapper<InputUser> lineMapper() {
DefaultLineMapper<InputUser> lineMapper = new DefaultLineMapper<>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
lineTokenizer.setDelimiter(",");
lineTokenizer.setStrict(false);
lineTokenizer.setNames("name", "salary");
BeanWrapperFieldSetMapper<InputUser> fieldSetMapper = new BeanWrapperFieldSetMapper<>();
fieldSetMapper.setTargetType(InputUser.class);
lineMapper.setLineTokenizer(lineTokenizer);
lineMapper.setFieldSetMapper(fieldSetMapper);
return lineMapper;
}
#Bean
public Processor processors() {
return new Processor();
}
public RepositoryItemWriter<User> writer(){
RepositoryItemWriter<User> writer = new RepositoryItemWriter<User>();
writer.setRepository(userRepository);
writer.setMethodName("save");
return writer;
}
#Bean
public Step step1(){
return stepBuilderFactory.get("csv-step").<InputUser, User>chunk(500)
.reader(reader())
.processor(processors())
.writer(writer())
.taskExecutor(taskExecutor())
.build();
}
#Bean
public Job job(){
return jobBuilderFactory.get("importUsers")
.flow(step1())
.end().build();
}
public TaskExecutor taskExecutor(){
SimpleAsyncTaskExecutor simpleAsyncTaskExecutor = new SimpleAsyncTaskExecutor();
simpleAsyncTaskExecutor.setConcurrencyLimit(50);
return simpleAsyncTaskExecutor;
}
}
This is my Processor class, where I need access to my job parameters.
#Component
#Scope("step")
public class Processor implements ItemProcessor<InputUser, User> {
#Value("#{jobParameters['myParam']}")
private String fileName;
public Processor() {
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
#Override
public User process(InputUser item) throws Exception {
Random random = new Random();
return new User(random.nextInt(), item.getName(), item.getSalary());
}
}
Remove #Component and #Scope from the Processor class. In the configuration just declare the bean
#Bean
#StepScope
public Processor processors(#Value("#jobParameters['myParam']}") String myParam) {
return new Processor(myParam);
}
Add a constructor in the Processor class that takes a string.
As is you are creating two beans of Processor type one in the configuration class and one with #Component But when defining the step you are using the method in the configuration that is not step scoped and I think spring is not able to inject the value in it.
I'm testing my spring batch job with restart feature, I'm processing total of nine records with chunk size 5, after processing the first chunk - I'm intentionally failing the second chunk to test the failure scenario. As expected, my batch got failed after the first chunk processed successfully and in my table - batch_job_execution I've the record with execution id and status as FAILED. Now I'm running the restart job by passing the execution id to verify the failed records are processing or not. But I'm getting the below exception when I run the failed job:
2022-05-03 18:58:44,829 [ JOB=scheduler.id_IS_UNDEFINED ] [THREAD=main] ERROR [RestartJobTasklet]
Exception java.lang.IllegalStateException: Existing transaction detected in JobRepository. Please fix this and try again (e.g. remove #Transactional annotations from client).while restart the failed job executionId8
Could you please assist me here - what I'm missing here. Please find my code below:
Appreciated your help in advance!
TestJobConfig.java
#Configuration
#Profile("myJob-config")
public class TestJobConfig {
#Autowired
private JobBuilderFactory jobBuilderFactory;
#Autowired
private StepBuilderFactory stepBuilderFactory;
#Autowired
private MyItemWriter myItemWriter;
#Autowired
private RestartJobTasklet restartJobTasklet;
#Bean("myJob-config")
public Job job(#Qualifier("validateStep") Step validateStep,
#Qualifier("processRecords") Step processRecords) {
Job job = jobBuilderFactory.get("myJob-config")
.incrementer(new RunIdIncrementer())
.start(validateStep)
.on("FAILED")
.end()
.from(validateStep).on("*").to(processRecords)
.end()
.build();
return job;
}
#Bean("restart-myjob")
public Job restartJob(#Qualifier("restartMyJobStep") Step restartMyJobStep) {
return jobBuilderFactory.get("restart-myjob")
.incrementer(new RunIdIncrementer())
.start(restartMyJobStep)
.build();
}
#Bean(name = "restartMyJobStep")
public Step restartMyJobStep() {
return this.stepBuilderFactory.get("restart-failed-job")
.tasklet(restartJobTasklet)
.build();
}
#Bean(name = "processRecords")
public Step processRecords() {
return this.stepBuilderFactory.get("process-csv-records").<Employee, Employee>chunk(5)
.reader(reader())
.writer(itemWriter())
.build();
}
#Bean(name = "validateStep")
public Step validateStep(#Qualifier("validateTasklet") Tasklet validateTasklet) {
return stepBuilderFactory.get("validateStep")
.tasklet(validateTasklet)
.allowStartIfComplete(true)
.build();
}
#Bean(name = "validateTasklet")
public Tasklet validateTasklet() {
return new Tasklet() {
#Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
return RepeatStatus.FINISHED;
}
};
}
#Bean
public FlatFileItemReader<Employee> reader() {
FlatFileItemReader<Employee> flatFileItemReader = new FlatFileItemReader<>();
flatFileItemReader.setLinesToSkip(1);
flatFileItemReader.setResource(new ClassPathResource("/csv/emps.csv"));
DefaultLineMapper<Employee> empDefaultLineMapper = new DefaultLineMapper<>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
lineTokenizer.setNames(new String[]{"id", "firstName", "lastName"});
empDefaultLineMapper.setLineTokenizer(lineTokenizer);
empDefaultLineMapper.setFieldSetMapper(new EmployeeFieldSetMapper());
empDefaultLineMapper.afterPropertiesSet();
flatFileItemReader.setLineMapper(empDefaultLineMapper);
return flatFileItemReader;
}
#Bean
public MyItemWriter<Employee> itemWriter() {
return myItemWriter;
}
}
RestartJobTasklet.java
#Component
public class RestartJobTasklet implements Tasklet, StepExecutionListener {
#Autowired
JobExplorer jobExplorer;
#Autowired
JobOperator jobOperator;
private StepExecution stepExecution;
private JobExecution jobExecution;
#Autowired
private OpsJobProperties props;
#Override
public void beforeStep(StepExecution stepExecution) {
this.stepExecution = stepExecution;
this.jobExecution = stepExecution.getJobExecution();
}
#Override
public RepeatStatus execute(StepContribution stepContribution,
ChunkContext chunkContext) throws Exception {
Long executionId = 8l;
try {
Long restartId = jobOperator.restart(executionId);
JobExecution restartExecution = jobExplorer.getJobExecution(restartId);
} catch (JobRestartException e) {
throw e;
} catch (Exception exception) {
throw exception;
}
return RepeatStatus.FINISHED;
}
#Override
public ExitStatus afterStep(StepExecution stepExecution) {
return ExitStatus.COMPLETED;
}
}
DBConfig.java
#Configuration
public class DBConfig extends DefaultBatchConfigurer {
#Autowired
public JobBuilderFactory jobBuilderFactory;
#Bean
public JobRepository jobRepository(#Autowired DataSource dataSource,
#Autowired PlatformTransactionManager transactionManager) throws Exception {
JobRepositoryFactoryBean jobRepositoryFactory = new JobRepositoryFactoryBean();
jobRepositoryFactory.setDatabaseType(DatabaseType.POSTGRES.name());
jobRepositoryFactory.setDataSource(dataSource);
jobRepositoryFactory.setTransactionManager(transactionManager);
jobRepositoryFactory.setIsolationLevelForCreate("ISOLATION_SERIALIZABLE");
jobRepositoryFactory.setTablePrefix("BATCH_");
jobRepositoryFactory.setMaxVarCharLength(1000);
jobRepositoryFactory.setValidateTransactionState(Boolean.FALSE);
return jobRepositoryFactory.getObject();
}
#Bean()
public DataSource dataSource() {
PGSimpleDataSource pgSimpleDataSource = new PGSimpleDataSource();
pgSimpleDataSource.setServerName("my-db-server");
pgSimpleDataSource.setDatabaseName("test-db");
pgSimpleDataSource.setUser("test");
pgSimpleDataSource.setPassword("test");
return pgSimpleDataSource;
}
#Bean
public JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor(final JobRegistry jobRegistry) {
JobRegistryBeanPostProcessor postProcessor = new JobRegistryBeanPostProcessor();
postProcessor.setJobRegistry(jobRegistry);
return postProcessor;
}
#Bean
public JobOperator jobOperator(final JobLauncher jobLauncher, final JobRepository jobRepository,
final JobRegistry jobRegistry, final JobExplorer jobExplorer) {
final SimpleJobOperator jobOperator = new SimpleJobOperator();
jobOperator.setJobLauncher(jobLauncher);
jobOperator.setJobRepository(jobRepository);
jobOperator.setJobRegistry(jobRegistry);
jobOperator.setJobExplorer(jobExplorer);
return jobOperator;
}
#Bean
public JobExplorer jobExplorer(#Autowired DataSource dataSource) throws Exception {
final JobExplorerFactoryBean bean = new JobExplorerFactoryBean();
bean.setDataSource(dataSource);
bean.setTablePrefix("BATCH_");
bean.setJdbcOperations(new JdbcTemplate(dataSource));
bean.afterPropertiesSet();
return bean.getObject();
}
#Bean
public PlatformTransactionManager transactionManager(#Autowired DataSource dataSource) {
return new DataSourceTransactionManager(dataSource);
}
}
Error Log
2022-05-03 18:58:44,865 [ JOB=scheduler.id_IS_UNDEFINED ] [THREAD=main] ERROR [org.springframework.batch.core.step.AbstractStep]
Encountered an error executing step restart-failed-job in job restart-myjob
java.lang.IllegalStateException: Existing transaction detected in JobRepository. Please fix this and try again (e.g. remove #Transactional annotations from client).
at org.springframework.batch.core.repository.support.AbstractJobRepositoryFactoryBean$1.invoke(AbstractJobRepositoryFactoryBean.java:177)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:215)
at com.sun.proxy.$Proxy68.createJobExecution(Unknown Source)
at org.springframework.batch.core.launch.support.SimpleJobLauncher.run(SimpleJobLauncher.java:137)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:344)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.batch.core.configuration.annotation.SimpleBatchConfiguration$PassthruAdvice.invoke(SimpleBatchConfiguration.java:128)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:215)
at com.sun.proxy.$Proxy73.run(Unknown Source)
at org.springframework.batch.core.launch.support.SimpleJobOperator.restart(SimpleJobOperator.java:283)
at org.springframework.batch.core.launch.support.SimpleJobOperator$$FastClassBySpringCGLIB$$44ee6049.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:689)
at org.springframework.batch.core.launch.support.SimpleJobOperator$$EnhancerBySpringCGLIB$$e5e87de1.restart(<generated>)
You are trying to restart a job from within a tasklet in another job. This tasklet (RestartJobTasklet) is already running in a transaction, and calling a method that results in another transaction being created with JobRepository#createJobExecution down the line. Hence the error. In other words, you should not use the JobRepository in a transactional context.
I am not sure if it's a good idea to create a job to restart another job. Restarting a failed job instance is typically a technical task that does not require a job. I would recommend designing jobs to implement business logic, but not for technical tasks.
That said, if you extract the restart code in a main method, your sample should work as expected.
I want to parse a CSV file to db, and for that I am using Batch framework. But only a small part of my CSV is being saved in the db and I am getting an error saying "Attempt to update step execution id=34 with wrong version (47), where current version is 50" and I am getting it multiple times during a request. Here is my BatchConfig.
#Configuration
#EnableBatchProcessing
#AllArgsConstructor
public class BatchConfiguration {
private JobBuilderFactory jobBuilderFactory;
private StepBuilderFactory stepBuilderFactory;
private BookRepository bookRepository;
//Step1 saving the book info in a db
#Bean
public FlatFileItemReader<Book> reader(){
FlatFileItemReader<Book> itemReader = new FlatFileItemReader<>();
itemReader.setResource(new FileSystemResource("src/main/resources/BX-Books.csv"));
itemReader.setName("csvReader");
itemReader.setLinesToSkip(1);
itemReader.setLineMapper(lineMapper());
itemReader.setStrict(false);
return itemReader;
}
private LineMapper<Book> lineMapper() {
DefaultLineMapper<Book> lineMapper = new DefaultLineMapper<>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
lineTokenizer.setDelimiter(";");
lineTokenizer.setStrict(false);
lineTokenizer.setNames("ISBN","Book-Title","Book-Author","Year-Of-Publication","Publisher","Image-URL-S","Image-URL-M","Image-URL-L");
BeanWrapperFieldSetMapper<Book> fieldSetMapper = new BeanWrapperFieldSetMapper<>();
fieldSetMapper.setTargetType(Book.class);
lineMapper.setLineTokenizer(lineTokenizer);
lineMapper.setFieldSetMapper(fieldSetMapper);
return lineMapper;
}
#Bean
public BookProcessor processor(){ return new BookProcessor(); }
//Whatever data we get here, we just apply save method from the book repository
#Bean
public RepositoryItemWriter<Book> writer(){
RepositoryItemWriter<Book> writer = new RepositoryItemWriter<>();
writer.setRepository(bookRepository);
return writer;
}
#Bean
public Step step1(){
return stepBuilderFactory.get("csv-step").<Book, Book>chunk(100)
.reader(reader())
.processor(processor())
.writer(writer())
.taskExecutor(taskExecutor())
.build();
}
#Bean
public Job job(){
return jobBuilderFactory.get("importBooks")
.flow(step1())
.end().build();
}
#Bean
public ThreadPoolTaskExecutor taskExecutor(){
ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
threadPoolTaskExecutor.setCorePoolSize(10);
threadPoolTaskExecutor.setMaxPoolSize(10);
threadPoolTaskExecutor.afterPropertiesSet();
return threadPoolTaskExecutor;
}
}
I'm using a spring batch to read a CSV file and write it to the DB, using the controller trigger. On starting the application, before I hit from the browser url, I see the print statements from my reader, on the startup. Although it doesn't print it for my processor or writer, which are in separate classes which I have autowired. Is it because the reader is a bean?
I see the print statements from my FlatFileItemReader in the log on the application startup. But the print statements for my processor and writer only show up in the console when I hit the controller url.
I've tried adding spring.batch.job.enabled=false in the application.properties file, but it doesnt stop the execution of the reader bean. How can I prevent auto execution of the reader bean in the SpringBatchConfig class:
SpringBatchConfig class:
#Configuration
#EnableBatchProcessing
public class SpringBatchConfig {
#Autowired
private JobBuilderFactory jobBuilderFactory;
#Autowired
private StepBuilderFactory stepBuilderFactory;
#Autowired
private DataSource dataSource;
#Autowired
private DBWriter writer1;
#Autowired
private Processor processor1;
//Step 1 - CSV to DB
#Bean
public FlatFileItemReader<User> itemReader() {
FlatFileItemReader<User> flatFileItemReader = new FlatFileItemReader<>();
flatFileItemReader.setResource(new FileSystemResource("src/main/resources/users.csv"));
flatFileItemReader.setName("CSV-Reader");
flatFileItemReader.setLinesToSkip(1);
flatFileItemReader.setLineMapper(lineMapper());
System.out.println("inside file reader 1 !!!!!");
return flatFileItemReader;
}
#Bean
public LineMapper<User> lineMapper() {
DefaultLineMapper<User> defaultLineMapper = new DefaultLineMapper<>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
lineTokenizer.setDelimiter(",");
lineTokenizer.setStrict(false);
lineTokenizer.setNames(new String[]{"id", "name", "dept", "salary"});
BeanWrapperFieldSetMapper<User> fieldSetMapper = new BeanWrapperFieldSetMapper<>();
fieldSetMapper.setTargetType(User.class);
defaultLineMapper.setLineTokenizer(lineTokenizer);
defaultLineMapper.setFieldSetMapper(fieldSetMapper);
return defaultLineMapper;
}
#Bean
public Step step1() throws Exception{ // Step 1 - Read CSV and Write to DB
return stepBuilderFactory.get("step1")
.<User,User>chunk(100)
.reader(itemReader())
.processor(processor1)
.writer(writer1)
.build();
}
#Bean
public Job job() throws Exception{
return this.jobBuilderFactory.get("BATCH JOB")
.incrementer(new RunIdIncrementer())
.start(step1())
.build();
}
DBWriter class:
#Component
public class DBWriter implements ItemWriter<User> {
#Autowired
private UserRepository userRepository;
#Override
public void write(List<? extends User> users) throws Exception {
System.out.println("Inside DB Writer");
System.out.println("Data Saved for Users: " + users);
userRepository.save(users);
}
}
Processor class:
#Component
public class Processor implements ItemProcessor<User, User> {
private static final Map<String, String> DEPT_NAMES =
new HashMap<>();
public Processor() {
DEPT_NAMES.put("001", "Technology");
DEPT_NAMES.put("002", "Operations");
DEPT_NAMES.put("003", "Accounts");
}
#Override
public User process(User user) throws Exception {
String deptCode = user.getDept();
String dept = DEPT_NAMES.get(deptCode);
user.setDept(dept);
user.setTime(new Date());
System.out.println(String.format("Converted from [%s] to [%s]", deptCode, dept));
return user;
}
}
Controller Class:
#RestController
#RequestMapping("/load")
public class LoadController {
#Autowired
JobLauncher jobLauncher;
#Autowired
Job job;
#GetMapping("/users")
public BatchStatus load() throws JobParametersInvalidException, JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException {
Map<String, JobParameter> maps = new HashMap<>();
maps.put("time", new JobParameter(System.currentTimeMillis()));
JobParameters parameters = new JobParameters(maps);
JobExecution jobExecution = jobLauncher.run(job, parameters);
System.out.println("JobExecution: " + jobExecution.getStatus());
System.out.println("Batch is Running...");
while (jobExecution.isRunning()) {
System.out.println("...");
}
return jobExecution.getStatus();
}
}
The spring.batch.job.enabled=false property is used to prevent running jobs at application startup.
The method that creates the reader will be still be called at configuration time, so it's normal that you see the print statement. But that does not mean the reader was called inside a running job.