We Keep Coding

Spring Batch Transaction issue

I am implementing application using spring Batch. I am following ItemReader, processor, ItemWriter Approach. I have created Partitioner component which is partitioning Data. Through ItemReader I am reading Data and processing it.
After processing I am writing back data in DB. Once job is finished, I observed there is some data missing in DB. Sometimes execution of the one partition fails. Sometimes Job executes successfully.
Sometimes I get exceptions. Its random.
"java.lang.RuntimeException: java.lang.reflect.UndeclaredThrowableException
is mapped to a primary key column in the database. Updates are not allowed.
org.eclipse.persistence.exceptions.DatabaseException
Internal Exception: java.sql.SQLException: Closed Resultset: getObject
Is there any Thread synchronization or transaction, we need to maintain ?
E.g.
Total no of records - 1000
chunk - 100
Partition1 - 500
Partition2 - 500
This scenario works fine without using partitioning or using MultiThreaded Step
Sample Code -: This code some times works and commit all data and some times fails.. sometimes I observed few data is not committed in DB (even commit count in BATCH_STEP_EXECUTION table is correct). It is kind of random.
#Configuration
#EnableBatchProcessing
#EnableTransactionManagement
#EnableAspectJAutoProxy(proxyTargetClass = true)
public class BatchConfig {
#Autowired
private JobBuilderFactory jobBuilderFactory;
#Autowired
private StepBuilderFactory stepBuilderFactory;
#Bean
public SimpleJobLauncher jobLauncher(JobRepository jobRepository) {
SimpleJobLauncher launcher = new SimpleJobLauncher();
launcher.setJobRepository(jobRepository);
return launcher;
}
#Bean(name = "customerJob")
public Job prepareBatch1() {
return jobBuilderFactory.get("customerJob").incrementer(new RunIdIncrementer()).start(masterStep()).listener(listener())
.build();
}
#Bean
public Step masterStep() {
return stepBuilderFactory.get("masterStep").
partitioner(slaveStep().getName(), partitioner())
.partitionHandler(partitionHandler())
.build();
}
#Bean
public BatchListener listener() {
return new BatchListener();
}
#Bean
#JobScope
public BatchPartitioner partitioner() {
return new BatchPartitioner();
}
#Bean
#StepScope
public PartitionHandler partitionHandler() {
TaskExecutorPartitionHandler taskExecutorPartitionHandler = new TaskExecutorPartitionHandler();
taskExecutorPartitionHandler.setGridSize(2);
taskExecutorPartitionHandler.setTaskExecutor(taskExecutor());
taskExecutorPartitionHandler.setStep(slaveStep());
try {
taskExecutorPartitionHandler.afterPropertiesSet();
} catch (Exception e) {
return taskExecutorPartitionHandler;
}
#Bean
#StepScope
public Step slaveStep() {
return stepBuilderFactory.get("slaveStep").<Customer, CustomerWrapperDTO>chunk(100)
.reader(getReader())
.processor(processor())
.writer(writer())
.build();
}
#Bean
#StepScope
public BatchWriter writer() {
return new BatchWriter();
}
#Bean
#StepScope
public BatchProcessor processor() {
return new BatchProcessor();
}
#Bean
#StepScope
public BatchReader getReader() {
return new BatchReader();
}
#Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(Runtime.getRuntime().availableProcessors());
taskExecutor.setCorePoolSize(Runtime.getRuntime().availableProcessors());
taskExecutor.afterPropertiesSet();
return taskExecutor;
}
}
class CustomerWrapperDTO {
private Address address;
private Customer customer;
//setter getter address, customer
}
Entity
class Customer {
String processStatus; // "U" : unprocessed, "C" : completed, "F" : Failed
}
public class BatchListener implements JobExecutionListener {
#Autowired
private CustomerRepo customerRepo;
public BatchListener() {
}
#Override
public void beforeJob(JobExecution jobExecution) {
List<Customer> customers;
try {
customers = customerRepo.getAllUnprocessedCustomer);
} catch (Exception e) {
throw new CustomerException("failed in BatchListener", e);
}
jobExecution.getExecutionContext().put("customers",customers);
}
#Override
public void afterJob(JobExecution jobExecution) {
}
}
public class BatchPartitioner implements Partitioner {
#Value("#{jobExecutionContext[customers]}")
private List<Customer> customers;
#Override
public Map<String, ExecutionContext> partition(int gridSize) {
Map<String, ExecutionContext> result = new HashMap<>();
int size = customers.size() / gridSize;
List<List<Customer>> lists = IntStream.range(0, customers.size()).boxed()
.collect(Collectors.groupingBy(i -> i / size,
Collectors.mapping(customers::get, Collectors.toList())))
.values().stream().collect(Collectors.toList());
for (int i = 0; i < gridSize; i++) {
ExecutionContext executionContext = new ExecutionContext();
executionContext.putString("name", "Thread_" + i);
executionContext.put("customers", lists.get(i));
result.put("partition" + i, executionContext);
}
return result;
}
}
#Component
#StepScope
Class BatchReader {
private int index;
#Value("#{stepExecutionContext[customers]}")
private List<Customer> customers;
#Override
public Customer read() {
Customer Customer = null;
if (index < customers.size()) {
Customer = customers.get(index);
index++;
} else {
index = 0;
}
return Customer;
}
}
#Component
#StepScope
public class BatchProcessor implements ItemProcessor<Customer, CustomerWrapperDTO> {
public BatchProcessor() {
}
#Override
public BatchProcessor process(Customer item) {
CustomerWrapperDTO customerWrapper = new CustomerWrapperDTO();
try {
// logic to get address
Address address = // API call or some business logic.
item.setAddress(address);
item.setProcessStatus("C"); // Completed
}catch(Exception e) {
item.setProcessStatus("F");// failed
}
//logic to get Address
customerWrapper.setCustomer(item);
return customerWrapper;
}
}
#Component
#StepScope
public class BatchWriter implements ItemWriter<CustomerBatchWrapperDTO> {
#Autowired
private CustmerRepo customerRepo;
#Autowired
private AddressRepo addessRepo;
public BatchWriter() {
}
#Override
public void write(List<? extends CustomerBatchWrapperDTO> items) {
items.forEach(item -> {
try {
if(item.getCustomer() != null) {
customerRepo.merge(item.getCustomer());
}
if(item.getAddress() != null) {
addessRepo.save(item.getAddress());
}
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
}

Spring batch is gonna process by chunks. If a chunk fails (this means at least one item failed to process), the transaction is gonna be rolled back.

The issue is with your item reader:
The implementation is not thread-safe, yet it is used in a multi-threaded step. You should synchronize the read method or wrap your reader in a SynchronizedItemStreamReader
The execution context is not safe to share between threads, and you seem to be sharing items between threads through the execution context. BTW, storing items in the execution context is not recommended even for single threaded cases, because the context will be persisted (possibly several times) during the job execution.

Custom Spring validator not working properly

I'm trying to make artificial CONSTRAINT violation by Spring instead of throwing exception from DB (an expert sad DB-produced errors have high performance cost):
import javax.validation.ConstraintViolation;
import javax.validation.Validator;
#Component
public class AccountValidator implements org.springframework.validation.Validator {
#Autowired
private Validator validator;
private final AccountService accountService;
public AccountValidator(#Qualifier("accountServiceAlias")AccountService accountService) {
this.accountService = accountService;
}
#Override
public boolean supports(Class<?> clazz) {
return AccountRequestDTO.class.equals(clazz);
}
#Override
public void validate(Object target, Errors errors) {
Set<ConstraintViolation<Object>> validates = validator.validate(target);
for (ConstraintViolation<Object> constraintViolation : validates) {
String propertyPath = constraintViolation.getPropertyPath().toString();
String message = constraintViolation.getMessage();
errors.rejectValue(propertyPath, "", message);
}
AccountRequestDTO account = (AccountRequestDTO) target;
if(accountService.getPhone(account.getPhone()) != null){
errors.rejectValue("phone", "", "Validator in action! This number is already in use.");
}
}
}
However, second part of validate() method never works for reasons I cant understand and always pass a call from controller to be handled in try-catch block throwing exception from DB:
public void saveAccount(AccountRequestDTO accountRequestDTO) throws Exception {
LocalDate birthday = LocalDate.parse(accountRequestDTO.getBirthday());
if (LocalDate.from(birthday).until(LocalDate.now(), ChronoUnit.YEARS) < 18) {
throw new RegistrationException("You must be 18+ to register");
}
Account account = new Account(accountRequestDTO.getName(), accountRequestDTO.getSurname(),
accountRequestDTO.getPhone(), birthday, BCrypt.hashpw
(accountRequestDTO.getPassword(), BCrypt.gensalt(4)));
account.addRole(Role.CLIENT);
try {
accountRepository.save(account);
}
catch (RuntimeException exc) {
throw new PersistenceException("Database exception: this number is already in use.");
}
}
Here's a controller method:
#PostMapping("/confirm")
public String signIn(#ModelAttribute("account") #Valid AccountRequestDTO accountRequestDTO,
BindingResult result, Model model) {
accountValidator.validate(accountRequestDTO, result);
if(result.hasErrors()) {
return "/auth/register";
}
try {
accountService.saveAccount(accountRequestDTO);
}
catch (Exception exc) {
model.addAttribute("message", exc.getMessage());
return "/auth/register";
}
return "/auth/login";
}
At service:
#Transactional(readOnly = true)
public String getPhone(String phone){
return accountRepository.getPhone(phone);
}
JpaRepository query:
#Query("SELECT phone FROM Account accounts WHERE phone=:check")
String getPhone(String check);
Tests are green:
#BeforeAll
static void prepare() {
search = new String("0000000000");
}
#BeforeEach
void set_up() {
account = new Account
("Admin", "Adminov", "0000000000", LocalDate.of(2001, 01, 01), "superadmin");
accountRepository.save(account);
}
#Test
void check_if_phone_presents() {
assertThat(accountRepository.getPhone(search).equals(account.getPhone())).isTrue();
}
#Test
void check_if_phone_not_presents() {
String newPhone = "9999999999";
assertThat(accountRepository.getPhone(newPhone)).isNull();
}
#AfterEach
void tear_down() {
accountRepository.deleteAll();
account = null;
}
#AfterAll
static void clear() {
search = null;
}

You need to register your validator.
After we've defined the validator, we need to map it to a specific
event which is generated after the request is accepted.
This can be done in three ways:
Add Component annotation with name “beforeCreateAccountValidator“.
Spring Boot will recognize prefix beforeCreate which determines the
event we want to catch, and it will also recognize WebsiteUser class
from Component name.
#Component("beforeCreateAccountValidator")
public class AccountValidator implements Validator {
...
}
Create Bean in Application Context with #Bean annotation:
#Bean
public AccountValidator beforeCreateAccountValidator () {
return new AccountValidator ();
}
Manual registration:
#SpringBootApplication
public class SpringDataRestApplication implements RepositoryRestConfigurer {
public static void main(String[] args) {
SpringApplication.run(SpringDataRestApplication.class, args);
}
#Override
public void configureValidatingRepositoryEventListener(
ValidatingRepositoryEventListener v) {
v.addValidator("beforeCreate", new AccountValidator ());
}
}

Mockito - verify condition inside a method that is supposed to throw error

I have the following code that I'm trying to test:
class foo:
public void myFunction(Business business) {
...
if(someCondition) {
disconnect(business);
throw exception("message");
}
}
public void disconnect(Business business) {
...
myService.expireTokens(business);
...
}
foo test:
#Test
public void test(){
Assert.assertThrows(Exception.class, () -> {
service.myFunction(business);
verify(myService, times(1)).expireTokens(any());
});
}
So the problem above is that I can comment out the disconnect() in my if clause and the assertThrows would 'catch' the erorr / pass the test, and if the normal flow goes through, it would still throw the expected error and pass, how can I test the nested disconnect actually went through before throwing?

Move verify(myService, times(1)).expireTokens(any()) out of the Assert.assertThrows function context.
Assert.assertThrows equal next construction:
try {
//run exceptional operation
service.myFunction(business);
//code after exceptional operation will not be executed
verify(myService, times(1)).expireTokens(any());
} catch (Throwable actualThrown) {
if (expectedThrowable.isInstance(actualThrown)) {
return actualThrown;
} else {
throw new AssertionError();
}
}
So the code defined after the exceptional function will not be executed because of raised exception. This is the reason of the incorrect test behavior.
Example of the correct test implementation:
#Test
public void test() {
MyService myService = mock(MyService.class);
TestService testService = new TestService(myService);
Business business = new Business();
Assert.assertThrows(Exception.class, () -> {
testService.myFunction(business);
});
verify(myService, times(1)).expireTokens(any());
}
public class TestService {
private MyService myService;
public TestService(MyService myService) {
this.myService = myService;
}
public void myFunction(Business business) throws Exception {
if(business != null) {
disconnect(business);
throw new Exception("message");
}
}
public void disconnect(Business business) {
myService.expireTokens(business);
}
}
public class MyService {
public void expireTokens(Business business) {
}
}

Spring repository custom methods

I am trying to add some custom functionality to a spring data repository.
Using this as my starting point http://docs.spring.io/spring-data/jpa/docs/current/reference/html/#repositories.single-repository-behaviour I have created the following code:
public interface TableLock<T> {
void checkout(T entity);
void checkin(T entity, boolean cmpltBatch);
}
public interface BatchTableLock extends TableLock<MyEntity> {
}
public class BatchTableLockImpl implements BatchTableLock {
private static final Logger logger = LoggerFactory.getLogger(BatchTableLockImpl.class);
#PersistenceContext(unitName = "mysql")
private EntityManager em;
#Override
public void checkout(MyEntity batch) {
Long id = batch.getMyEntityId();
try {
MyEntity p = em.find(MyEntity.class, id, LockModeType.PESSIMISTIC_WRITE);
if (p == null) {
logger.error("checkout : MyEntity id {} must be valid", id);
throw new PessimisticLockException();
}
if (myCondition is true) {
return;
}
} catch (LockTimeoutException | PessimisticLockException e) {
logger.error("checkout : Unable to get write lock on MyEntity id {}", id, e);
}
throw new PessimisticLockException();
}
#Override
public void checkin(MyEntity batch, boolean cmplt) {
Long id = batch.getMyEntityId();
try {
MyEntity p = em.find(MyEntity.class, id, LockModeType.PESSIMISTIC_WRITE);
if (p == null) {
logger.error("complete : MyEntity id {} must be valid", id);
return;
}
if (this is true) {
if (cmplt) {
yep;
} else {
nope;
}
} else if (cmplt) {
logger.error("complete: Unable to complete MyEntity {} with status.", id);
}
} catch (LockTimeoutException | PessimisticLockException e) {
logger.error("complete : Unable to get write lock on MyEntity id {}", id, e);
}
}
}
#Repository
public interface MyDao extends CrudRepository<MyEntity, BigInteger>, BatchTableLock {
... My queries ...
}
unfortunately I am getting the following error:
org.springframework.data.mapping.PropertyReferenceException: No property checkin found for type MyEntity!
This if I'm not mistaken means that spring is trying to generate a query based on the method 'checkin' and it can't find a field in MyEntity with the name 'checkin'. which is correct there is no such field. how do I make it stop doing this? based on the link above I don't think it should be trying to generate a query for this method, but it seems to be doing it anyway. I may be missing something, that is usually the case, but I don't see what it is.

As stated in the reference documentation section you linked to, you need a MyDaoImpl class that implements the custom methods. I guess the easiest way is to either rename BatchTableLockImpl to that or just create an empty MyDaoImpl extending that class.

Mockito - Spring Mvc Service test (Null pointer)

I'm new to Testing side.I'm using Spring Mvc in my application. I followed some tutorials to write for controller and service Test Case. I'm facing error in service test. Please help !
Service :
#Autowired
private PatientDao patientDao;
#Autowired
private PrefixDao prefixDao;
public Patient createPatient(Patient patient) throws Exception {
patient.setAgeorDob();
return createPatientInSync(patient);
}
private synchronized Patient createPatientInSync(Patient patient)
throws Exception {
try {
Prefix prefix = prefixDao.getPrefixForType(PrefixType.PATIENT);
patient.setPatientNo(prefix.getPrefixedNumber());
patientDao.createPatient(patient); //SAVE PATIENT
prefixDao.incrementPrefix(prefix);
} catch (ConstraintViolationException ex) {
throw new InternalErrorException("Please enter valid data", ex);
} catch (NullPointerException e) {
e.printStackTrace();
throw new InternalErrorException(
"Please create Prefix for Patient", e);
}
return patient;
}
Service Test case:
#ContextConfiguration(locations = {
"classpath:/applicationContext-resources.xml",
"classpath:/applicationContext-service.xml",
"classpath:/applicationContext-dao.xml",
"classpath:/applicationContext.xml" })
#RunWith(SpringJUnit4ClassRunner.class)
public class PatientServiceTest {
#Autowired
#Mock
private PatientDao patientDao;
#InjectMocks
private PatientServiceImpl patientService = new PatientServiceImpl();
private PrefixDao prefixDao;
#Before
public void doSetup() {
patientDao = mock(PatientDao.class);
prefixDao = mock(PrefixDao.class);
// Mockito.mock(PatientDao.class);
}
#Before
public void initMocks() {
MockitoAnnotations.initMocks(this);
}
#Test
public void testSaveUser() throws Exception {
Patient mockPatient = new Patient();
mockPatient.setFirstName("Aravinth");
mockPatient.setSex(Gender.Male);
mockPatient.setAgeOrDob("24");
Prefix prefix = new Prefix();
prefix.setPrefixType(PrefixType.PATIENT);
prefix.setPrefix("Pat-");
prefix.setSequenceNo(23);
when(prefixDao.getPrefixForType(PrefixType.PATIENT)).thenReturn(prefix);
System.out.println(prefix.getSequenceNo());
mockPatient = patientService.createPatient(mockPatient);
assertEquals("Aravinth", mockPatient.getFirstName());
verify(patientDao, times(1)).createPatient(mockPatient);
}
}
Verify times works fine.I got Nullpointer in assertEquals.

Need to #Mock PrefixDao first.
If you are using Junit 5, no need to run initMocks(this). Otherwise you need this: MockitoAnnotations.initMocks(this);
With that, the mockito will wire two mock Dao objects to your service.
Also I don't see you mock action for patientDao.
When(patientDao.create()).thenReturn(...);