We Keep Coding

Spring Boot Batch DB to CSV Cannot sort the values by Id and show column titles in csv file

I tried to implement an example of Spring Boot Batch Processing from db to csv.
The issue which I cannot solve is based on not sorting all values by id in csv file as well as showing column titles.
Here is the output in csv file. (First value is related with id)
3,9,2013-04-15,Japan,jheino3#mayoclinic.com,Jard,MALE,Heino,87cdda81-45d0-451a-a62f-f8450eae1b64
2,23,1999-01-25,Panama,nloynes2#woothemes.com,Natala,FEMALE,Loynes,24be24e6-525f-42de-855d-52d4fef21608
6,9,2013-02-16,China,rcossans4#harvard.edu,Roseline,FEMALE,Cossans,06f70b0d-2c98-4f46-b933-528499ab91b3
4,8,2014-05-09,Indonesia,jcarlaw1#t.co,Jilleen,FEMALE,Carlaw,c722e6d5-9024-49c5-80e0-c2555f1eb9cc
1,22,2000-08-15,China,gspearing0#flickr.com,Ginnie,FEMALE,Spearing,fa26fa96-97d3-4e8e-856a-fdf07499e13e
5,22,2000-03-18,Indonesia,rgillino6#china.com.cn,Rainer,MALE,Gillino,5302a199-f313-4a24-9550-d643001d9faf
I want all values are sorted by id.
How can I do that?
Here is the batch configuration file shown below.
#Configuration // Informs Spring that this class contains configurations
#EnableBatchProcessing // Enables batch processing for the application
#RequiredArgsConstructor
public class BatchConfiguration {
private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
private final UserRepository userRepository;
Date now = new Date(); // java.util.Date, NOT java.sql.Date or java.sql.Timestamp!
String format1 = new SimpleDateFormat("yyyy-MM-dd'-'HH-mm-ss-SSS",Locale.forLanguageTag("tr-TR")).format(now);
private Resource outputResource = new FileSystemResource("output/customers_" + format1 + ".csv");
#Bean
public RepositoryItemReader<User> reader(){
RepositoryItemReader<User> repositoryItemReader = new RepositoryItemReader<>();
repositoryItemReader.setRepository(userRepository);
repositoryItemReader.setMethodName("findAll");
final HashMap<String, Sort.Direction> sorts = new HashMap<>();
sorts.put("id", Sort.Direction.ASC);
repositoryItemReader.setSort(sorts);
return repositoryItemReader;
}
#Bean
public FlatFileItemWriter<User> writer() {
FlatFileItemWriter<User> writer = new FlatFileItemWriter<>();
writer.setResource(outputResource);
writer.setAppendAllowed(true);
writer.setLineAggregator(new DelimitedLineAggregator<User>() {
{
setDelimiter(",");
setFieldExtractor(new BeanWrapperFieldExtractor<User>() {
{
setNames(new String[]{"id", "age", "birthday", "country", "email", "firstName", "gender", "lastName", "personId"});
}
});
}
});
return writer;
}
#Bean
public UserProcessor processor() {
return new UserProcessor();
}
#Bean
public UserJobExecutionNotificationListener stepExecutionListener() {
return new UserJobExecutionNotificationListener(userRepository);
}
#Bean
public UserStepCompleteNotificationListener jobExecutionListener() {
return new UserStepCompleteNotificationListener();
}
#Bean
public Step step1() {
return stepBuilderFactory.get("csv-step").<User, User>chunk(10)
.reader(reader())
.processor(processor())
.writer(writer())
.listener(stepExecutionListener())
.taskExecutor(taskExecutor())
.build();
}
#Bean
public Job runJob() {
return jobBuilderFactory.get("importuserjob")
.listener(jobExecutionListener())
.flow(step1()).end().build();
}
#Bean
public TaskExecutor taskExecutor() {
SimpleAsyncTaskExecutor asyncTaskExecutor = new SimpleAsyncTaskExecutor();
asyncTaskExecutor.setConcurrencyLimit(10);
return asyncTaskExecutor;
}
}
Here is the link of example : Link

After I added FlatFileHeaderCallback into FlatFileItemWriter, I fixed the issue.
Here is the code snippets shown below.
writer.setHeaderCallback(new FlatFileHeaderCallback() {
#Override
public void writeHeader(Writer writer) throws IOException {
for(int i=0;i<headers.length;i++){
if(i!=headers.length-1)
writer.append(headers[i] + ",");
else
writer.append(headers[i]);
}
}
});

Apache Ignite Cache Store + HikariCP DataSource

I am trying to set up Apache Ignite cache store using PostgreSQL as an external storage.
public class MyCacheStore extends CacheStoreAdapter<String, MyCache> {
private static final String GET_QUERY= "SELECT * FROM ..";
private static final String UPDATE_QUERY = "UPDATE ...";
private static final String DELETE_QUERY = "DELETE FROM ..";
#CacheStoreSessionResource
private CacheStoreSession session;
#Override
public MyCache load(String key) throws CacheLoaderException {
Connection connection = session.attachment();
try (PreparedStatement preparedStatement = connection.prepareStatement(GET_QUERY)) {
// some stuff
}
}
#Override
public void loadCache(IgniteBiInClosure<String, MyCache> clo, Object... args) {
super.loadCache(clo, args);
}
#Override
public void write(Cache.Entry<? extends String, ? extends MyCache> entry) throws CacheWriterException {
Connection connection = session.attachment();
try (PreparedStatement preparedStatement = connection.prepareStatement(UPDATE_QUERY)) {
// some stuff
}
}
#Override
public void delete(Object key) throws CacheWriterException {
Connection connection = session.attachment();
try (PreparedStatement preparedStatement = connection.prepareStatement(DELETE_QUERY)) {
// some stuff
}
}
}
MyCache is a standard class:
public class MyCache implements Serializable {
#QuerySqlField(index = true, name = "id")
private String id;
public MyCache() {
}
public MyCache(String id) {
this.id = id;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
Here is a configuration class
import javax.cache.configuration.Factory;
import javax.cache.configuration.FactoryBuilder;
#Configuration
public class ServiceConfig {
// no problems here
#Bean
#ConfigurationProperties(prefix = "postgre")
DataSource dataSource() {
return DataSourceBuilder
.create()
.build();
}
#Bean
public Ignite igniteInstance(IgniteConfiguration igniteConfiguration) {
return Ignition.start(igniteConfiguration);
}
#Bean
public IgniteConfiguration igniteCfg () {
// some other stuff here
IgniteConfiguration cfg = new IgniteConfiguration();
cfg.setClientMode(true);
CacheConfiguration myCacheConfiguration = new CacheConfiguration("MY_CACHE")
.setIndexedTypes(String.class, MyCache.class)
.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
.setReadThrough(true)
.setReadThrough(true)
.setCacheStoreSessionListenerFactories(new MyCacheStoreSessionListenerFactory(dataSource))
.setCacheStoreFactory(FactoryBuilder.factoryOf(MyCacheStore.class));
cfg.setCacheConfiguration(myCacheConfiguration);
return cfg;
}
private static class MyCacheStoreSessionListenerFactory implements Factory {
DataSource dataSource;
MyCacheStoreSessionListenerFactory(DataSource dataSource) {
this.dataSource = dataSource;
}
#Override
public CacheStoreSessionListener create() {
// Data Source
CacheJdbcStoreSessionListener listener = new CacheJdbcStoreSessionListener();
listener.setDataSource(dataSource);
return listener;
}
}
}
And this is what I get in logs:
...
Caused by: class org.apache.ignite.IgniteCheckedException: Failed to validate cache configuration
(make sure all objects in cache configuration are serializable): MyCache
at org.apache.ignite.internal.processors.cache.GridCacheProcessor$11.applyx(GridCacheProcessor.java:4766)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor$11.applyx(GridCacheProcessor.java:4743)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor.withBinaryContext(GridCacheProcessor.java:4788)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor.cloneCheckSerializable(GridCacheProcessor.java:4743)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor.addCacheOnJoin(GridCacheProcessor.java:818)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor.addCacheOnJoinFromConfig(GridCacheProcessor.java:891)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor.startCachesOnStart(GridCacheProcessor.java:753)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor.start(GridCacheProcessor.java:795)
at org.apache.ignite.internal.IgniteKernal.startProcessor(IgniteKernal.java:1700)
... 77 more
Caused by: class org.apache.ignite.IgniteCheckedException: Failed to serialize object: CacheConfiguration [name=MyCache, grpName=null, memPlcName=null, storeConcurrentLoadAllThreshold=5, rebalancePoolSize=2, rebalanceTimeout=10000, evictPlc=null, evictPlcFactory=null, onheapCache=false, sqlOnheapCache=false, sqlOnheapCacheMaxSize=0, evictFilter=null, eagerTtl=true, dfltLockTimeout=0, nearCfg=null, writeSync=null, storeFactory=javax.cache.configuration.FactoryBuilder$ClassFactory#d87782a1, storeKeepBinary=false, loadPrevVal=false, aff=null, cacheMode=PARTITIONED, atomicityMode=TRANSACTIONAL, backups=0, invalidate=false, tmLookupClsName=null, rebalanceMode=ASYNC, rebalanceOrder=0, rebalanceBatchSize=524288, rebalanceBatchesPrefetchCnt=2, maxConcurrentAsyncOps=500, sqlIdxMaxInlineSize=-1, writeBehindEnabled=false, writeBehindFlushSize=10240, writeBehindFlushFreq=5000, writeBehindFlushThreadCnt=1, writeBehindBatchSize=512, writeBehindCoalescing=true, maxQryIterCnt=1024, affMapper=null, rebalanceDelay=0, rebalanceThrottle=0, interceptor=null, longQryWarnTimeout=3000, qryDetailMetricsSz=0, readFromBackup=true, nodeFilter=null, sqlSchema=null, sqlEscapeAll=false, cpOnRead=true, topValidator=null, partLossPlc=IGNORE, qryParallelism=1, evtsDisabled=false, encryptionEnabled=false]
at org.apache.ignite.marshaller.jdk.JdkMarshaller.marshal0(JdkMarshaller.java:103)
at org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.marshal(AbstractNodeNameAwareMarshaller.java:70)
at org.apache.ignite.marshaller.jdk.JdkMarshaller.marshal0(JdkMarshaller.java:117)
at org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.marshal(AbstractNodeNameAwareMarshaller.java:58)
at org.apache.ignite.internal.util.IgniteUtils.marshal(IgniteUtils.java:10250)
at org.apache.ignite.internal.processors.cache.GridCacheProcessor$11.applyx(GridCacheProcessor.java:4762)
... 85 more
Caused by: java.io.NotSerializableException: com.zaxxer.hikari.HikariDataSource
at java.base/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1185)
at java.base/java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1553)
I have read all official documentation about it and examined many other examples, but can't make it run.
HikariCP is the most popular connection pool library, I can't understand why Ignite throws an exception about not being able to serialize DataSource.
Any advice or idea would be appreciated, thank you!

Since your Cache Store is not serializable, you should not use Factory.factoryOf (which is a no-op wrapper) but instead supply a real serializable factory implementation which will acquire local HikariCP on node and then construct the Cache Store.

Exception in combination of #DirtiesContext and FlywayConfig bean in integration test

I have a problem when adding new test. And the problem is I think related to #DirtiesContext. I tried removing and adding it but nothing works in combination. Test 1 is using Application Context as well.
the following two are running together and no issue.
Test 1
#ActiveProfiles({"aws", "local"})
#RunWith(SpringJUnit4ClassRunner.class)
#SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class UnauthorizedControllerTest {
private static final Logger LOGGER = LoggerFactory.getLogger(UnauthorizedControllerTest.class);
#Autowired
private TestRestTemplate testRestTemplate;
#LocalServerPort
private int port;
#Autowired
private ApplicationContext context;
private Map<Class<?>, List<String>> excludedMethodsPerController;
#Before
public void setUp() {
excludedMethodsPerController = excludedMethodsPerController();
}
#Test
public void contextStarts() {
assertNotNull(context);
}
#Test
public void controllerCall_WithoutAuthorization_ReturnsUnauthorized() {
Map<String, Object> controllerBeans = context.getBeansWithAnnotation(Controller.class);
for (Object controllerInstance : controllerBeans.values()) {
LOGGER.info("Checking controller {}", controllerInstance);
checkController(controllerInstance);
}
}
public void checkController(Object controllerInstance) {
// Use AopUtils for the case that spring wraps the controller in a proxy
Class<?> controllerClass = AopProxyUtils.ultimateTargetClass(controllerInstance);
Method[] allMethods = controllerClass.getDeclaredMethods();
for (Method method : allMethods) {
LOGGER.info("Checking method: {}", method.getName());
if (!isCallable(controllerClass, method)) {
continue;
}
String urlPrefix = urlPrefix(controllerClass);
Mapping mapping = Mapping.of(method.getAnnotations());
for (String url : mapping.urls) {
for (RequestMethod requestMethod : mapping.requestMethods) {
ResponseEntity<String> exchange = exchange(urlPrefix + url, requestMethod);
String message = String.format("Failing %s.%s", controllerClass.getName(), method.getName());
assertEquals(message, HttpStatus.UNAUTHORIZED, exchange.getStatusCode());
}
}
}
}
private ResponseEntity<String> exchange(String apiEndpoint, RequestMethod requestMethod) {
return testRestTemplate.exchange(url(replacePathVariables(apiEndpoint)), HttpMethod.resolve(requestMethod.name()), null, String.class);
}
private String urlPrefix(Class<?> aClass) {
if (!aClass.isAnnotationPresent(RequestMapping.class)) {
return "";
}
RequestMapping annotation = aClass.getAnnotation(RequestMapping.class);
return annotation.value()[0];
}
private String url(String url) {
return "http://localhost:" + port + url;
}
private boolean isCallable(Class<?> controller, Method method) {
return Modifier.isPublic(method.getModifiers())
&& !isExcluded(controller, method)
&& !isExternal(controller);
}
private boolean isExcluded(Class<?> controller, Method method) {
List<String> excludedMethodsPerController = this.excludedMethodsPerController.getOrDefault(controller, new ArrayList<>());
return excludedMethodsPerController.contains(method.getName());
}
private boolean isExternal(Class<?> controller) {
return controller.getName().startsWith("org.spring");
}
private String replacePathVariables(String url) {
return url.replaceAll("\\{[^\\/]+}", "someValue");
}
/**
* There must be a really good reason to exclude the method from being checked.
*
* #return The list of urls that must not be checked by the security
*/
private static Map<Class<?>, List<String>> excludedMethodsPerController() {
Map<Class<?>, List<String>> methodPerController = new HashMap<>();
methodPerController.put(AuthenticationController.class, Collections.singletonList("generateAuthorizationToken"));
methodPerController.put(SystemUserLoginController.class, Arrays.asList("systemUserLogin", "handleException"));
methodPerController.put(ValidationController.class, Collections.singletonList("isValid"));
return methodPerController;
}
}
Test 2
#RunWith(SpringJUnit4ClassRunner.class)
#SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
#ActiveProfiles({"aws", "local"})
public class RoleAdminControllerAuditTest {
private static final String DOMAIN_NAME = "testDomain";
private static final String APP_NAME_1 = "testApp_1";
private static final String APP_NAME_2 = "testApp_2";
private static final String ROLE_NAME = "testRole";
private static final String USER_NAME = "testUser";
#Autowired
AuditRepository auditRepository;
#Autowired
RoleAdminController roleAdminController;
#MockBean
RoleAdminService roleAdminService;
#MockBean
RoleAdminInfoBuilder infoBuilder;
#MockBean
AppInfoBuilder appInfoBuilder;
#MockBean
BoundaryValueService boundaryValueService;
#MockBean
RoleService roleService;
#MockBean
private SecurityService securityService;
private static final String URS_USER = "loggedInUser";
private static final String BOUNDARY_VALUE_KEY = "11";
private static final String BOUNDARY_VALUE_NAME = "Schulenberg";
private String auditEventDate = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
#BeforeClass
public static void setupTestEnv() {
// https://github.com/localstack/localstack/issues/592
System.setProperty("com.amazonaws.sdk.disableCbor", "true");
}
#Before
public void setUp() throws Exception {
auditRepository.clean();
when(securityService.getLoggedInUser()).thenReturn(new TestHelper.FakeUser(URS_USER));
//when(roleService.addRoleToApp(any(), any(), eq(ROLE_NAME))).thenReturn(TestHelper.initRole(ROLE_NAME));
when(boundaryValueService.findBoundaryValueById(eq(123L))).thenReturn(initBoundaryValue(BOUNDARY_VALUE_KEY, BOUNDARY_VALUE_NAME));
when(boundaryValueService.findBoundaryValueById(eq(666L))).thenReturn(initBoundaryValue(BOUNDARY_VALUE_KEY, BOUNDARY_VALUE_NAME));
}
#Test
public void addUserAsRoleAdminLogged() throws UserIsAlreadyRoleAdminException, RoleNotFoundException, BoundaryValueNotFoundException {
User user = initUser(USER_NAME);
List<RoleAdminInfo> roleAdminInfos = getRoleAdminInfos();
roleAdminController.addUserAsRoleAdmin(user, roleAdminInfos);
List<String> result = auditRepository.readAll();
assertEquals("some data", result.toString());
}
#Test
public void removeUserAsRoleAdminLogged() throws RoleNotFoundException, BoundaryValueNotFoundException {
User user = initUser(USER_NAME);
Long roleId = Long.valueOf(444);
Role role = initRole("test-role");
role.setApp(initApp("test-app"));
role.setDomain(initDomain("test-domain"));
when(roleService.getRoleByIdOrThrow(roleId)).thenReturn(role);
roleAdminController.removeUserAsRoleAdmin(user, roleId, Long.valueOf(666));
List<String> result = auditRepository.readAll();
assertEquals("some data", result.toString());
}
#Test
public void removeRoleAdminPermission() throws RoleNotFoundException, BoundaryValueNotFoundException {
User user = initUser(USER_NAME);
List<RoleAdminInfo> roleAdminInfos = getRoleAdminInfos();
roleAdminController.removeRoleAdminPermission(user, roleAdminInfos);
List<String> result = auditRepository.readAll();
assertEquals(1, result.size());
assertEquals("some data", result.toString());
}
private List<RoleAdminInfo> getRoleAdminInfos() {
RoleAdminInfo info1 = initRoleAdminInfo(DOMAIN_NAME, ROLE_NAME, APP_NAME_1);
RoleAdminInfo info2 = initRoleAdminInfo(DOMAIN_NAME, ROLE_NAME, APP_NAME_2);
info1.setBoundaryValueId(123L);
info1.setBoundaryValueKey(BOUNDARY_VALUE_KEY);
info1.setBoundaryValueName(BOUNDARY_VALUE_NAME);
return Arrays.asList(info1, info2);
}
}
Test 3 (newly added one)
#RunWith(SpringJUnit4ClassRunner.class)
#SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = FlywayConfig.class)
#DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS)
#ActiveProfiles({"aws", "local"})
public class BoundaryValueDeltaControllerTest {
private static final String API_V1 = "/api/v1/";
#Autowired
TestRestTemplate testRestTemplate;
#Autowired
private DomainBuilder domainBuilder;
#Autowired
private AppBuilder appBuilder;
#Autowired
private UserBuilder userBuilder;
#Autowired
private DomainAdminBuilder domainAdminBuilder;
#Autowired
private BoundarySetBuilder boundarySetBuilder;
#MockBean
private LoginUserProvider loginUserProvider;
#MockBean
private LoginTokenService loginTokenService;
#MockBean
private BoundaryServiceAdapter serviceAdapter;
#LocalServerPort
private int port;
LoginUserInfo loggedInUser;
#Before
public void setUp() {
clear();
}
#After
public void tearDown() {
clear();
}
#Test
public void updateBoundaryValuesFromApi() throws UrsBusinessException {
Domain domain = domainBuilder.persist();
App app = appBuilder.persist(domain);
BoundarySet boundarySet = boundarySetBuilder.persist(domain);
User user = userBuilder.persist(domain.getAuthor().getUsername());
aLoggedInUser(domain.getAuthor().getUsername());
domainAdminBuilder.persist(user, domain);
mockReadInfoFromApiUsingApp();
ResponseEntity<String> response = callUpdateBoundaryValuesFromApi(domain, boundarySet, app);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getBody());
}
private void mockReadInfoFromApiUsingApp() throws UrsBusinessException {
BoundaryValueInfo boundaryValueInfo = new BoundaryValueInfo();
boundaryValueInfo.setBoundaryValueId(10L);
boundaryValueInfo.setBoundaryValueKey("boundaryValueKey");
boundaryValueInfo.setBoundaryValueName("boundaryValuename");
when(serviceAdapter.readInfoFromApiUsingApp(any(), any(), any())).thenReturn(new BoundaryValueInfo[]{boundaryValueInfo});
}
private ResponseEntity<String> callUpdateBoundaryValuesFromApi(Domain domain, BoundarySet boundarySet, App app) {
String url = url(API_V1 + "domains/" + domain.getName() + "/boundarysets/" + boundarySet.getBoundarySetName() + "/app/" + app.getName()+ "/updatefromapi/");
return testRestTemplate.exchange(url,HttpMethod.GET, null, String.class);
}
private String url(String url) {
return "http://localhost:" + port + url;
}
private void aLoggedInUser(String username) {
Claims claims = Jwts.claims();
claims.put("username", username);
loggedInUser = LoginUserInfo.parse(claims);
when(loginUserProvider.getLoggedInUser()).thenReturn(loggedInUser);
when(loginTokenService.parseToken(any())).thenReturn(loggedInUser);
}
private void clear() {
appBuilder.deleteAll();
boundarySetBuilder.deleteAll();
domainAdminBuilder.deleteAll();
domainBuilder.deleteAll();
userBuilder.deleteAll();
}
}
Flyway config
#TestConfiguration
public class FlywayConfig {
#Bean
public FlywayMigrationStrategy clean() {
return flyway -> {
flyway.clean();
flyway.migrate();
};
}
}
And I am getting below exception while running all together.
java.lang.IllegalStateException: Failed to load ApplicationContext
at org.springframework.test.context.cache.DefaultCacheAwareContextLoaderDelegate.loadContext(DefaultCacheAwareContextLoaderDelegate.java:125)
at org.springframework.test.context.support.DefaultTestContext.getApplicationContext(DefaultTestContext.java.....
Caused by: org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'flywayInitializer' defined in class path resource [org/springframework/boot/autoconfigure/flyway/FlywayAutoConfiguration$FlywayConfiguration.class]: Invocation of init method failed; nested exception is org.flywaydb.core.internal.exception.FlywaySqlException:
Unable to obtain connection from database: Too many connections
---------------------------------------------------------------
SQL State : 08004
Error Code : 1040
Message : Too many connections
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1762)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:593)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:515)
... 49 more
Caused by: org.flywaydb.core.internal.exception.FlywaySqlException:
Unable to obtain connection from database: Too many connections
I am struggling since yesterday's and you might find duplicate but I tried to add the more details today. please guide me here.

you must add the configuration for flyway
flyway.url=jdbc:postgresql://xxx.eu-west-2.rds.amazonaws.com:5432/xxx
flyway.user=postgres
flyway.password=xxx

Having issue in testing with spring cloud stream with kafka binding

I have following Binding
public interface KafkaBinding {
String DATA_OUT = "dataout";
#Output(DATA_OUT)
MessageChannel dataOut();
}
Here is kafka utility
#EnableBinding(KafkaBinding.class)
public class KafkaStreamUtil {
private final MessageChannel out;
public KafkaStreamUtil(KafkaBinding binding) {
this.out = binding .dataOut();
}
public void SendToKafkaTopic(List<data> dataList){
dataList
.stream()
.forEach(this::sendMessgae);
}
private void sendMessgae(Data data) {
Message<Data> message = MessageBuilder
.withPayload(data)
.setHeader(KafkaHeaders.MESSAGE_KEY, data.getRequestId().getBytes())
.build();
try {
this.out.send(message);
} catch (Exception e) {
log.error(e.getMessage(),e);
}
}
My test class
#RunWith(SpringRunner.class)
#ActiveProfiles("test")
#SpringBootTest(classes = {KafkaStreamUtil.class,KafkaBinding.class})
#DirtiesContext(classMode= DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
public class KafkaStreamUtilTest {
private static final String TEST_TOPIC1 = "data";
private static final String GROUP_NAME = "embeddedKafkaApplication";
#ClassRule
public static EmbeddedKafkaRule kafkaRule =
new EmbeddedKafkaRule(1, true, TEST_TOPIC1);
public static EmbeddedKafkaBroker embeddedKafka = kafkaRule.getEmbeddedKafka();
#Autowired
private KafkaStreamUtil kUtil;
#Autowired
private KafkaBinding binding;
#BeforeClass
public static void setupProperties() {
System.setProperty("spring.cloud.stream.kafka.binder.brokers", embeddedKafka.getBrokersAsString());
System.setProperty("spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms", "1000");
System.setProperty("spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde", "org.apache.kafka.common.serialization.Serdes$StringSerde");
System.setProperty("spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde", "org.apache.kafka.common.serialization.Serdes$StringSerde");
System.setProperty("spring.cloud.stream.bindings.dataout.destination", "data");
System.setProperty("spring.cloud.stream.bindings.dataout.producer.header-mode", "raw");
System.setProperty("spring.autoconfigure.exclude","org.springframework.cloud.stream.test.binder.TestSupportBinderAutoConfiguration");
System.setProperty("spring.kafka.consumer.value-deserializer","org.apache.kafka.common.serialization.StringDeserializer");
System.setProperty("spring.cloud.stream.bindings.input.consumer.headerMode","raw");
System.setProperty("spring.cloud.stream.bindings.input.group","embeddedKafkaApplication");
System.setProperty("spring.kafka.consumer.group-id","EmbeddedKafkaIntTest");
}
#Before
public void setUp() throws Exception {
kUtil = new KafkaStreamUtil(binding);
}
#Test
public void sendToKafkaTopic() {
kUtil.SendToKafkaTopic(dataList);
Map<String, Object> consumerProps = KafkaTestUtils.consumerProps(GROUP_NAME, "false", embeddedKafka);
consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
consumerProps.put("key.deserializer", StringDeserializer.class);
consumerProps.put("value.deserializer", StringDeserializer.class);
DefaultKafkaConsumerFactory<String, String> cf = new DefaultKafkaConsumerFactory<>(consumerProps);
Consumer<String, String> consumer = cf.createConsumer();
consumer.subscribe(Collections.singleton(TEST_TOPIC1));
ConsumerRecords<String, String> records = consumer.poll(10_000);
consumer.commitSync();
consumer.close();
Assert.assertNotNull(records);
}
}
I am unable to get binding with Kafka util in test the binding is always null. Please let know what I am missing. I am able to test with springboottest but it is loading all beans I just want to load necessary componens required for this test.

spring batch 3 ItemReader reading CSV file data only one time

Hi I am new to Spring Batch and I referred this link to run the sample program.
I wanted to insert the large amount of CSV file data into MySQL database and through the example I am able to execute it fine with limited number of lines of CSV data.
But when I inserted more number of lines into CSV file and execute the batch program it was not inserting any data in other words it was not reading the file itself and not even showing any error/exception(s). I think I need to configure ItemReader to read for more than one time.
Following is the BatchConfiguration.java code
#Configuration
#EnableBatchProcessing
public class BatchConfiguration {
#Bean
public ItemReader<Student> reader() {
FlatFileItemReader<Student> reader = new FlatFileItemReader<Student>();
reader.setResource(new ClassPathResource("student-data.csv"));
reader.setLineMapper(new DefaultLineMapper<Student>() {{
setLineTokenizer(new DelimitedLineTokenizer() {{
setNames(new String[] {"stdId", "subMarkOne", "subMarkTwo" });
}});
setFieldSetMapper(new BeanWrapperFieldSetMapper<Student>() {{
setTargetType(Student.class);
}});
}});
return reader;
}
#Bean
public ItemWriter<Marksheet> writer(DataSource dataSource) {
JdbcBatchItemWriter<Marksheet> writer = new JdbcBatchItemWriter<Marksheet>();
writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<Marksheet>());
writer.setSql("INSERT INTO marksheet (studentId,totalMark) VALUES (:stdId,:totalSubMark)");
writer.setDataSource(dataSource);
return writer;
}
#Bean
public ItemProcessor<Student, Marksheet> processor() {
return new StudentItemProcessor();
}
#Bean
public Job createMarkSheet(JobBuilderFactory jobs, Step step) {
return jobs.get("createMarkSheet")
.flow(step)
.end()
.build();
}
#Bean
public Step step(StepBuilderFactory stepBuilderFactory, ItemReader<Student> reader,
ItemWriter<Marksheet> writer, ItemProcessor<Student, Marksheet> processor) {
return stepBuilderFactory.get("step")
.<Student, Marksheet> chunk(5)
.reader(reader)
.processor(processor)
.writer(writer)
.build();
}
#Bean
public JdbcTemplate jdbcTemplate(DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
#Bean
public DataSource getDataSource() {
BasicDataSource dataSource = new BasicDataSource();
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl("jdbc:mysql://localhost:3306/concretepage");
dataSource.setUsername("root");
dataSource.setPassword("root123");
return dataSource;
}
main class to run
#ComponentScan
#EnableAutoConfiguration
public class Main {
public static void main(String[] args) {
ApplicationContext ctx = SpringApplication.run(Main.class, args);
List<Marksheet> result = ctx.getBean(JdbcTemplate.class).query("select studentId,totalMark FROM marksheet",
new RowMapper<Marksheet>() {
#Override
public Marksheet mapRow(ResultSet rs, int row) throws SQLException {
return new Marksheet(rs.getString(1), Integer.parseInt(rs.getString(2)));
}
});
System.out.println("Number of Record:"+result.size());
}
}
process class
public class StudentItemProcessor implements ItemProcessor<Student, Marksheet> {
#Override
public Marksheet process(final Student student) throws Exception {
int totalMark = student.getSubMarkOne()+student.getSubMarkTwo();
System.out.println("student id:"+student.getStdId() +" and Total mark:"+ totalMark);
Marksheet marksheet = new Marksheet(student.getStdId(), totalMark);
return marksheet;
}
}
Any help will be appreciated. thanks.