I have two database structures, example:
1- MAIN_DATABASE: USER PASSWORD
2: CUSTOMER DATABASE: CUSTOMER_A CUSTOMER_B CUSTOMER_C
I want to access the main database and after validating the data, redirect to the customer database.
I currently use spring and configure it in applicationContext.xml
Example:
<bean id = "encryptionPassword" class = "utils.EncryptionPasswordSpring" />
<bean id = "dataSource" class = "com.mchange.v2.c3p0.ComboPooledDataSource" destroy-method = "close">
<property name = "driverClass" value = "com.mysql.jdbc.Driver" />
<property name = "user" value = "user" />
<property name = "password" value = "123456" />
<property name = "jdbcUrl" value = "jdbc:mysql://localhost/testdb?useSSL = false" />
</bean>
Any example, suggestion? Thanks.
The below is my code for dynamic datasource with mybatis. one is main ds. another is read ds. Hope it useful to you.
use AbstractRoutingDataSource to define a DynamicDataSource
import java.util.Map;
import javax.sql.DataSource;
import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;
public class DynamicDataSource extends AbstractRoutingDataSource {
#Override
protected DataSource determineTargetDataSource() {
return super.determineTargetDataSource();
}
/**
*/
#Override
protected Object determineCurrentLookupKey() {
return DynamicDataSourceContextHolder.getDataSourceKey();
}
/**
* #param defaultDataSource
*/
public void setDefaultDataSource(Object defaultDataSource) {
super.setDefaultTargetDataSource(defaultDataSource);
}
/**
* #param dataSources
*/
public void setDataSources(Map<Object, Object> dataSources) {
super.setTargetDataSources(dataSources);
DynamicDataSourceContextHolder.addDataSourceKeys(dataSources.keySet());
}
}
use ThreadLocal to switch datasource in context
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
public class DynamicDataSourceContextHolder {
private static final ThreadLocal<String> contextHolder = new ThreadLocal<String>() {
/**
make main as default ds
*/
#Override
protected String initialValue() {
return "main";
}
};
/**
*
*/
private static List<Object> dataSourceKeys = Collections.synchronizedList(new ArrayList<>());
/**
* switch ds
*
* #param key
*/
public static void setDataSourceKey(String key) {
contextHolder.set(key);
}
/**
* get ds
*
* #return
*/
public static String getDataSourceKey() {
return contextHolder.get();
}
/**
* reset ds
*/
public static void clearDataSourceKey() {
contextHolder.remove();
}
/**
* judge if ds existed
*
* #param key
* #return
*/
public static boolean containDataSourceKey(String key) {
return dataSourceKeys.contains(key);
}
/**
* add ds
*
* #param keys
* #return
*/
public static boolean addDataSourceKeys(Collection<? extends Object> keys) {
return dataSourceKeys.addAll(keys);
}
}
inject different datasource via application.yml or application.properties
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.sql.DataSource;
import org.apache.ibatis.plugin.Interceptor;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.PlatformTransactionManager;
#Configuration
public class DataSourceConfig {
#Primary
#Bean
#ConfigurationProperties("spring.datasource.main")
public DataSource main() {
return DataSourceBuilder.create().build();
}
#Bean
#ConfigurationProperties("spring.datasource.read")
public DataSource read() {
return DataSourceBuilder.create().build();
}
#Bean
public DataSource dynamicDataSource(
#Qualifier("main") DataSource main,
#Qualifier("read") DataSource read
) {
Map<Object, Object> targetDataSources = new HashMap<>(2);
targetDataSources.put("main", main);
targetDataSources.put("read", read);
DynamicDataSource dynamicDataSource = new DynamicDataSource();
dynamicDataSource.setDefaultTargetDataSource(main); //default
dynamicDataSource.setDataSources(targetDataSources);
return dynamicDataSource;
}
#Bean
public SqlSessionFactory sqlSessionFactory(
#Qualifier("dynamicDataSource") DataSource dynamicDataSource)
throws Exception {
SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
bean.setDataSource(dynamicDataSource);
bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath*:mappings/**/*.xml"));
return bean.getObject();
}
#Bean(name = "sqlSessionTemplate")
public SqlSessionTemplate sqlSessionTemplate(
#Qualifier("sqlSessionFactory") SqlSessionFactory sqlSessionFactory)
throws Exception {
return new SqlSessionTemplate(sqlSessionFactory);
}
#Bean
public PlatformTransactionManager transactionManager(
#Qualifier("dynamicDataSource") DataSource dynamicDataSource
) {
return new DataSourceTransactionManager(dynamicDataSource);
}
}
define an AOP to control which Dao method use which datasource. Dao is interface to access DB via mybatis.
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.After;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
#Aspect
#Order(-1)
#Component
public class DynamicDataSourceAspect {
private final static Logger logger = LoggerFactory.getLogger(DynamicDataSourceAspect.class);
private final String[] QUERY_PREFIX = {
"select","get","find","query","quickGet"
};
#Pointcut("execution( * com.biz.dao..*.*(..))")
public void daoAspect() {
}
#Before("daoAspect()")
public void beforeDao(JoinPoint point) {
boolean isQueryMethod = isQueryMethod(point.getSignature().getName());
if (isQueryMethod) {
switchDataSource("read");
}
}
#After("daoAspect()")
public void afterDao(JoinPoint point) {
restoreDataSource();
}
//===============================private method
private void switchDataSource(String key) {
if (!DynamicDataSourceContextHolder.containDataSourceKey(key)) {
logger.debug("======>DataSource [{}] doesn't exist, use default DataSource [{}] " + key);
} else {
// switch ds
DynamicDataSourceContextHolder.setDataSourceKey(key);
logger.debug("======>Switch DataSource to " + DynamicDataSourceContextHolder.getDataSourceKey());
}
}
private void restoreDataSource() {
// reset to default ds
DynamicDataSourceContextHolder.clearDataSourceKey();
}
private boolean isQueryMethod(String methodName) {
for (String prefix : QUERY_PREFIX) {
if (methodName.startsWith(prefix)) {
return true;
}
}
return false;
}
}
Configure two beans with two different set of configs in app.props
For one configuration you can use this (beanName = dataSource1):
#Configuration
#EnableRetry
public class DataSourceConfiguration {
#Value("${datasource1.username}")
private String username;
#Value("${datasource1.password}")
private String password;
#Value("${datasource1.url}")
private String connection;
#Bean(name = "dataSource1")
#Primary
public DataSource mainDataSource() {
DriverManagerDataSource dataSource = new DriverManagerDataSource();
dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver");
dataSource.setUrl(connection);
dataSource.setUsername(username);
dataSource.setPassword(password);
return dataSource;
}
}
Related
I am trying to download files from multiple SFTP servers then handle those files. But I can not get the information of remote SFTP server such as: IpAddress, remoteDirectory depending on which file MessageHandler handling. Instead Payload only contains the information of the dowloaded files at local. Here the source code I use from the guide:
How to dynamically define file filter pattern for Spring Integration SFTP Inbound Adapter?
SFTIntegration.java
import com.jcraft.jsch.ChannelSftp.LsEntry;
import java.io.File;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.NullChannel;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.dsl.Pollers;
import org.springframework.integration.dsl.SourcePollingChannelAdapterSpec;
import org.springframework.integration.expression.FunctionExpression;
import org.springframework.integration.file.remote.aop.RotatingServerAdvice;
import org.springframework.integration.file.remote.session.DelegatingSessionFactory;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.scheduling.PollerMetadata;
import org.springframework.integration.sftp.dsl.Sftp;
import org.springframework.integration.sftp.dsl.SftpInboundChannelAdapterSpec;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
import org.springframework.messaging.MessageChannel;
import org.springframework.stereotype.Component;
/**
* flow.
*/
#Configuration
#Component
public class SFTIntegration {
public static final String TIMEZONE_UTC = "UTC";
public static final String TIMESTAMP_FORMAT_OF_FILES = "yyyyMMddHHmmssSSS";
public static final String TEMPORARY_FILE_SUFFIX = ".part";
public static final int POLLER_FIXED_PERIOD_DELAY = 60000;
public static final int MAX_MESSAGES_PER_POLL = 100;
private static final Logger LOG = LoggerFactory.getLogger(SFTIntegration.class);
private static final String CHANNEL_INTERMEDIATE_STAGE = "intermediateChannel";
#Autowired
private ImportHandler importHandler;
/** database access repository */
private final SFTPServerConfigRepo SFTPServerConfigRepo;
#Value("${sftp.local.directory.download:${java.io.tmpdir}/localDownload}")
private String localTemporaryPath;
public SFTIntegration(final SFTPServerConfigRepo SFTPServerConfigRepo) {
this.SFTPServerConfigRepo = SFTPServerConfigRepo;
}
/**
* The default poller with 5s, 100 messages, RotatingServerAdvice and transaction.
*
* #return default poller.
*/
#Bean(name = PollerMetadata.DEFAULT_POLLER)
public PollerMetadata poller() {
return Pollers
.fixedDelay(POLLER_FIXED_PERIOD_DELAY)
.advice(advice())
.maxMessagesPerPoll(MAX_MESSAGES_PER_POLL)
.transactional()
.get();
}
/**
* The direct channel for the flow.
*
* #return MessageChannel
*/
#Bean
public MessageChannel stockIntermediateChannel() {
return new DirectChannel();
}
/**
* Get the files from a remote directory. Add a timestamp to the filename
* and write them to a local temporary folder.
*
* #return IntegrationFlow
*/
#Bean
public IntegrationFlow collectionInboundFlowFromSFTPServer() {
// Source definition
final SftpInboundChannelAdapterSpec sourceSpec = Sftp.inboundAdapter(delegatingSFtpSessionFactory())
.preserveTimestamp(true)
.patternFilter("*.*")
.deleteRemoteFiles(true)
.maxFetchSize(MAX_MESSAGES_PER_POLL)
.remoteDirectory("/")
.localDirectory(new File(localTemporaryPath))
.temporaryFileSuffix(TEMPORARY_FILE_SUFFIX)
.localFilenameExpression(new FunctionExpression<String>(s -> {
final int fileTypeSepPos = s.lastIndexOf('.');
return
DateTimeFormatter
.ofPattern(TIMESTAMP_FORMAT_OF_FILES)
.withZone(ZoneId.of(TIMEZONE_UTC))
.format(Instant.now())
+ "_"
+ s.substring(0, fileTypeSepPos)
+ s.substring(fileTypeSepPos);
}));
// Poller definition
final Consumer<SourcePollingChannelAdapterSpec> collectionInboundPoller = endpointConfigurer -> endpointConfigurer
.id("collectionInboundPoller")
.autoStartup(true)
.poller(poller());
return IntegrationFlows
.from(sourceSpec, collectionInboundPoller)
.transform(File.class, p -> {
// log step
LOG.info("flow=collectionInboundFlowFromSFTPServer, message=incoming file: " + p);
return p;
})
.channel(CHANNEL_INTERMEDIATE_STAGE)
.get();
}
#Bean
public IntegrationFlow collectionIntermediateStageChannel() {
return IntegrationFlows
.from(CHANNEL_INTERMEDIATE_STAGE)
.handle(importHandler)
.channel(new NullChannel())
.get();
}
public DefaultSftpSessionFactory createNewSftpSessionFactory(final SFTPServerConfig pc) {
final DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory(
false);
factory.setHost(pc.getServerIp());
factory.setPort(pc.getPort());
factory.setUser(pc.getUsername());
factory.setPassword(pc.getPassword());
factory.setAllowUnknownKeys(true);
return factory;
}
#Bean
public DelegatingSessionFactory<LsEntry> delegatingSFtpSessionFactory() {
final List<SFTPServerConfig> partnerConnections = SFTPServerConfigRepo.findAll();
if (partnerConnections.isEmpty()) {
return null;
}
final Map<Object, SessionFactory<LsEntry>> factories = new LinkedHashMap<>(10);
for (SFTPServerConfig pc : partnerConnections) {
// create a factory for every key containing server type, url and port
if (factories.get(pc.getKey()) == null) {
factories.put(pc.getKey(), createNewSftpSessionFactory(pc));
}
}
// use the first SF as the default
return new DelegatingSessionFactory<>(factories, factories.values().iterator().next());
}
#Bean
public RotatingServerAdvice advice() {
final List<SFTPServerConfig> sftpConnections = SFTPServerConfigRepo.findAll();
final List<RotatingServerAdvice.KeyDirectory> keyDirectories = new ArrayList<>();
for (SFTPServerConfig pc : sftpConnections) {
keyDirectories
.add(new RotatingServerAdvice.KeyDirectory(pc.getKey(), pc.getServerPath()));
}
return new RotatingServerAdvice(delegatingSFtpSessionFactory(), keyDirectories, true);
}
}
ImportHandler.java
import org.springframework.messaging.Message;
import org.springframework.stereotype.Service;
#Service
public class ImportHandler {
public void handle(Message<?> message) {
System.out.println("Hello " + message);
System.out.println(message.getPayload());
System.out.println(message.getHeaders());
//How can I get the information of remote server Ip address, remoteDirectory here where the file comes from
}
}
If you have any ideas, please let me know. Thank you so much!.
It's not currently supported; please open a new feature request.
I have created a scheduler using thread . The main motive of this scheduler is to execute a particular task everyday at 12am. But it seems to be executing the same task two times(not sure,seems like two different threads executing at same time).
ScheduledTask.java
package com.abc.advertising.scheduler;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import com.heb.advertising.service.ReportEmail;
import com.heb.advertising.service.ReportService;
#Component
public class ScheduledTask {
private static final Logger LOG = Logger.getLogger(ScheduledTask.class.getName());
#Autowired
#Qualifier(value="ReportService")
private ReportService reportService;
#Autowired
#Qualifier(value="reportEmail")
private ReportEmail reportEmail;
SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
public static Date primaryDate=null;
public Date existDate;
#PostConstruct
public void scheduledTask(){
//System.out.println("Hii Scheduled!!");
Calendar timeOfDay = Calendar.getInstance();
timeOfDay.set(Calendar.HOUR_OF_DAY, 00);
timeOfDay.set(Calendar.MINUTE,30);
timeOfDay.set(Calendar.SECOND, 00);
new DailyRunnerDaemon(timeOfDay, new Runnable()
{
#Override
public void run()
{
try
{
reportService.initScheduled();
}
catch(Exception e)
{
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
reportEmail.sentErrorEmail("PA",sw.toString());
LOG.error("An error occurred performing daily Scheduled Task", e);
}
}
}, "Scheduled-Task").start();
}
}
DailyRunnerDaemon.java
package com.abc.advertising.scheduler;
import java.util.Calendar;
import java.util.Date;
import java.util.Timer;
import java.util.TimerTask;
public class DailyRunnerDaemon
{
private final Runnable dailyTask;
private final int hour;
private final int minute;
private final int second;
private final String runThreadName;
public DailyRunnerDaemon(Calendar timeOfDay, Runnable dailyTask, String runThreadName)
{
this.dailyTask = dailyTask;
this.hour = timeOfDay.get(Calendar.HOUR_OF_DAY);
this.minute = timeOfDay.get(Calendar.MINUTE);
this.second = timeOfDay.get(Calendar.SECOND);
this.runThreadName = runThreadName;
}
public void start()
{
startTimer();
}
private void startTimer()
{
new Timer(runThreadName, true).schedule(new TimerTask()
{
#Override
public void run()
{
dailyTask.run();
startTimer();
}
}, getNextRunTime());
}
private Date getNextRunTime()
{
Calendar startTime = Calendar.getInstance();
Calendar now = Calendar.getInstance();
startTime.set(Calendar.HOUR_OF_DAY, hour);
startTime.set(Calendar.MINUTE, minute);
startTime.set(Calendar.SECOND, second);
startTime.set(Calendar.MILLISECOND, 0);
if(startTime.before(now) || startTime.equals(now))
{
startTime.add(Calendar.DATE, 1);
}
return startTime.getTime();
}
}
ApplicationConfiguration.java
package com.abc;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executor;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.context.embedded.FilterRegistrationBean;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.web.SpringBootServletInitializer;
import org.springframework.boot.orm.jpa.EntityScan;
import org.springframework.context.MessageSource;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.PropertySource;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import org.springframework.context.support.ReloadableResourceBundleMessageSource;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.instrument.classloading.InstrumentationLoadTimeWeaver;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.orm.jpa.support.PersistenceAnnotationBeanPostProcessor;
import org.springframework.orm.jpa.vendor.HibernateJpaDialect;
import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.web.filter.DelegatingFilterProxy;
import org.springframework.web.multipart.support.MultipartFilter;
import org.springframework.web.servlet.View;
import org.springframework.web.servlet.view.ContentNegotiatingViewResolver;
import org.springframework.web.servlet.view.json.MappingJackson2JsonView;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.jolbox.bonecp.BoneCPDataSource;
/**
* Application configuration in JavaConfig (replacing what would be the web.xml and spring-context.xml).
*/
#EnableTransactionManagement()
#EnableJpaRepositories(basePackages= {"com.abc"},entityManagerFactoryRef="entityManagerFactoryBean")
#ComponentScan({"com.abc"})
#EnableJpaAuditing
#EntityScan(basePackages={"com.abc.advertising.entity"})
#SpringBootApplication(exclude = {RabbitAutoConfiguration.class})
#PropertySource("classpath:application-${spring.profiles.active}.properties")
public class ApplicationConfiguration extends SpringBootServletInitializer{
private static final String DATABASE = "org.hibernate.dialect.Oracle10gDialect";
private static final String PUNIT_LOCATION = "classpath:META- INF/persistence.xml";
private static final String PUNIT = "punit";
private static final String V2_AUTH_SIGNER = "S3SignerType";
private static Logger LOG = Logger.getLogger(ApplicationConfiguration.class.getName());
private #Value("${cloud.aws.credentials.accessKey}") String accessKey;
private #Value("${cloud.aws.credentials.secretKey}") String secretKey;
private #Value("${cloud.aws.s3.endpoint}") String s3endpoint;
/**
* Spring Boot Runner.
*
* #param args Optional parameters from command-line.
*/
public static void main(String[] args) {
LOG.info("ApplicationConfiguration ....");
PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer = propertyPlaceholderConfigurer();
SpringApplication.run(ApplicationConfiguration.class, args);
}
#Bean
public static PropertySourcesPlaceholderConfigurer propertyPlaceholderConfigurer() {
PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer = new PropertySourcesPlaceholderConfigurer();
return propertySourcesPlaceholderConfigurer;
}
#Override
protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
return application.sources(ApplicationConfiguration.class);
}
#Bean
public MessageSource messageSource() {
ReloadableResourceBundleMessageSource messageSource = new ReloadableResourceBundleMessageSource();
messageSource.setBasenames("classpath:/messages");
messageSource.setDefaultEncoding("UTF-8");
messageSource.setCacheSeconds(3600);
messageSource.setConcurrentRefresh(true);
return messageSource;
}
private #Value("${spring.datasource.primary.driver-class-name}") String driverClassName;
private #Value("${spring.datasource.primary.url}") String primaryDataSourceURL;
private #Value("${spring.datasource.primary.username}") String primaryDataSourUserName;
private #Value("${spring.datasource.primary.password}") String primaryDataSourUserPassword;
#Bean(name="promoDataSource")
#Primary
public DataSource primaryDataSource() {
BoneCPDataSource dataSource = new BoneCPDataSource();
dataSource.setDriverClass(driverClassName);
dataSource.setJdbcUrl(primaryDataSourceURL);
dataSource.setUsername(primaryDataSourUserName);
dataSource.setPassword(primaryDataSourUserPassword);
dataSource.setIdleConnectionTestPeriodInMinutes(60);
dataSource.setIdleMaxAgeInSeconds(300);
dataSource.setMaxConnectionsPerPartition(100);
dataSource.setMinConnectionsPerPartition(10);
dataSource.setPartitionCount(3);
dataSource.setAcquireRetryDelayInMs(10000);
dataSource.setAcquireIncrement(5);
dataSource.setStatementsCacheSize(100);
dataSource.setReleaseHelperThreads(5);
dataSource.setConnectionTimeoutInMs(6000000);
dataSource.setServiceOrder("FIFO");
dataSource.setDefaultAutoCommit(true);
return dataSource;
}
#Bean
#ConfigurationProperties(prefix = "datasource.arbaf")
public DataSource secondaryDataSource() {
return DataSourceBuilder.create().build();
}
#Bean(name="transactionManager")
public JpaTransactionManager jpaTransactionManager() {
JpaTransactionManager transactionManager = new JpaTransactionManager();
transactionManager.setEntityManagerFactory(entityManagerFactoryBean().getObject());
transactionManager.setDataSource(primaryDataSource());
transactionManager.setJpaDialect(getHibernateJpaDialect());
return transactionManager;
}
private HibernateJpaDialect getHibernateJpaDialect(){
HibernateJpaDialect promoPlannerJpaDialect = new HibernateJpaDialect();
return promoPlannerJpaDialect;
}
private HibernateJpaVendorAdapter vendorAdaptor() {
HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
vendorAdapter.setDatabasePlatform(DATABASE);
vendorAdapter.setShowSql(false);
vendorAdapter.setGenerateDdl(false);
return vendorAdapter;
}
#Bean
public LocalContainerEntityManagerFactoryBean entityManagerFactoryBean() {
LocalContainerEntityManagerFactoryBean entityManagerFactoryBean = new LocalContainerEntityManagerFactoryBean();
entityManagerFactoryBean.setJpaVendorAdapter(vendorAdaptor());
entityManagerFactoryBean.setDataSource(primaryDataSource());
/* entityManagerFactoryBean.setPersistenceProviderClass(HibernatePersistenceProvider.class);*/
entityManagerFactoryBean.setPackagesToScan(getClass().getPackage().getName());
/* entityManagerFactoryBean.setJpaProperties(jpaHibernateProperties());*/
entityManagerFactoryBean.setPersistenceXmlLocation(PUNIT_LOCATION);
entityManagerFactoryBean.setPersistenceUnitName(PUNIT);
entityManagerFactoryBean.setLoadTimeWeaver(new InstrumentationLoadTimeWeaver());
return entityManagerFactoryBean;
}
public PersistenceAnnotationBeanPostProcessor getPersistenceAnnotationBeanPostProcessor(){
return new PersistenceAnnotationBeanPostProcessor();
}
/*
* Reorder the filters so that the multipart filter resolves before the spring security filter as prescribe
* by Spring when trying to handle those types of requests.
*/
#Bean
public FilterRegistrationBean multipartFilterRegistration() {
FilterRegistrationBean registration =
new FilterRegistrationBean(new MultipartFilter());
registration.setOrder(0);
return registration;
}
#Bean
public FilterRegistrationBean springSecurityFilterRegistration() {
FilterRegistrationBean registrationBean =
new FilterRegistrationBean(new DelegatingFilterProxy("springSecurityFilterChain"));
registrationBean.setOrder(1);
return registrationBean;
}
#Bean
public ContentNegotiatingViewResolver contentNegotiatingViewResolver() {
ContentNegotiatingViewResolver contentNegotiatingViewResolver = new ContentNegotiatingViewResolver();
contentNegotiatingViewResolver.setOrder(1);
Map<String,String> m=new HashMap<String,String>();
m.put("json", "application/json");
m.put("file", "application/vnd.ms-excel");
contentNegotiatingViewResolver.setMediaTypes(m);
List<View> list=new ArrayList<>();
list.add(new MappingJackson2JsonView());
contentNegotiatingViewResolver.setDefaultViews(list);
return contentNegotiatingViewResolver;
}
}
You are using Spring/SpringBoot, your component is running task periodically, you should consider of using #Scheduled(cron="0 0 0 0 * *")
I am using AbstractRoutingDataSource to create multitenancy in my application. I noticed that after a few redeployments of the webapp from my IDE I eventually get the MySQL error "Too many connections".
After further investigations, I found out that when I run the MySQL command show processlist;, I see that the open connection amount is increased by 10 after each deployment, which probably means that the connection pool is somehow still alive somewhere.
Before I used AbstractRoutingDataSource I used the default spring datasource configuration (using application.properties) and it worked fine.
Here's the Multitenant configuration class:
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.sql.DataSource;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.stream.Collectors;
/**
* Created by Alon Segal on 16/03/2017.
*/
#Configuration
public class MultitenantConfiguration {
#Autowired
private DataSourceProperties properties;
/**
* Defines the data source for the application
*
* #return
*/
#Bean
#ConfigurationProperties(
prefix = "spring.datasource"
)
public DataSource dataSource() {
//Creating datasources map "resolvedDataSources" here
MultitenantDataSource dataSource = new MultitenantDataSource();
dataSource.setDefaultTargetDataSource(defaultDataSource());
dataSource.setTargetDataSources(resolvedDataSources);
// Call this to finalize the initialization of the data source.
dataSource.afterPropertiesSet();
return dataSource;
}
/**
* Creates the default data source for the application
*
* #return
*/
private DataSource defaultDataSource() {
.
.
.
}
}
And the datasource class:
import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;
/**
* Created by Alon Segal on 16/03/2017.
*/
public class MultitenantDataSource extends AbstractRoutingDataSource {
#Override
protected Object determineCurrentLookupKey() {
return TenantContext.getCurrentTenant();
}
}
I also tried to use #Bean(destroyMethod = "close") but there is not close method defined on AbstractRoutingDataSource.
I searched everywhere but could not find and answer. Can someone help me understand what's preventing the connection pool from being released between redeployments?
Thanks in advance.
Ok so I eventually solved the issue by giving up on using Spring's AbstractRoutingDataSource and instead using Hibernate's mechanism for multitenancy based on the solution that can be found in this article.
Long story short
You need to do 3 steps:
Step 1: Create a CurrentTenantIdentifierResolver
#Component
public class TenantIdentifierResolver implements CurrentTenantIdentifierResolver {
#Override
public String resolveCurrentTenantIdentifier() {
String tenantId = TenantContext.getCurrentTenant();
if (tenantId != null) {
return tenantId;
}
return DEFAULT_TENANT_ID;
}
#Override
public boolean validateExistingCurrentSessions() {
return true;
}
}
Step 2: Create a MultiTenantConnectionProvider
#Component
public class MultiTenantConnectionProviderImpl implements MultiTenantConnectionProvider {
#Autowired
private DataSource dataSource;
#Override
public Connection getAnyConnection() throws SQLException {
return dataSource.getConnection();
}
#Override
public void releaseAnyConnection(Connection connection) throws SQLException {
connection.close();
}
#Override
public Connection getConnection(String tenantIdentifie) throws SQLException {
String tenantIdentifier = TenantContext.getCurrentTenant();
final Connection connection = getAnyConnection();
try {
if (tenantIdentifier != null) {
connection.createStatement().execute("USE " + tenantIdentifier);
} else {
connection.createStatement().execute("USE " + DEFAULT_TENANT_ID);
}
}
catch ( SQLException e ) {
throw new HibernateException(
"Could not alter JDBC connection to specified schema [" + tenantIdentifier + "]",
e
);
}
return connection;
}
#Override
public void releaseConnection(String tenantIdentifier, Connection connection) throws SQLException {
try {
connection.createStatement().execute( "USE " + DEFAULT_TENANT_ID );
}
catch ( SQLException e ) {
throw new HibernateException(
"Could not alter JDBC connection to specified schema [" + tenantIdentifier + "]",
e
);
}
connection.close();
}
#SuppressWarnings("rawtypes")
#Override
public boolean isUnwrappableAs(Class unwrapType) {
return false;
}
#Override
public <T> T unwrap(Class<T> unwrapType) {
return null;
}
#Override
public boolean supportsAggressiveRelease() {
return true;
}
}
Step 3: Wire it up
#Configuration
public class HibernateConfig {
#Autowired
private JpaProperties jpaProperties;
#Bean
public JpaVendorAdapter jpaVendorAdapter() {
return new HibernateJpaVendorAdapter();
}
#Bean
public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource,
MultiTenantConnectionProvider multiTenantConnectionProviderImpl,
CurrentTenantIdentifierResolver currentTenantIdentifierResolverImpl) {
Map<String, Object> properties = new HashMap<>();
properties.putAll(jpaProperties.getHibernateProperties(dataSource));
properties.put(Environment.MULTI_TENANT, MultiTenancyStrategy.SCHEMA);
properties.put(Environment.MULTI_TENANT_CONNECTION_PROVIDER, multiTenantConnectionProviderImpl);
properties.put(Environment.MULTI_TENANT_IDENTIFIER_RESOLVER, currentTenantIdentifierResolverImpl);
LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();
em.setDataSource(dataSource);
em.setPackagesToScan("com.autorni");
em.setJpaVendorAdapter(jpaVendorAdapter());
em.setJpaPropertyMap(properties);
return em;
}
}
I am newbie to Spring Integration. I am working on solution, but I am stuck on a specific issue while using inbound file adapter ( FileReadingMessageSource ).
I have to read files from different directories and process them and save the files in different directories. As I understand, the directory name is fixed at the start of the flow.
Can some one help me on changing the directory name for different requests.
I attempted the following. First of all, I am not sure whether it is correct way to about and although it worked for only one directory. I think Poller was waiting for more files and never came back to read another directory.
#SpringBootApplication
#EnableIntegration
#IntegrationComponentScan
public class SiSampleFileProcessor {
#Autowired
MyFileProcessor myFileProcessor;
#Value("${si.outdir}")
String outDir;
#Autowired
Environment env;
public static void main(String[] args) throws IOException {
ConfigurableApplicationContext ctx = new SpringApplication(SiSampleFileProcessor.class).run(args);
FileProcessingService gateway = ctx.getBean(FileProcessingService.class);
boolean process = true;
while (process) {
System.out.println("Please enter the input Directory: ");
String inDir = new Scanner(System.in).nextLine();
if ( inDir.isEmpty() || inDir.equals("exit") ) {
process=false;
} else {
System.out.println("Processing... " + inDir);
gateway.processFilesin(inDir);
}
}
ctx.close();
}
#MessagingGateway(defaultRequestChannel="requestChannel")
public interface FileProcessingService {
String processFilesin( String inputDir );
}
#Bean(name = PollerMetadata.DEFAULT_POLLER)
public PollerMetadata poller() {
return Pollers.fixedDelay(1000).get();
}
#Bean
public MessageChannel requestChannel() {
return new DirectChannel();
}
#ServiceActivator(inputChannel = "requestChannel")
#Bean
GenericHandler<String> fileReader() {
return new GenericHandler<String>() {
#Override
public Object handle(String p, Map<String, Object> map) {
FileReadingMessageSource fileSource = new FileReadingMessageSource();
fileSource.setDirectory(new File(p));
Message<File> msg;
while( (msg = fileSource.receive()) != null ) {
fileInChannel().send(msg);
}
return null; // Not sure what to return!
}
};
}
#Bean
public MessageChannel fileInChannel() {
return MessageChannels.queue("fileIn").get();
}
#Bean
public IntegrationFlow fileProcessingFlow() {
return IntegrationFlows.from(fileInChannel())
.handle(myFileProcessor)
.handle(Files.outboundAdapter(new File(outDir)).autoCreateDirectory(true).get())
.get();
}
}
EDIT: Based on Gary's response replaced some methods as
#MessagingGateway(defaultRequestChannel="requestChannel")
public interface FileProcessingService {
boolean processFilesin( String inputDir );
}
#ServiceActivator(inputChannel = "requestChannel")
public boolean fileReader(String inDir) {
FileReadingMessageSource fileSource = new FileReadingMessageSource();
fileSource.setDirectory(new File(inDir));
fileSource.afterPropertiesSet();
fileSource.start();
Message<File> msg;
while ((msg = fileSource.receive()) != null) {
fileInChannel().send(msg);
}
fileSource.stop();
System.out.println("Sent all files in directory: " + inDir);
return true;
}
Now it is working as expected.
You can use this code
FileProcessor.java
import org.springframework.messaging.Message;
import org.springframework.stereotype.Component;
#Component
public class FileProcessor {
private static final String HEADER_FILE_NAME = "file_name";
private static final String MSG = "%s received. Content: %s";
public void process(Message<String> msg) {
String fileName = (String) msg.getHeaders().get(HEADER_FILE_NAME);
String content = msg.getPayload();
//System.out.println(String.format(MSG, fileName, content));
System.out.println(content);
}
}
LastModifiedFileFilter.java
package com.example.demo;
import org.springframework.integration.file.filters.AbstractFileListFilter;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
public class LastModifiedFileFilter extends AbstractFileListFilter<File> {
private final Map<String, Long> files = new HashMap<>();
private final Object monitor = new Object();
#Override
protected boolean accept(File file) {
synchronized (this.monitor) {
Long previousModifiedTime = files.put(file.getName(), file.lastModified());
return previousModifiedTime == null || previousModifiedTime != file.lastModified();
}
}
}
Main Class= DemoApplication.java
package com.example.demo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import org.apache.commons.io.FileUtils;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.annotation.Aggregator;
import org.springframework.integration.annotation.InboundChannelAdapter;
import org.springframework.integration.annotation.Poller;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.core.MessageSource;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.dsl.channel.MessageChannels;
import org.springframework.integration.dsl.core.Pollers;
import org.springframework.integration.file.FileReadingMessageSource;
import org.springframework.integration.file.filters.CompositeFileListFilter;
import org.springframework.integration.file.filters.SimplePatternFileListFilter;
import org.springframework.integration.file.transformer.FileToStringTransformer;
import org.springframework.integration.scheduling.PollerMetadata;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.PollableChannel;
import org.springframework.stereotype.Component;
#SpringBootApplication
#Configuration
public class DemoApplication {
private static final String DIRECTORY = "E:/usmandata/logs/input/";
public static void main(String[] args) throws IOException, InterruptedException {
SpringApplication.run(DemoApplication.class, args);
}
#Bean
public IntegrationFlow processFileFlow() {
return IntegrationFlows
.from("fileInputChannel")
.transform(fileToStringTransformer())
.handle("fileProcessor", "process").get();
}
#Bean
public MessageChannel fileInputChannel() {
return new DirectChannel();
}
#Bean
#InboundChannelAdapter(value = "fileInputChannel", poller = #Poller(fixedDelay = "1000"))
public MessageSource<File> fileReadingMessageSource() {
CompositeFileListFilter<File> filters =new CompositeFileListFilter<>();
filters.addFilter(new SimplePatternFileListFilter("*.log"));
filters.addFilter(new LastModifiedFileFilter());
FileReadingMessageSource source = new FileReadingMessageSource();
source.setAutoCreateDirectory(true);
source.setDirectory(new File(DIRECTORY));
source.setFilter(filters);
return source;
}
#Bean
public FileToStringTransformer fileToStringTransformer() {
return new FileToStringTransformer();
}
#Bean
public FileProcessor fileProcessor() {
return new FileProcessor();
}
}
The FileReadingMessageSource uses a DirectoryScanner internally; it is normally set up by Spring after the properties are injected. Since you are managing the object outside of Spring, you need to call Spring bean initialization and lifecycle methods afterPropertiesSet() , start() and stop().
Call stop() when the receive returns null.
> return null; // Not sure what to return!
If you return nothing, your calling thread will hang in the gateway waiting for a response. You could change the gateway to return void or, since your gateway is expecting a String, just return some value.
However, your calling code is not looking at the result anyway.
> gateway.processFilesin(inDir);
Also, remove the #Bean from the #ServiceActivator; with that style, the bean type must be MessageHandler.
I have a java app with redis, and it throws the exception.
Here are classes.
Main class:
public class App {
public static void main( String[] args ) {
ManipulatingData manData = new ManipulatingData();
manData.addData();
}
}
ApplicationConfig:
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package com.mycompany.springredisdatabook;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.serializer.StringRedisSerializer;
#Configuration
public class ApplicationConfig {
#Bean
public JedisConnectionFactory connectionFactory() {
JedisConnectionFactory connectionFactory = new JedisConnectionFactory();
connectionFactory.setHostName("localhost");
connectionFactory.setPort(6379);
return connectionFactory;
}
#Bean
public StringRedisTemplate redisTemplate() {
StringRedisTemplate redisTemplate = new StringRedisTemplate();
redisTemplate.setConnectionFactory(connectionFactory());
return redisTemplate;
}
#Bean
public RedisTemplate<String, Long> longTemplate() {
StringRedisSerializer STRING_SERIALIZER = new StringRedisSerializer();
RedisTemplate<String, Long> redisTemplate = new RedisTemplate<String, Long>();
redisTemplate.setConnectionFactory(connectionFactory());
redisTemplate.setKeySerializer(STRING_SERIALIZER);
redisTemplate.setValueSerializer(LongSerializer.INSTANCE);
return redisTemplate;
}
}
ManipulatingData:
package com.mycompany.springredisdatabook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.core.StringRedisTemplate;
public class ManipulatingData {
public ManipulatingData() {}
#Autowired
StringRedisTemplate redisTemplate;
public void addData() {
double start = System.currentTimeMillis();
for (int i=1; i<=1000; i++) {
redisTemplate.opsForSet().add("k" + i, "v" + i);
}
double end = System.currentTimeMillis();
System.out.println("Add data time: " + (end-start));
}
public String getData (String key) {
return redisTemplate.opsForValue().get(key);
}
public void deleteData(String key) {
redisTemplate.opsForValue().getOperations().delete(key);
}
}
The Exception:
Exception in thread "main" java.lang.NullPointerException
at com.mycompany.springredisdatabook.ManipulatingData.addData(ManipulatingData.java:25)
at com.mycompany.springredisdatabook.App.main(App.java:11)
Java Result: 1
So, what is it? I have no idea. I'm using spring, by the way
your redisTemplate instance is null. check your spring.xml for the configuration of redisTemplate and possibilities which causes it to be null