I am having following spring-batch application
SpringBatchApplication.java
package com.spbt.job.sample;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
#SpringBootApplication
public class SpringBatchApplication {
public static void main(String[] args) {
SpringApplication.run(SpringBatchApplication.class, args);
}
}
TraverseJob.java
package com.spbt.job.sample;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
#Configuration
#EnableBatchProcessing
public class TraverseJob {
#Autowired
protected JobBuilderFactory jobBuilderFactory;
#Autowired
protected StepBuilderFactory stepBuilderFactory;
private String inputFolderPath = "/tmp/inputFolder";
#Bean("TraverseJob")
public Job job() {
return jobBuilderFactory.get("TraverseJob")
.incrementer(new RunIdIncrementer())
.start(traverseStep())
.build();
}
#Bean("TraverseStep")
public Step traverseStep() {
return stepBuilderFactory.get("TraverseStep")
.tasklet(traverseJobTasklet(null))
.build();
}
#Bean("TraverseJobTasklet")
#StepScope
public TraverseJobTasklet traverseJobTasklet(#Value("#{jobParameters[date]}") String date) {
TraverseJobTasklet tasklet = new TraverseJobTasklet();
tasklet.setJobDate(date);
tasklet.setJobDirPath(inputFolderPath);
return tasklet;
}
}
TraverseJobTasklet.java
package com.spbt.job.sample;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.File;
public class TraverseJobTasklet implements Tasklet {
private String jobDirPath;
private String jobDate;
#Autowired
private RemoteFilePush remoteFilePush;
#Override
public RepeatStatus execute(StepContribution stepContribution, ChunkContext chunkContext) throws Exception {
try {
traverseDir(new File(jobDirPath));
} catch (Exception ex) {
throw ex;
}
return RepeatStatus.FINISHED;
}
private void traverseDir(File filePath) throws Exception {
try {
File[] files = filePath.listFiles();
if (files != null) {
for (File file : files) {
String name = file.getName();
if (file.isDirectory()) {
if (remoteFilePush.isRemoteDirExist(name)) {
continue;
} else {
remoteFilePush.createRemoteDir(name);
traverseDir(file);
}
} else {
remoteFilePush.pushFile(file.getPath());
}
}
} else {
throw new Exception("empty/null dir -> " + filePath.getName());
}
} catch (Exception ex) {
throw ex;
}
}
public String getJobDirPath() {
return jobDirPath;
}
public void setJobDirPath(String jobDirPath) {
this.jobDirPath = jobDirPath;
}
public String getJobDate() {
return jobDate;
}
public void setJobDate(String jobDate) {
this.jobDate = jobDate;
}
}
RemoteFilePushLogic.java
package com.spbt.job.sample;
import org.springframework.stereotype.Component;
#Component
public class RemoteFilePush {
public boolean isRemoteDirExist(String name) throws InterruptedException {
boolean isRemoteDirExist = false;
// code to check dir on remote server
return isRemoteDirExist;
}
public void createRemoteDir(String name) throws InterruptedException {
// code to create dir on remote server
}
public void pushFile(String path) throws InterruptedException {
// code to push file on remote server
System.out.println("Pushed");
}
}
I want to do parallel traversal and execution in traverseDir method in TraverseJobTasklet, by keeping my RemoteFilePush Logic intact, my inputFolderPath can contain multiple child directories each of which contains some files in it.
I have tried to follow link for spring-batch version which I am using, But its xml based and I do not seem to get how can I create multiple step out of single traverseStep I have?
input a sub-folder string path per worker step is where i am hitting wall with spring code, if you can point me to some ref. it will be helpful, most of the example on net is xml based.
Here is a quick self-contained example with Java config:
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
#Configuration
#EnableBatchProcessing
public class PartitionJobSample {
private final JobBuilderFactory jobs;
private final StepBuilderFactory steps;
public PartitionJobSample(JobBuilderFactory jobs, StepBuilderFactory steps) {
this.jobs = jobs;
this.steps = steps;
}
#Bean
public Step managerStep() {
return steps.get("masterStep")
.partitioner(workerStep().getName(), partitioner(null))
.step(workerStep())
.gridSize(4)
.taskExecutor(taskExecutor())
.build();
}
#Bean
public SimpleAsyncTaskExecutor taskExecutor() {
return new SimpleAsyncTaskExecutor();// TODO useful for testing, use a more robust task executor in production
}
#Bean
#StepScope
public Partitioner partitioner(#Value("#{jobParameters['rootFolder']}") String rootFolder) {
List<String> subFolders = getSubFolders(rootFolder);
return new Partitioner() {
#Override
public Map<String, ExecutionContext> partition(int gridSize) {
Map<String, ExecutionContext> map = new HashMap<>(gridSize);
for (String folder : subFolders) {
ExecutionContext executionContext = new ExecutionContext();
executionContext.put("filePath", folder);
map.put("partition-for-" + folder, executionContext);
}
return map;
}
};
}
private List<String> getSubFolders(String rootFolder) {
// TODO implement this
return Arrays.asList("/data/folder1", "/data/folder2");
}
#Bean
public Step workerStep() {
return steps.get("workerStep")
.tasklet(getTasklet(null))
.build();
}
#Bean
#StepScope
public Tasklet getTasklet(#Value("#{stepExecutionContext['filePath']}") String filePath) {
return new TraverseJobTasklet(filePath);
}
#Bean
public Job job() {
return jobs.get("job")
.start(managerStep())
.build();
}
public static void main(String[] args) throws Exception {
ApplicationContext context = new AnnotationConfigApplicationContext(PartitionJobSample.class);
JobLauncher jobLauncher = context.getBean(JobLauncher.class);
Job job = context.getBean(Job.class);
JobParameters jobParameters = new JobParametersBuilder()
.addString("rootFolder", "/data")
.toJobParameters();
jobLauncher.run(job, jobParameters);
}
class TraverseJobTasklet implements Tasklet {
private String filePath;
public TraverseJobTasklet(String filePath) {
this.filePath = filePath;
}
#Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
// TODO call traversePath for filePath which is a sub-folder here
System.out.println(Thread.currentThread().getName() + " processing sub-folder " + filePath);
return RepeatStatus.FINISHED;
}
}
}
It passes the root directory as a job parameter and executes a partitioned step where each worker processes a sub-folder (calling your tasklet).
If you run it, you should see something like:
SimpleAsyncTaskExecutor-2 processing sub-folder /data/folder1
SimpleAsyncTaskExecutor-1 processing sub-folder /data/folder2
I will let you adapt it to your situation accordingly.
Related
Can someone tell me how to add an attachment using ProducerTemplate?
I have been searching but I can not find an answer to my case.
I am using Camen 2.1 and I have these three clases:
MailSender2.java
import java.util.HashMap;
import java.util.Map;
import java.util.ResourceBundle;
import org.apache.camel.Exchange;
import org.apache.camel.ExchangePattern;
import org.apache.camel.ProducerTemplate;
public class MailSender2 extends TypeMail{
private static final ResourceBundle RES = ResourceBundle.getBundle("mail");
protected static final String MAIL_NOTIFICATION_ENDPOINT=RES.getString("mail.host.location").trim()+":"+RES.getString("mail.port").trim();
private Map<String, Object> header;
public MailSender2() {
this.header=new HashMap<>();
}
public void send(ProducerTemplate template) {
this.header.put("From", this.getT_from());
this.header.put("To", this.getT_to());
this.header.put("Subject", this.getT_subject());
this.header.put(Exchange.CONTENT_TYPE, "text/html; charset=UTF-8");
//this.getF_ficher() <-- I have here the file to attach
//this.getT_ficnon() <-- I have here the name ot the file
//this.getT_ficext() <-- I have here the extension ot the file
template.sendBodyAndHeaders(MAIL_NOTIFICATION_ENDPOINT, this.getT_mensaje(), header);
}
}
TypeMail.java:
public class TypeMail {
private String t_id;
private String t_from;
private String t_to;
private String t_subject;
private String t_mensaje;
private byte[] f_ficher;
private String t_ficnon;
private String t_ficext;
public String getT_id() {
return t_id;
}
public void setT_id(String t_id) {
this.t_id = t_id;
}
public String getT_from() {
return t_from;
}
public void setT_from(String t_from) {
this.t_from = t_from;
}
public String getT_to() {
return t_to;
}
public void setT_to(String t_to) {
this.t_to = t_to;
}
public String getT_subject() {
return t_subject;
}
public void setT_subject(String t_subject) {
this.t_subject = t_subject;
}
public String getT_mensaje() {
return t_mensaje;
}
public void setT_mensaje(String t_mensaje) {
this.t_mensaje = t_mensaje;
}
public byte[] getF_ficher() {
return f_ficher;
}
public void setF_ficher(byte[] f_ficher) {
this.f_ficher = f_ficher;
}
public String getT_ficnon() {
return t_ficnon;
}
public void setT_ficnon(String t_ficnon) {
this.t_ficnon = t_ficnon;
}
public String getT_ficext() {
return t_ficext;
}
public void setT_ficext(String t_ficext) {
this.t_ficext = t_ficext;
}
}
MailCommunicationTransformer.java:
import org.apache.camel.CamelContext;
import org.apache.camel.Exchange;
import org.apache.camel.ProducerTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ws.soap.client.SoapFaultClientException;
public class MailCommunicationTransformer {
MailSender2 mailSender = null;
static Logger logger = LoggerFactory.getLogger(MailCommunicationTransformer.class);
public MailCommunicationTransformer()
{
}
public MailLog transform(Object actualMessage, Exchange exchange, CamelContext context)
{
mailSender = exchange.getIn().getBody(MailSender2.class);
try {
MailSenderDAO mailSenderDAO = (MailSenderDAO)context.getRegistry().lookup("MailSenderDAO");
mailSenderDAO.validarInput(mailSender);
if (mailSender!=null) {
ProducerTemplate template=exchange.getContext().createProducerTemplate();
try {
mailSender.send(template);
}
catch (Throwable ex) {
ex.printStackTrace();
exchange.setProperty(Exchange.EXCEPTION_CAUGHT,ex);
}
}
}catch (MailException me) {
me.printStackTrace();
exchange.setProperty(Exchange.EXCEPTION_CAUGHT,me);
}
Throwable e = exchange.getProperty(Exchange.EXCEPTION_CAUGHT,
Throwable.class);
String response = "OK";
if (e != null) {
StringBuffer mensaje = new StringBuffer();
if (e instanceof SoapFaultClientException) {
mensaje.append("MAIL fault exception: CLIENT. ");
} else {
mensaje.append("MAIL fault exception: MAIL. ");
}
logger.info("MailCommunicationTransformer",e);
while (e != null) {
e.printStackTrace();
mensaje.append(e.getMessage());
e = e.getCause();
}
response = mensaje.toString();
}
MailLog log = new MailLog(mailSender, response); //, protocolo
return log;
}
}
In TypeMail I have the file in f_ficher, and the fileName (t_ficnon) and extension (t_ficext), but I can not find how to attach this file in MailSender2 before template.sendBodyAndHeaders(.....)
Any help would be very appreciated.
Regards.
Perhaps I don't fully understand your question, but the ProducerTemplate don't know about the message type.
You just send a body and perhaps also headers to an endpoint.
Therefore the body just needs to be a fully constructed MimeMessage object as documented in the Camel Mail docs.
You can simply construct the mail message with Java and then use the object with the ProducerTemplate (what you already do).
template.sendBodyAndHeaders("your-smtp-endpoint", yourMimeMessageInstance, yourHeaderMap);
Thanks for the answer!
But, finally, I could do it this way:
new class EmailProcessor.java
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.util.Objects;
import java.util.ResourceBundle;
import javax.activation.DataHandler;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.Processor;
import org.apache.commons.codec.binary.Base64;
public class EmailProcessor implements Processor {
// Atributos de la clase
private TypeMail typeMail;
public EmailProcessor(TypeMail typeMail) {
this.typeMail = typeMail;
}
#Override
public void process(Exchange exchange) throws Exception {
Message ms = exchange.getIn();
ms.setHeader("From", this.typeMail.getT_from());
ms.setHeader("To", this.typeMail.getT_to());
ms.setHeader("Subject", this.typeMail.getT_subject());
ms.setHeader(Exchange.CONTENT_TYPE, "text/html; charset=UTF-8");
ms.setBody("<p style='font-family: Calibri;'>" + this.typeMail.getT_mensaje() + "</p>");
if (this.typeMail.getF_ficher() != null) {
String mimeType = "application/pdf";
if ("zip".equals(typeMail.getT_ficext())) {
mimeType = "application/zip";
}
ms.addAttachment(typeMail.getT_ficnom() + "." + typeMail.getT_ficext(), new DataHandler(typeMail.getF_ficher(), mimeType));
}
}
}
MailSender.java:
import java.util.ResourceBundle;
import org.apache.camel.ExchangePattern;
import org.apache.camel.ProducerTemplate;
public class MailSender extends TypeMail{
private static final ResourceBundle RES = ResourceBundle.getBundle("mail");
protected static final String MAIL_NOTIFICATION_ENDPOINT=RES.getString("mail.host.location").trim()+":"+RES.getString("mail.port").trim();
public MailSender() {
}
public void send(ProducerTemplate template) {
template.send(MAIL_NOTIFICATION_ENDPOINT, ExchangePattern.InOnly, new EmailProcessor(this));
}
}
I have to create a unit testcases for a method which acquires the lock with zookeeper and data is processed with CompletableFuture are used
Below is the high level code :
import lombok.Data;
#Data
public class ConfigurationsIntegrationModel {
public enum InteractionType {
TEST,
DEV;
}
private InteractionType interactionType;
private String lockName;
}
import org.springframework.stereotype.Service;
import java.util.Arrays;
import java.util.List;
#Service("configurationsIntegrationService")
public interface ConfigurationsIntegrationService {
public default List<ConfigurationsIntegrationModel> getRecords(ConfigurationsIntegrationModel.InteractionType integrationType) {
return Arrays.asList(getDynamicIntegrationConfigurationMock(integrationType));
}
private static ConfigurationsIntegrationModel getDynamicIntegrationConfigurationMock(ConfigurationsIntegrationModel.InteractionType integrationType) {
ConfigurationsIntegrationModel configurationsIntegration = new ConfigurationsIntegrationModel();
configurationsIntegration.setLockName("Test_Lock");
configurationsIntegration.setInteractionType(integrationType);
return configurationsIntegration;
}
}
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.locks.InterProcessLock;
import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex;
import org.apache.curator.retry.RetryNTimes;
import java.util.concurrent.TimeUnit;
public class DistributedLockProcessor {
private CuratorFramework client;
private String path;
public DistributedLockProcessor(String host, String path) {
RetryPolicy retryPolicy = new RetryNTimes(5, 90);
client = CuratorFrameworkFactory.newClient(host, retryPolicy);
client.start();
}
public InterProcessLock acquire(String lockName) throws Exception {
InterProcessSemaphoreMutex sharedLock = new InterProcessSemaphoreMutex(client, path + "/" + lockName);
if (!sharedLock.acquire(0, TimeUnit.SECONDS)) {
return null;
}
return sharedLock;
}
public boolean release(InterProcessLock sharedLock) throws Exception {
sharedLock.release();
return true;
}
}
import org.apache.curator.framework.recipes.locks.InterProcessLock;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
public class LockingExecutorProcessor<T> {
private Executor executor = null;
private DistributedLockProcessor distributedLock = null;
public LockingExecutorProcessor(String host, String path, int executorCount) {
executor = Executors.newFixedThreadPool(executorCount);
distributedLock = new DistributedLockProcessor(host, path);
}
public void process(List<String> locks, List<T> items, Consumer<T> consumer) throws ExecutionException, InterruptedException {
final List<CompletableFuture<Void>> completableFutures = new ArrayList<>();
for (int i = 0; i < locks.size(); i++) {
final int record = i;
CompletableFuture<Void> future =
CompletableFuture.runAsync(
() -> {
InterProcessLock interProcessLock = null;
try {
interProcessLock = distributedLock.acquire(locks.get(record));
} catch (Exception e) {
e.printStackTrace();
}
if (interProcessLock != null) {
consumer.accept(items.get(record));
}
}, executor);
completableFutures.add(future);
}
CompletableFuture<Void> completable = CompletableFuture.allOf(completableFutures.toArray(new CompletableFuture[completableFutures.size()]));
completable.get();
}
}
import org.springframework.stereotype.Service;
import java.util.Arrays;
import java.util.List;
#Service("messageService")
public interface MessageService {
public default List<String> getMessagesList(ConfigurationsIntegrationModel.InteractionType integrationType) {
return Arrays.asList("Message1", "Message2", "Message3","Message4");
}
}
import com.google.common.util.concurrent.RateLimiter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
#Component("sampleJob")
public class SampleJob {
#Autowired
private LockingExecutorProcessor<ConfigurationsIntegrationModel> lockingExecutorProcessor;
#Autowired
private ConfigurationsIntegrationService configurationsIntegrationService;
#Autowired
private RateLimiter rateLimiter;
#Autowired
private MessageService messageService;
private List<String> getLockNames(List<ConfigurationsIntegrationModel> integrationConfigurations) {
List<String> lockNames = new ArrayList<>();
for (ConfigurationsIntegrationModel integrationConfiguration : integrationConfigurations) {
lockNames.add(integrationConfiguration.getLockName());
}
return lockNames;
}
#Scheduled(fixedRateString = "100")
public void executeJob() throws ExecutionException, InterruptedException {
List<ConfigurationsIntegrationModel> testRecords = configurationsIntegrationService.getRecords(ConfigurationsIntegrationModel.InteractionType.TEST);
List<String> lockNames = getLockNames(testRecords);
lockingExecutorProcessor.process(
lockNames,
testRecords,
recordsConfig -> {
final List<CompletableFuture<Void>> completableFutures = new ArrayList<>();
List<String> msgList = messageService.getMessagesList(recordsConfig.getInteractionType());
for (String message : msgList) {
completableFutures.add(
CompletableFuture.runAsync(
() -> {
System.out.println("Message is #### "+ message);
}, Executors.newFixedThreadPool(10)));
}
});
}
}
Below is the testcase which i tried so far :
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.concurrent.ExecutionException;
#RunWith(MockitoJUnitRunner.class)
public class SampleJobTest {
#InjectMocks
private SampleJob sampleJob = new SampleJob();
#Mock
private ConfigurationsIntegrationService configurationsIntegrationService;
#Mock
private MessageService messageService;
#Mock
private LockingExecutorProcessor<ConfigurationsIntegrationModel> lockingExecutorProcessor;
#Test
public void testSampleJob() throws ExecutionException, InterruptedException {
Mockito.doCallRealMethod().when(lockingExecutorProcessor).process(Mockito.any(), Mockito.any(), Mockito.any());
Mockito.doCallRealMethod().when(configurationsIntegrationService).getRecords(Mockito.any());
Mockito.doCallRealMethod().when(messageService).getMessagesList(Mockito.any());
sampleJob.executeJob();
}
}
When i debug the code then it breaks at the line CompletableFuture.runAsync of LockingExecutorProcessor and throws null pointer error; the reason is distributedLockProcessor object is null;
How can we mock it and how to connect to test zookeeper server instead of original to ensure locking is working fine
This is done. Now the testcases are working fine. I have used curator-test dependency and also used reflection to mock the private objects.
I have the following class,
public class Processor {
JobUpdate jobUpdate = new JobUpdate();
public void process(Job job){
try {
doProcess(job);
}catch (Exception e){
handleError(job);
System.out.println("This is error");
throw e; // Here throw Runtime Exception.
}
}
private void handleError(Job job) {
if (job.getFailure() > 0){
jobUpdate.updateJobStatus(job, JobStatus.SUBMITTED);
}else{
jobUpdate.updateJobStatus(job, JobStatus.FAILED);
}
}
private void doProcess(Job job){
String name = job.getName();
name += "added";
// ....
}
}
and my test cases as below,
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
#ExtendWith(MockitoExtension.class)
public class ProcessTest {
Processor processor = new Processor();
#Mock
private Job job;
#Mock
private JobUpdate jobUpdate;
#Captor
private ArgumentCaptor<Job> captorJob;
#Captor
private ArgumentCaptor<JobStatus> captorStatus;
#Test
public void shouldThrowException(){
Mockito.when(job.getName()).thenThrow(RuntimeException.class);
Mockito.when(job.getFailure()).thenReturn(2);
processor.process(job);
Mockito.verify(jobUpdate).updateJobStatus(captorJob.capture(),captorStatus.capture());
assertThat(captorJob.getValue().getFailure(), equalTo( 2));
assertThat(captorStatus.getValue(), equalTo( JobStatus.SUBMITTED));
}
}
The flow goes fine in catch block and after executing System.out.println("This is error"); statement, it throws exception. The controller not backing to test class to verify my ArgumentCaptor.
am i missing something or need to be changed?
I need to create a non-blocking functionality where I search text files within a given folder and it returns the count of search terms found in it.
I am able to execute the test in a blocking manner. I am wondering if anyone could help me transform it into a non-blocking task so that whenever a file is finished being scanned the result delivered without waiting for all the files to be scanned.
The main point is: I don't want to wait for all files to be scanned to start delivering the result to the client (Angular app).
public interface SearchService {
List<SearchResponse> search(SearchRequest searchRequest);
}
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
#ToString
#Getter
#RequiredArgsConstructor(staticName = "of")
public class SearchResponse implements Serializable {
private final String server;
private final String filePath;
private final long count;
private final boolean errorReadingFile;
}
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
#ToString
#Getter
#RequiredArgsConstructor(staticName = "of")
public class SearchRequest implements Serializable {
#NotNull
private final String server;
#NotNull
private final String rootPath;
#NotNull
private final String searchTerm;
}
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
#Slf4j
#Service
public class FileSearchService implements SearchService {
#Override
public List<SearchResponse> search(SearchRequest searchRequest) {
Path start = Paths.get(searchRequest.getRootPath());
EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
int maxDepth = Integer.MAX_VALUE;
SearchTermFileVisitor visitor = new SearchTermFileVisitor(searchRequest, new ArrayList<>());
try {
Files.walkFileTree(start,opts,maxDepth, visitor);
return visitor.getSearchResponseList();
} catch (IOException e) {
System.out.println(e.getMessage());
}
}
}
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
#Slf4j
#Getter
#RequiredArgsConstructor
public class SearchTermFileVisitor extends SimpleFileVisitor<Path> {
private final SearchRequest searchRequest;
private final List<SearchResponse> searchResponseList;
private SearchResponse searchFileContent(Path path, SearchRequest searchRequest) {
SearchResponse response;
try (BufferedReader br = Files.newBufferedReader(path)) {
response = SearchResponse.of(
searchRequest.getServer(),
Paths.get(path.toUri()).toString(),
countWordsInFile(searchRequest.getSearchTerm(), br.lines()),
false);
} catch (Exception e) {
response = SearchResponse.of(
searchRequest.getServer(),
path.toString(),
0,
true);
}
log.debug(response.toString());
return response;
}
private int countWordsInFile(String searchTerm, Stream<String> linesStream) {
return linesStream
.parallel()
.map(line -> countWordsInLine(line, searchTerm))
.reduce(0, Integer::sum);
}
private int countWordsInLine(String line, String searchTerm) {
Pattern pattern = Pattern.compile(searchTerm.toLowerCase());
Matcher matcher = pattern.matcher(line.toLowerCase());
int count = 0;
int i = 0;
while (matcher.find(i)) {
count++;
i = matcher.start() + 1;
}
return count;
}
private boolean isTextFile(Path path) throws IOException {
String type = Files.probeContentType(path);
if (type == null) {
//type couldn't be determined, assume binary
return false;
} else if (type.startsWith("text")) {
return true;
} else {
//type isn't text
return false;
}
}
#Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
log.debug("Visited: " + (Path) dir);
return FileVisitResult.CONTINUE;
}
#Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
return FileVisitResult.CONTINUE;
}
#Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (attrs.isRegularFile()
&& !attrs.isDirectory()
&& !attrs.isSymbolicLink()
&& isTextFile(file)) {
searchResponseList.add(searchFileContent(file, searchRequest));
}
return FileVisitResult.CONTINUE;
}
#Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
return FileVisitResult.CONTINUE;
}
}
The test case:
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.util.List;
class FileSearchServiceTest {
private SearchService searchService = new FileSearchService();
#Test
void test_search_window_root_c_path() {
SearchRequest sr = SearchRequest.of("localhost", "c:\\", "a");
final List<SearchResponse> searchResult = searchService.search(sr);
Assertions.assertNotNull(searchResult.size());
}
}
I want to use WebFlux to receive the results one by one without waiting for all the files to be scanned.
Consider (1) performing the search in a separate thread, (2) using the observer pattern for receiving intermediate results back to the thread created code (references below), and (3) join the search thread to the implementing code so you can return the result list once it is finished. It means you need to pass 'this' to the thread so it has a reference to call back to a separate method for intermediate results. The references below contain sample code.
Think of this like a GUI. You can run the GUI in a separate thread, and for every button click it calls back to the controller code (including a 'finish' once Exit is clicked).
Re: https://dzone.com/articles/the-observer-pattern-using-modern-java
https://en.wikipedia.org/wiki/Observer_pattern
I am integrating with an old system and have a need to parse the following xml into my object. I am trying to do this with jackson but I can't get the mapping to work. Anyone know how to map the following xml to the pojo?
#JacksonXmlRootElement(localName = "properties")
#Data
public class Example {
private String token;
private String affid;
private String domain;
}
xml example:
<properties>
<entry key="token">rent</entry>
<entry key="affid">true</entry>
<entry key="domain">checking</entry>
</properties>
I have tried adding
#JacksonXmlProperty(isAttribute = true, localName = "key")
to the properties but this of course doesn't work and I do not see another way to get this to work. Any ideas?
I am using the mapper like so...
ObjectMapper xmlMapper = new XmlMapper();
dto = xmlMapper.readValue(XML_STRING, Example .class);
I am using the following dependencies
compile('org.springframework.boot:spring-boot-starter-web')
runtime('org.springframework.boot:spring-boot-devtools')
compileOnly('org.projectlombok:lombok')
testCompile('org.springframework.boot:spring-boot-starter-test')
compile('org.apache.commons:commons-lang3:3.5')
compile('com.fasterxml.jackson.dataformat:jackson-dataformat-xml')
compile('com.squareup.okhttp3:okhttp:3.10.0')
This does work.
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlText;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;
public class XmlParserDemo {
public static void main(String[] args) throws IOException, XMLStreamException {
String xmlString = "<properties>\n" +
" <entry key=\"token\">rent</entry>\n" +
" <entry key=\"affid\">true</entry>\n" +
" <entry key=\"domain\">checking</entry>\n" +
"</properties>";
XMLStreamReader sr = null;
sr = XMLInputFactory.newFactory().createXMLStreamReader(new StringReader(xmlString));
sr.next();
XmlMapper mapper = new XmlMapper();
List<Entry> entries = mapper.readValue(sr, new TypeReference<List<Entry>>() {
});
sr.close();
entries.forEach(e ->
System.out.println(e.key + ":" + e.value));
}
public static class Entry {
#JacksonXmlProperty(isAttribute = true, localName = "key")
private String key;
#JacksonXmlText
private String value;
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
}
}
Output is:
token:rent
affid:true
domain:checking
I have looked through Jackson thoroughly and it doesn't seem that there is a way to accomplish this. However, I will share my solution here in case it is useful to someone else.
package com.example.config;
import com.example.dto.Example;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.http.HttpInputMessage;
import org.springframework.http.HttpOutputMessage;
import org.springframework.http.MediaType;
import org.springframework.http.converter.AbstractHttpMessageConverter;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.http.converter.HttpMessageNotWritableException;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
public class Converter extends AbstractHttpMessageConverter<Example> {
private static final XPath XPATH_INSTANCE = XPathFactory.newInstance().newXPath();
private static final StringHttpMessageConverter MESSAGE_CONVERTER = new StringHttpMessageConverter();
#Override
protected boolean supports(Class<?> aClass) {
return aClass == Example.class;
}
#Override
protected Example readInternal(Class<? extends LongFormDTO> aClass, HttpInputMessage httpInputMessage) throws IOException, HttpMessageNotReadableException {
String responseString = MESSAGE_CONVERTER.read(String.class, httpInputMessage);
Reader xmlInput = new StringReader(responseString);
InputSource inputSource = new InputSource(xmlInput);
Example dto = new Example();
Node xml;
try {
xml = (Node) XPATH_INSTANCE.evaluate("/properties", inputSource, XPathConstants.NODE);
} catch (XPathExpressionException e) {
log.error("Unable to parse response", e);
return dto;
}
log.info("processing populate application response={}", responseString);
dto.setToken(getString("token", xml));
dto.setAffid(getInt("affid", xml, 36));
dto.domain(getString("domain", xml));
xmlInput.close();
return dto;
}
private String getString(String propName, Node xml, String defaultValue) {
String xpath = String.format("//entry[#key='%s']/text()", propName);
try {
String value = (String) XPATH_INSTANCE.evaluate(xpath, xml, XPathConstants.STRING);
return StringUtils.isEmpty(value) ? defaultValue : value;
} catch (XPathExpressionException e) {
log.error("Received error retrieving property={} from xml", propName, e);
}
return defaultValue;
}
private String getString(String propName, Node xml) {
return getString(propName, xml, null);
}
private int getInt(String propName, Node xml, int defaultValue) {
String stringValue = getString(propName, xml);
if (!StringUtils.isEmpty(stringValue)) {
try {
return Integer.parseInt(stringValue);
} catch (NumberFormatException e) {
log.error("Attempted to parse value={} as integer but received error", stringValue, e);
}
}
return defaultValue;
}
private int getInt(String propName, Node xml) {
return getInt(propName, xml,0);
}
private boolean getBoolean(String propName, Node xml) {
String stringValue = getString(propName, xml );
return Boolean.valueOf(stringValue);
}
#Override
protected void writeInternal(Example dto, HttpOutputMessage httpOutputMessage) throws IOException, HttpMessageNotWritableException {
throw new UnsupportedOperationException("Responses of type=" + MediaType.TEXT_PLAIN_VALUE + " are not supported");
}
}
I chose to hide this in a message converter so I don't have to look at it again but you can apply these steps where you see fit. If you choose this route, you will need to configure a rest template to use this converter. If not, it is important to cache the xml into a Node object as regenerating each time will be very costly.
package com.example.config;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.http.MediaType;
import org.springframework.http.client.OkHttp3ClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
#Configuration
public class RestConfig {
#Bean
#Primary
public RestTemplate restTemplate() {
return new RestTemplate(new OkHttp3ClientHttpRequestFactory());
}
#Bean
public RestTemplate restTemplateLe(RestTemplateBuilder builder) {
List<HttpMessageConverter<?>> messageConverters = new ArrayList<>();
ExampleConverter exampleConverter = new ExampleConverter();
exampleConverter.setSupportedMediaTypes(Collections.singletonList(MediaType.TEXT_PLAIN));
messageConverters.add(exampleConverter);
return builder.messageConverters(messageConverters)
.requestFactory(new OkHttp3ClientHttpRequestFactory())
.build();
}
}