Integration of SolrJ with Spring Batch - java

I am using Spring Batch.Following is the jobContext.xml file, JdbcCursorItemReader is reading data from MySQL Database.
<?xml version="1.0" encoding="UTF-8"?>
<beans>
<import resource="infrastructureContext.xml"/>
<batch:job id="readProcessWriteProducts">
<batch:step id="readWriteProducts">
<tasklet>
<chunk reader="reader" processor="processer" writer="writer" commit-interval="5"> </chunk>
</tasklet>
</batch:step>
</batch:job>
<bean id="reader" class="org.springframework.batch.item.database.JdbcCursorItemReader">
<property name="dataSource" ref="dataSource"></property>
<property name="sql" value="SELECT id, name, description, price FROM product"></property>
<property name="rowMapper" ref="productItemReader"></property>
</bean>
<bean id="productItemReader" class="com.itl.readprocesswrite.reader.ProductItemReader"></bean>
<bean id="processer" class="com.itl.readprocesswrite.processor.ProductItemProcessor">
<constructor-arg ref="jdbcTemplate"></constructor-arg>
</bean>
<bean id="writer" class="com.itl.readprocesswrite.writer.ProductJdbcItemWriter">
<constructor-arg ref="jdbcTemplate"></constructor-arg>
</bean>
</beans>
Now, I want to read data from Apache Solr.
I tried following code to read data from Apache Solr.
public class SolrJDrive {
public static void main(String[] args) throws MalformedURLException, SolrServerException {
System.out.println("SolrJDrive::main");
SolrServer solr = new CommonsHttpSolrServer("http://localhost:8983/solr");
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/select");
params.set("q", "*:*");
params.set("spellcheck", "on");
params.set("spellcheck.build", "true");
QueryResponse response = solr.query(params);
SolrDocumentList results = response.getResults();
for (int i = 0; i < results.size(); ++i) {
System.out.println(results.get(i));
}
}//end of method main
}//end of class SolrJDrive
Now how do I integrate this with Spring Batch?

In addition to what I said on your other question (Is it possible to integrate Apache Solr with Spring Batch?), here's an example of your Solr custom ItemReader :
public class SolrReader implements ItemReader<SolrDocumentList> {
#Override
public SolrDocumentList read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
SolrServer solr = new CommonsHttpSolrServer("http://localhost:8983/solr");
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/select");
params.set("q", "*:*");
params.set("spellcheck", "on");
params.set("spellcheck.build", "true");
QueryResponse response = solr.query(params);
SolrDocumentList results = response.getResults();
return results;
}
}
You would then need an ItemProcessor to convert your SolrDocumentList to something you can work with (ie. a POJO).

Related

Interceptor for validating XML request in REST API

I am developing a REST API which is a #POST and #Consumes both MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON . I need to implement validations for the incoming request. I don't want to have Bean Level validation JSR-303. I need to have a Validation class which handles all the validations and i need to configure interceptor for the incoming XML request before Unmarshalling. I looked into Apache cxf interceptors and it is mainly if you are enabling the Bean Validations. How shall i do it?
I have found the way how to do this without the usage of BeanValidationFeature.
public class YourFilter implements ContainerRequestFilter{
#Override
public void filter(ContainerRequestContext request){
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
final InputStream inputStream = request.getEntityStream();
final StringBuilder builder = new StringBuilder();
try
{
IOUtils.copy(inputStream, outStream);
byte[] requestEntity = outStream.toByteArray();
if (requestEntity.length == 0) {
builder.append("");
} else {
builder.append(new String(requestEntity,"UTF-8"));
request.setEntityStream(new ByteArrayInputStream(requestEntity));
setRequest(builder.toString());
validateYourRequest(builder.toString());
}
}catch (Exception ex) {
logger.log(Level.TRACE,"Error occured while converting the request into Stream",ex.getCause());
}
}
}
And in application.context.xml
<bean id="YourFilter" class="com.test.YourFilter"/>
<jaxrs:server id="restContainer" address="/">
<jaxrs:serviceBeans>
<bean class="com.test.YourController" />
</jaxrs:serviceBeans>
<jaxrs:extensionMappings>
<entry key="json" value="application/json" />
<entry key="xml" value="application/xml" />
</jaxrs:extensionMappings>
<jaxrs:providers>
<ref bean="jsonProvider" />
<ref bean="jaxbXmlProvider" />
<ref bean="YourFilter"/>
</jaxrs:providers>
</jaxrs:server>

Job in spring batch is getting executed multiple times and not stopping

I am trying to read data from cassandra using spring batch, where I have implemented ItemReader, ItemProcessor, and ItemWriter. I am able to read the data , process it and write back the data to the same table. I am creating xml file to execute the job:
xml:
<job id="LoadStatusIndicator" job-repository="jobRepository" restartable="false">
<step id="LoadStatus" next="">
<tasklet>
<chunk reader="StatusReader" processor="ItemProcessor" writer="ItemWriter"
commit-interval="10" />
</tasklet>
</step>
</job>
<beans:bean id="ItemWriter" scope="step"
class="com.batch.writer.ItemWriter">
</beans:bean>
<beans:bean id="ItemProcessor" scope="step"
class="com.batch.processor.ItemProcessor">
</beans:bean>
<beans:bean id="Reader" scope="step"
class="com.reader.ItemReader">
<beans:property name="dataSource" ref="CassandraSource" />
</beans:bean>
applicationcontext.xml:
<beans:bean id="CassandraSource" parent="DataSourceParent">
<beans:property name="url" value="jdbc:cassandra://${cassandra.hostName}:${cassandra.port}/${cassandra.keyspace}" />
<beans:property name="driverClassName" value="org.apache.cassandra.cql.jdbc.CassandraDriver" />
</beans:bean>
reader class:
public static final String query = "SELECT * FROM test_1 allow filtering;";
#Override
public List<Item> read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException
{
List<Item> results = new ArrayList<Item>();
try {
results = cassandraTemplate.select(query,Item.class);
} catch (Exception e) {
e.printStackTrace();
}
return results;
}
writer classs:
#Override
public void write(List<? extends Item> item) throws Exception {
try {
cassandraTemplate.insert(item);
}catch(Exception e){e.printStackTrace();}
But the problem is the whole job is getting executed multiple times , infact it is not stopping at all. I have to force stop the job execution. I have only 2 rows in the table.
I think it is because of the commit-interval defined in xml, but having commit-interval = 10, job executes more than 10 times
According to my understanding, when I run the xml file that means I am running the job only one time, it calls the reader once keeps the data in the run time memory (job repository), calls item processor once (I use list ) and the whole list is inserted at once
SOLVED
In reader class I wrote:
if (results.size!=0)
return results;
else
return null;

spring batch getting data from file and passing to procedure

For a Spring batch project, I need to get the date from a file and then I need to pass this date to a procedure and then run the procedure.
Then the result of the procedure must be written to a csv file.
I tried using listeners but couldn't do this.
Can anyone please tell how this can be achieved or if possible can you share any sample code in github.
First of all, you will need to get the date from your file and store it in the JobExecutionContext. One of the most simple solution would be to create a custom Tasklet to read the text file and store the result String in the context via a StepExecutionListener
This tasklet takes a file parameter and stores the result string with the key file.date :
public class CustomTasklet implements Tasklet, StepExecutionListener {
private String date;
private String file;
#Override
public void beforeStep(StepExecution stepExecution) {}
#Override
public ExitStatus afterStep(StepExecution stepExecution) {
stepExecution.getJobExecution().getExecutionContext().put("file.date", date);
}
#Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
// Read from file using FileUtils (Apache)
date = FileUtils.readFileToString(file);
}
public void setFile(String file) {
this.file = file;
}
}
Use it this way :
<batch:step>
<batch:tasklet>
<bean class="xx.xx.xx.CustomTasklet">
<property name="file" value="${file.path}"></property>
</bean>
</batch:tasklet>
</batch:step>
Then you will use a Chunk with late binding to retrieve the previously stored value (i.e. using #{jobExecutionContext['file.date']}).
The reader will be a StoredProcedureItemReader :
<bean class="org.springframework.batch.item.database.StoredProcedureItemReader">
<property name="dataSource" ref="dataSource" />
<property name="procedureName" value="${procedureName}" />
<property name="fetchSize" value="50" />
<property name="parameters">
<list>
<bean class="org.springframework.jdbc.core.SqlParameter">
<constructor-arg index="0" value="#{jobExecutionContext['file.date']}" />
</bean>
</list>
</property>
<property name="rowMapper" ref="rowMapper" />
<property name="preparedStatementSetter" ref="preparedStatementSetter" />
</bean>
The writer will be a FlatFileItemWriter :
<bean class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
<property name="resource" value="${file.dest.path}" />
<property name="lineAggregator" ref="lineAggregator" />
</bean>

Spring #Transactional commiting partial results even exception is thrown

I am running an Apache CXF web service under spring. I use JPA to persist the information. The service has a method that updates a series of rows. Before persisting each row, I check that the values to be persisted really exist in the databes. If there is a value that does not exists, then an Exception is thrown. The problem is I need to rollback al the values updated. I though that using #Transactional in my web service method would do the trick, but instead of that, the values that got persisted are really modified in the database, wich is not the desired behavior.
This is the code of the web service method
#Transactional( propagation = Propagation.REQUIRED )
public UpdateDescriptionResponse updateDescription(UpdateDescriptionRequest updateDescriptionRequest) throws SIASFaultMessage {
try {
SubstanceEntity substance = service.findSubstanceBySubstanceID(updateDescriptionRequest.getUpdateDescriptionRequestData().getIdentity().getSubstanceID());
if (substance!=null){
for(DescriptionKeyValueType keyValue: updateDescriptionRequest.getUpdateDescriptionRequestData().getSubstanceDescriptionData() ){
boolean descriptionExists = false;
for(DescriptionEntity desc: substance.getDescriptionsById()){
if (desc.getDescKey().equals(keyValue.getKey())) {
descriptionExists = true;
break;
}
}
if (!descriptionExists){
SIASFaultDetail faultDetail = new SIASFaultDetail();
faultDetail.setSIASFaultDescription("Description key does not match given substance ID");
faultDetail.setSIASFaultMessage(SIASFaultCode.INVALID_INPUT.toString());
faultDetail.setSIASFaultType(SIASFaultCode.INVALID_INPUT);
SIASFaultMessage fault = new SIASFaultMessage("Description key does not match given substance ID", faultDetail);
throw fault;
}
else
descriptionLogic.updateDescription(substance.getSubstanceId(), keyValue.getKey(),keyValue.getValue());
}
UpdateDescriptionResponse response = new UpdateDescriptionResponse();
UpdateDescriptionResponse.UpdateDescriptionResponsePackage responsePackage = new UpdateDescriptionResponse.UpdateDescriptionResponsePackage();
ResponseStatus status = new ResponseStatus();
status.setMessage(messageOk);
status.setReturn(BigInteger.valueOf(0));
responsePackage.setResponseStatus(status);
response.setUpdateDescriptionResponsePackage(responsePackage);
return response;
}
else
{
SIASFaultDetail faultDetail = new SIASFaultDetail();
faultDetail.setSIASFaultDescription("Substance ID does not exists");
faultDetail.setSIASFaultMessage(SIASFaultCode.INVALID_SUBSTANCE_ID.toString());
faultDetail.setSIASFaultType(SIASFaultCode.INVALID_SUBSTANCE_ID);
SIASFaultMessage fault = new SIASFaultMessage("Substance ID does not exists", faultDetail);
throw fault;
}
} catch (SIASFaultMessage ex) {
throw ex;
} catch (Exception ex) {
SIASFaultDetail a = new SIASFaultDetail();
a.setSIASFaultDescription("Unknown error processing enroll request");
a.setSIASFaultMessage("SERVICE_ERROR");
a.setSIASFaultType(SIASFaultCode.UNKNOWN_ERROR);
SIASFaultMessage fault = new SIASFaultMessage("Something happened", a);
throw fault;
}
}
This is the code for the instance of descriptionLogic.updateDescription(...)
#Override
public void updateDescription(String substanceID, String key, String value) {
PageRequest page = new PageRequest(1, 1);
Map<String, Object> filters = new HashMap<String, Object>();
filters.put("SUBSTANCE_ID", substanceID);
List<SubstanceEntity> substances = substanceService.findAll(page, filters);
if (substances.size() == 0) {
return;
}
SubstanceEntity substanceEntity = substances.get(0);
for (DescriptionEntity desc : substanceEntity.getDescriptionsById()) {
if (desc.getDescKey().equals(key)) {
desc.setDescValue(value);
descriptionService.persist(desc);
}
}
}
This is the test that fails
#Test()
public void testUpdateDescription_does_not_modify_description_with_invalid_values() throws Exception {
UpdateDescriptionRequest request = new UpdateDescriptionRequest();
UpdateDescriptionRequest.UpdateDescriptionRequestData data = new UpdateDescriptionRequest.UpdateDescriptionRequestData();
SIASIdentity identity = new SIASIdentity();
identity.setSubstanceID("804ab00f-d5e9-40ff-a4d3-11c51c2e7479");
data.getSubstanceDescriptionData().add(new DescriptionKeyValueType() {{
setKey("KEY3_1");
setValue("NEW_VALUE_1");
}});
data.getSubstanceDescriptionData().add(new DescriptionKeyValueType() {{
setKey("KEY3_5");
setValue("NEW_VALUE_2");
}});
data.setIdentity(identity);
request.setUpdateDescriptionRequestData(data);
try {
siasService.updateDescription(request);
}
catch (SIASFaultMessage ex){
}
DescriptionEntity descriptionEntity1 = descriptionService.findById(1);
DescriptionEntity descriptionEntity2 = descriptionService.findById(2);
assertThat("The value does not math",descriptionEntity1.getDescValue(), not(equalTo("NEW_VALUE_1")));
assertThat("The value does not math",descriptionEntity2.getDescValue(), not(equalTo("NEW_VALUE_2")));
Assert.assertEquals("The description does not math","KEY3_1", descriptionEntity1.getDescKey());
Assert.assertEquals("The description does not math","KEY3_2", descriptionEntity2.getDescKey());
}
It fails in this line:
assertThat("The value does not math",descriptionEntity1.getDescValue(), not(equalTo("NEW_VALUE_1")));
This is my datasource configuration in my spring context configuration file
.
.
.
<bean id="myDataSource"
class="org.springframework.jdbc.datasource.DriverManagerDataSource">
<property name="driverClassName" value="${jdbc.driverClassName}"/>
<property name="url" value="${jdbc.url}"/>
<property name="username" value="${jdbc.username}"/>
<property name="password" value="${jdbc.password}"/>
</bean>
<jdbc:initialize-database data-source="myDataSource">
<jdbc:script location="classpath:test-data.sql" />
</jdbc:initialize-database>
<bean id="entityManagerFactory"
class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="myDataSource"/>
<property name="packagesToScan" value="cu.jpa"/>
<property name="persistenceProviderClass" value="org.hibernate.ejb.HibernatePersistence"/>
<property name="jpaDialect">
<bean class="cu.jpa.specifications.IsolationSupportHibernateJpaDialect" />
</property>
<property name="jpaProperties">
<props>
<prop key="hibernate.dialect">${hibernate.dialect}</prop>
<prop key="hibernate.show_sql">true</prop>
<prop key="hibernate.hbm2ddl.auto">${hdm2ddl.auto}</prop>
</props>
</property>
<property value="/META-INF/persistence.xml" name="persistenceXmlLocation"/>
</bean>
<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
<property name="entityManagerFactory" ref="entityManagerFactory"/>
</bean>
<tx:annotation-driven/>
.
.
.
This is my persistence.xml file content:
<?xml version="1.0" encoding="UTF-8"?>
<persistence xmlns="http://java.sun.com/xml/ns/persistence" version="2.0">
<persistence-unit name="NewPersistenceUnit">
<class>cu.jpa.entities.PatternEntity</class>
.
.
.
<class>cu.jpa.entities.TraceRegEntity</class>
</persistence-unit>
</persistence>
Extract of the test class:
#DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
#RunWith(SpringJUnit4ClassRunner.class)
#ContextConfiguration(locations = {"classpath:/repositories.xml"})
public class ServiceImplUpdateDescriptionTest {
.
.
.
#Test()
public void testUpdateDescription_does_not_modify_description_with_invalid_values() throws Exception{
.
.
.
}
}
Spring will only rollback the transaction if it is an unchecked exception, if the exception is a checked exception then you will have to add that to your #Transactional annotation.
#Transactional(rollbackFor = SIASFaultMessage.class)

Spring MVC - marshalling an XML parameter to object (JAXB) in a multipart request

I created a file upload service using Spring MVC with apache commons multipart resolver support which specifies that a file should be attached as part of a multipart HTTP Post request. The request also contains a parameter containing an XML string with meta-data about the object. The XML can be marshalled using JAXB.
Other services that are not multipart handle the marshalling transparently, e.g.:
#RequestMapping(value = "/register", method = RequestMethod.POST)
#ResponseStatus(HttpStatus.CREATED)
public ModelAndView createUser(#RequestBody CreateUserDTO createUserDTO) throws Exception {
UserDTO user = userService.createUser(createUserDTO);
return createModelAndView(user);
}
Here CreateUserDTO is a JAXB annotated object which is automatically marshalled.
In the multipart case I'd like to have the same transparency. Ideally I would like to do the following:
RequestMapping(method = RequestMethod.POST)
#ResponseStatus(HttpStatus.CREATED)
public ModelAndView createAttachment(#RequestParam AttachmentDTO attachment,
HttpServletRequest request) throws Exception {
final MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
AttachmentDTO attachment = null;
final MultipartFile dataFile = multipartRequest.getFile("data");
AttachmentDTO createdAttachment = attachmentService.createAttachment(attachment,
dataFile);
return createModelAndView(createdAttachment);
}
Unfortunately this does not work. I am able to bind the attachment parameter as String, but the automatic marshalling does not work. My work around is to manually do the marshalling like the following, but I don't like this approach (especially since the parameter may be specified both in JSON and XML form):
#Autowired
private Jaxb2Marshaller jaxb2Marshaller;
#Autowired
private ObjectMapper jacksonMapper;
#RequestMapping(method = RequestMethod.POST)
#ResponseStatus(HttpStatus.CREATED)
public ModelAndView createAttachment(#RequestParam(ATTACHMENT_PARAMETER) String attachmentString,
final HttpServletRequest request) throws Exception {
final MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
AttachmentDTO attachment = null;
try {
attachment = (AttachmentDTO)jaxb2Marshaller.unmarshal(new StreamSource(new StringReader(attachmentString)));
} catch (XmlMappingException e) {
//Try JSON
try {
attachment = jacksonMapper.readValue(attachmentString, AttachmentDTO.class);
} catch (IOException e1) {
throw new BadRequestException("Could not interpret attachment parameter, both JSON and XML parsing failed");
}
}
Does anyone have a better suggestion for resolving this issue?
For completeness I also specify the relevant Spring config here:
<bean id="viewResolver" class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
<property name="order" value="1"/>
<property name="favorPathExtension" value="true"/>
<property name="ignoreAcceptHeader" value="false"/>
<property name="mediaTypes">
<map>
<entry key="xml" value="application/xml"/>
<entry key="json" value="application/json"/>
</map>
</property>
<property name="viewResolvers">
<list>
<bean class="org.springframework.web.servlet.view.BeanNameViewResolver"/>
<bean class="org.springframework.web.servlet.view.InternalResourceViewResolver">
<property name="prefix" value="/WEB-INF/jsp/"/>
<property name="suffix" value=".jsp"/>
</bean>
</list>
</property>
<property name="defaultViews">
<list>
<bean class="org.springframework.web.servlet.view.xml.MarshallingView">
<property name="modelKey" value="object"/>
<property name="marshaller" ref="jaxbMarshaller"/>
</bean>
<bean class="org.springframework.web.servlet.view.json.MappingJacksonJsonView">
<property name="objectMapper" ref="jaxbJacksonObjectMapper"/>
</bean>
</list>
</property>
</bean>
<!--Use JAXB OXM marshaller to marshall/unmarshall following class-->
<bean id="jaxbMarshaller"
class="org.springframework.oxm.jaxb.Jaxb2Marshaller">
<property name="contextPath" value="com.behindmedia.btfd.model.dto"/>
</bean>
<bean id="jaxbJacksonObjectMapper" class="org.codehaus.jackson.map.ObjectMapper"/>
<!-- allows for integration of file upload functionality -->
<bean class="org.springframework.web.multipart.commons.CommonsMultipartResolver" id="multipartResolver">
<property name="maxUploadSize" value="100000000"/>
</bean>

Categories