We Keep Coding

How do I exstract data from multiple websites using restful API and Spring? [closed]

Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 1 year ago.
Improve this question
I got a task at school in which I have to do the following:
Implement the RESTful endpoint API, which simultaneously makes calls
to the following websites:
https://pizzerijalimbo.si/meni/
https://pizzerijalimbo.si/kontakt/
https://pizzerijalimbo.si/my-account/
https://pizzerijalimbo.si/o-nas/
The input for the endpoint is ‘integer’, which represents the number
of simultaneous calls to the above web pages (min 1 represents all
consecutive calls, max 4 represents all simultaneous calls).
Extracts a short title text from each page and saves this text in a
common global structure (array, folder (). The program should also
count successful calls. Finally, the service should list the number of
successful calls, the number of failed calls and the saved address
texts from all web pages.
With some help I managed to do something, but I still need help with data exstraction using Jsoup or any other method.
Here is the code that I have:
import java.util.Arrays;
import java.util.List;
import java.io.IOException;
import java.net.URL;
import java.util.Scanner;
#RestController
public class APIcontroller {
#Autowired
private RestTemplate restTemplate;
List<String> websites = Arrays.asList("https://pizzerijalimbo.si/meni/",
"https://pizzerijalimbo.si/kontakt/",
"https://pizzerijalimbo.si/my-account/",
"https://pizzerijalimbo.si/o-nas/");
#GetMapping("/podatki")
public List<Object> getData(#RequestParam(required = true) int numberOfWebsites) {
List<String> websitesToScrape = websites.subList(0, numberOfWebsites);
for (String website : websitesToScrape) {
Document doc = Jsoup.connect("https://pizzerijalimbo.si/meni/").get();
log(doc.title());
Elements newsHeadlines = doc.select("#mp-itn b a");
for (Element headline : newsHeadlines) {
log("%s\n\t%s",
headline.attr("title"), headline.absUrl("href"));
}
}
}
}
I also need to do it parallel, so the calls to a secific website go on at the same time.
But the main problem now is with the log funcion which does not work properly.
What I have tried:
I tried to solve the problem using Jsoup library, but I dont seem to
undersand it well, so I got an error in the for loop which says that
the method log is undefined. I also need to do a try catch to count possible failed calls and count the calls that are successfull as you can see in the task description.

WebScrapperController.java
package com.stackovertwo.stackovertwo;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
//import org.w3c.dom.Document;
//import org.w3c.dom.DocumentFragment;
import org.jsoup.nodes.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
#RestController
public class WebScrapperController {
#GetMapping("/")
public String index() {
return "Greetings from Spring Boot!";
}
// #Autowired
// private RestTemplate restTemplate;
#Autowired
WebScrapperService webScrapperService;
List<String> websites = Arrays.asList("https://pizzerijalimbo.si/meni/",
"https://pizzerijalimbo.si/kontakt/",
"https://pizzerijalimbo.si/my-account/",
"https://pizzerijalimbo.si/o-nas/");
#GetMapping("/podatki")
public ResponseEntity<Object> getData(#RequestParam(required = true) int numberOfWebsites) throws InterruptedException, ExecutionException {
List<SiteResponse> webSitesToScrape = new ArrayList<>();
// List<String> websitesToScrape = websites.subList(0, numberOfWebsites);
List<SiteResponse> responseResults = new ArrayList<SiteResponse>();
CompletableFuture<SiteResponse> futureData1 = webScrapperService.getWebScrappedContent(websites.get(0));
CompletableFuture<SiteResponse> futureData2 = webScrapperService.getWebScrappedContent(websites.get(1));
//CompletableFuture.allOf(futureData1, futureData2).join();
webSitesToScrape.add(futureData1.get());
webSitesToScrape.add(futureData2.get());
List<SiteResponse> result = webSitesToScrape.stream().collect(Collectors.toList());
return ResponseEntity.ok().body(result);
}
}
WebScrapperService.java
package com.stackovertwo.stackovertwo;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import java.util.concurrent.CompletableFuture;
#Service
public class WebScrapperService {
#Autowired
private RestTemplate restTemplate;
Logger logger = LoggerFactory.getLogger(WebScrapperService.class);
#Async
public CompletableFuture<SiteResponse> getWebScrappedContent(String webSiteURL)
//throws InterruptedException
{
logger.info("Starting: getWebScrappedContent for webSiteURL {} with thread {}", webSiteURL, Thread.currentThread().getName());
HttpEntity<String> response = restTemplate.exchange(webSiteURL,
HttpMethod.GET, null, String.class);
//Thread.sleep(1000);
SiteResponse webSiteSummary = null ;
String resultString = response.getBody();
HttpHeaders headers = response.getHeaders();
int statusCode = ((ResponseEntity<String>) response).getStatusCode().value();
System.out.println(statusCode);
System.out.println("HEADERS"+headers);
try
{
Document doc = (Document) Jsoup.parse(resultString);
Elements header = doc.select(".elementor-inner h2.elementor-heading-title.elementor-size-default");
System.out.println(header.get(0).html());
// Return the fragment.
webSiteSummary = new SiteResponse(statusCode, header.get(0).html());
}
catch(Exception e) {
System.out.println("Exception "+e.getMessage());
}
logger.info("Complete: getWebScrappedContent for webSiteURL {} with thread {}", webSiteURL, Thread.currentThread().getName());
return CompletableFuture.completedFuture(webSiteSummary);
}
}
SpringBootApp.java
package com.stackovertwo.stackovertwo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
//import javax.net.ssl.HostnameVerifier;
//import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
//import javax.net.ssl.SSLSession;
//import javax.net.ssl.TrustManager;
//import javax.net.ssl.X509TrustManager;
//import javax.security.cert.X509Certificate;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.impl.client.*;
import org.apache.http.conn.ssl.*;
#SpringBootApplication
public class SpringBootApp
{
public static void main(String[] args)
{
SpringApplication.run(SpringBootApp.class, args);
}
#Bean
public RestTemplate restTemplate() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException {
TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true;
SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom()
.loadTrustMaterial(null, acceptingTrustStrategy)
.build();
SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
CloseableHttpClient httpClient = HttpClients.custom()
.setSSLSocketFactory(csf)
.build();
HttpComponentsClientHttpRequestFactory requestFactory =
new HttpComponentsClientHttpRequestFactory();
requestFactory.setHttpClient(httpClient);
//return new RestTemplate();
RestTemplate restTemplate = new RestTemplate(requestFactory);
return restTemplate;
}
}
Note: I disabled SSL verification while calling the webulr in resttemplate, but its not recommendd inproduction (For assignment its ok). But you need to import the keys via java keystore in case production : https://myshittycode.com/2015/12/17/java-https-unable-to-find-valid-certification-path-to-requested-target-2/

I can't pass parameters to foreach loop while implementing Structured Streaming + Kafka in Spark SQL

I followed the instructions at Structured Streaming + Kafka and built a program that receives data streams sent from kafka as input, when I receive the data stream I want to pass it to SparkSession variable to do some query work with Spark SQL, so I extend the ForeachWriter class again as follows:
package stream;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import org.apache.spark.sql.ForeachWriter;
import org.apache.spark.sql.SparkSession;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import dataservices.OrderDataServices;
import models.SuccessEvent;
public class MapEventWriter extends ForeachWriter<String>{
private SparkSession spark;
public MapEventWriter(SparkSession spark) {
this.spark = spark;
}
private static final long serialVersionUID = 1L;
#Override
public void close(Throwable errorOrNull) {
// TODO Auto-generated method stub
}
#Override
public boolean open(long partitionId, long epochId) {
// TODO Auto-generated method stub
return true;
}
#Override
public void process(String input) {
OrderDataServices services = new OrderDataServices(this.spark);
}
}
however in the process function, if I use spark variable, the program gives an error, the program passes in my spark as follows:
package demo;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.fs.Path;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.streaming.StreamingQuery;
import org.apache.spark.sql.streaming.StreamingQueryException;
import org.json.simple.parser.ParseException;
import dataservices.OrderDataServices;
import models.MapperEvent;
import models.OrderEvent;
import models.SuccessEvent;
import stream.MapEventWriter;
import stream.MapEventWriter1;
public class Demo {
public static void main(String[] args) throws TimeoutException, StreamingQueryException, ParseException, IOException {
try (SparkSession spark = SparkSession.builder().appName("Read kafka").getOrCreate()) {
Dataset<String> data = spark
.readStream()
.format("kafka")
.option("kafka.bootstrap.servers", "localhost:9092")
.option("subscribe", "tiki-1")
.load()
.selectExpr("CAST(value AS STRING)")
.as(Encoders.STRING());
MapEventWriter eventWriter = new MapEventWriter(spark);
StreamingQuery query = data
.writeStream()
.foreach(eventWriter)
.start();
query.awaitTermination();
}
}
}
The error is NullPointerException at the spark call location, that is, no spark variable is initialized.
Hope anyone can help me, I really appreciate it.
Caused by: java.lang.NullPointerException
at org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:151)
at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:149)
at org.apache.spark.sql.DataFrameReader.<init>(DataFrameReader.scala:998)
at org.apache.spark.sql.SparkSession.read(SparkSession.scala:655)
at dataservices.OrderDataServices.<init>(OrderDataServices.java:18)
at stream.MapEventWriter.process(MapEventWriter.java:38)
at stream.MapEventWriter.process(MapEventWriter.java:15)

do some query work with Spark SQL
You wouldn't use a ForEachWriter for that
.selectExpr("CAST(value AS STRING)")
.as(Encoders.STRING()); // or parse your JSON here using a schema
data.select(...) // or move this to a method / class that takes the Dataset as a parameter
// await termination

How to publish static collection with vertx and quarkus

I'm building an application with Quarkus and it's vert.x extension. Now I want to build a REST endpoint which should stream all saved addresses. To test this whiout a reactive data source I wan't to create a ArrayList with example addresses and stream them so I can check if my test works. But I don't find how I can stream a collection.
My actual code:
import io.vertx.reactivex.core.Vertx;
import org.reactivestreams.Publisher;
import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
import java.util.Collection;
#Path("/addresses")
public class AddressResource {
private Collection<Address> adresses;
#Inject private Vertx vertx;
public AddressResource() {
super();
adresses = new ArrayList<>();
}
#GET
#Produces(MediaType.SERVER_SENT_EVENTS)
public Publisher<Address> get() {
Address address = new Address();
address.setStreet("590 Holly Street");
address.setCity("Townsend");
address.setState("Ohio");
address.setZip(6794);
adresses.add(address);
adresses.add(address);
adresses.add(address);
adresses.add(address);
adresses.add(address);
adresses.add(address);
// What to do here?
return null;
}
}
And this is my test:
import io.quarkus.test.junit.QuarkusTest;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.Test;
import javax.json.bind.JsonbBuilder;
import java.util.ArrayList;
import java.util.List;
import static io.restassured.RestAssured.given;
#QuarkusTest
public class DBServiceTest {
#Test
void testGetAddresses() throws InterruptedException {
given()
.when()
.get("/addresses")
.then()
.statusCode(200)
.body(Matchers.containsInAnyOrder(readTestAdresses().toArray()));
}
private List<Address> readTestAdresses() {
return JsonbBuilder.create()
.fromJson(
this.getClass().getResourceAsStream("test-addresses.json"),
new ArrayList<Address>() {}.getClass().getGenericSuperclass());
}
}
Edit 1:
I tried the following:
#GET
#Produces(MediaType.SERVER_SENT_EVENTS)
public Publisher<String> get() {
Address address = new Address();
address.setStreet("590 Holly Street");
address.setCity("Townsend");
address.setState("Ohio");
address.setZip(6794);
adresses.add(address);
return Flowable.just("Test 1","Test 2","Test 3");
}
And this works. So the problem must have something to do with the address objects.

You can use something like
#GET
#Produces(MediaType.SERVER_SENT_EVENTS)
public Publisher<String> get() {
Address address = new Address();
address.setStreet("590 Holly Street");
address.setCity("Townsend");
address.setState("Ohio");
address.setZip(6794);
return Flowable.just(address, address, address,....).map(a -> yourToString(a));
}
Where yourToString is a method that will create the proper string representation (json perhaps).

I could fix it, I missed a dependency:
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-resteasy-jsonb</artifactId>
</dependency>
And had to convert my objects to json manually and so on change the return type to Publisher<String>:
import io.reactivex.Flowable;
import io.vertx.core.json.Json;
import org.reactivestreams.Publisher;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
import java.util.Collection;
#Path("/addresses")
#Consumes(MediaType.APPLICATION_JSON)
public class AddressResource {
private Collection<Address> addresses;
public AddressResource() {
super();
addresses = new ArrayList<>();
}
#GET
#Produces(MediaType.SERVER_SENT_EVENTS)
public Publisher<String> get() {
Address address = new Address();
address.setStreet("590 Holly Street");
address.setCity("Townsend");
address.setState("Ohio");
address.setZip(6794);
addresses.add(address);
return Flowable.fromIterable(addresses).map(Json::encode);
}
}
Also my test was wrong, this is how it looks like now:
import io.quarkus.test.common.http.TestHTTPResource;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import javax.json.bind.JsonbBuilder;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.sse.SseEventSource;
import java.net.URL;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import static org.assertj.core.api.Assertions.assertThat;
#QuarkusTest
public class DBServiceTest {
#TestHTTPResource("/addresses")
private URL enpointUrl;
#Test
#DisplayName("Get all addresses")
void testGetAddresses() {
Client client = ClientBuilder.newClient();
WebTarget target = client.target(String.valueOf(enpointUrl));
try (SseEventSource source = SseEventSource.target(target).build()) {
CompletableFuture<Collection<Address>> futureAddresses = new CompletableFuture<>();
Set<Address> foundAddresses = new HashSet<>();
source.register(
event -> {
if (!foundAddresses.add(
event.readData(Address.class, MediaType.APPLICATION_JSON_TYPE))) {
futureAddresses.complete(foundAddresses);
}
},
Assertions::fail);
source.open();
Collection<Address> addresses = futureAddresses.join();
assertThat(addresses).containsExactlyInAnyOrderElementsOf(readTestAdresses());
}
}
private List<Address> readTestAdresses() {
return JsonbBuilder.create()
.fromJson(
this.getClass().getResourceAsStream("test-addresses.json"),
new ArrayList<Address>() {}.getClass().getGenericSuperclass());
}
}

Where is "from" imported from camel in java?

I am trying to import csv file to xml file
i see apache has feature to do
from(in)
.to(out)
.split(body().tokenize("\n")).streaming()
.unmarshal().csv();
but i have a "cannot resolve method 'from(java.lang.String)' error
when I try to import then i cannot find any packages for camel
this one works:
import org.apache.camel.dataformat.bindy.csv.BindyCsvDataFormat;
what is the package to use "from" from org.apache.camel.???
this is my file:
import org.apache.camel.Exchange;
import org.apache.camel.dataformat.bindy.BindyAbstractDataFormat;
import org.apache.camel.dataformat.bindy.BindyAbstractFactory;
import org.apache.camel.dataformat.bindy.BindyFixedLengthFactory;
import org.apache.camel.dataformat.bindy.FormatFactory;
import org.apache.camel.dataformat.bindy.csv.BindyCsvDataFormat;
import org.apache.camel.dataformat.bindy.util.ConverterUtils;
import org.apache.camel.spi.DataFormat;
import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
public class Csvtoxml {
public static void convert(String in, String out) throws Exception {
DataFormat bindy = new BindyCsvDataFormat(Model.class);
from("myCsvFile.csv")
.to("myXmlFile.xml")
.split(body().tokenize("\n")).streaming()
.unmarshal().csv();
}
}

It is not imported from anywhere. In order to use it this way you have to inherit your class from org.apache.camel.builder.RouteBuilder

How to get Soap header without SOAP handler

I am trying edit my soap header using my java code before running the request. I am not using handler or jax-rs. I saw WsdlUtils but am not able figure how to use this in my code. please someone help me in this
Here's my code;
package soap.impl;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.xmlbeans.XmlException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.eviware.soapui.SoapUI;
import com.eviware.soapui.StandaloneSoapUICore;
import com.eviware.soapui.config.impl.TestStepConfigImpl;
import com.eviware.soapui.impl.wsdl.WsdlHeaderPart;
import com.eviware.soapui.impl.wsdl.WsdlProject;
import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
import com.eviware.soapui.impl.wsdl.submit.filters.SoapHeadersRequestFilter;
import com.eviware.soapui.impl.wsdl.support.wsdl.WsdlUtils;
import com.eviware.soapui.impl.wsdl.support.wsdl.WsdlUtils.SoapHeader;
import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCaseRunner;
import com.eviware.soapui.model.TestPropertyHolder;
import com.eviware.soapui.model.iface.MessageExchange;
import com.eviware.soapui.model.propertyexpansion.PropertyExpansionUtils;
import com.eviware.soapui.model.support.PropertiesMap;
import com.eviware.soapui.model.support.TestPropertyUtils;
import com.eviware.soapui.model.testsuite.TestCase;
import com.eviware.soapui.model.testsuite.TestProperty;
import com.eviware.soapui.model.testsuite.TestRunner;
import com.eviware.soapui.model.testsuite.TestStepResult;
import com.eviware.soapui.model.testsuite.TestRunner.Status;
import com.eviware.soapui.model.testsuite.TestSuite;
import com.eviware.soapui.support.SoapUIException;
import com.eviware.soapui.support.types.StringToObjectMap;
import com.eviware.soapui.support.types.StringToStringsMap;
public class RunTestImpl{
static Logger logger = LoggerFactory.getLogger(RunTestImpl.class);
public static void main(String[] args) throws XmlException, IOException, SoapUIException {
logger.info("Into the Class for running test cases");
String suiteName = "";
String reportStr = "";
InputData input=new InputData();
TestPropertyHolder holder = PropertyExpansionUtils.getGlobalProperties();
String testCaseName="";
holder.setPropertyValue("CRK", "CRK987909000000000075");
// variables for getting duration
WsdlTestCaseRunner runner = null;
List<TestSuite> suiteList = new ArrayList<TestSuite>();
List<TestCase> caseList = new ArrayList<TestCase>();
SoapUI.setSoapUICore(new StandaloneSoapUICore(true));
// specified soapUI project
WsdlProject project1 = new WsdlProject("D://my-project.xml");
WsdlTestSuite testSuite1= project1.getTestSuiteByName("my TestSuite");
WsdlTestCase testCase1= testSuite1.getTestCaseByName("myTestCase");
suiteList.add(testSuite1);
runner= testCase1.run(new StringToObjectMap(), false);
List<TestStepResult> list= runner.getResults();
StringToStringsMap headers1=null;
for (TestStepResult testStepResult : list) {
testStepResult).getRequestContent();
byte[] rawReq=((MessageExchange)testStepResult).getRawRequestData();
headers1 = ((MessageExchange)testStepResult).getRequestHeaders();
String response = ((MessageExchange)testStepResult).getResponseContent();
StringBuilder build=new StringBuilder();
for (byte b : rawReq) {
build.append((char)b);
}
System.out.println("build "+build.toString());
}
for (Map.Entry<String,List< String>> entry : headers1.entrySet()) {
System.out.println("key"+entry.getKey());
for (String string:entry.getValue()) {
System.out.println("value "+string);
}
}
// }
// string of the results
System.out.println(reportStr);
}
}