We Keep Coding

How do I exstract data from multiple websites using restful API and Spring? [closed]

Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 1 year ago.
Improve this question
I got a task at school in which I have to do the following:
Implement the RESTful endpoint API, which simultaneously makes calls
to the following websites:
https://pizzerijalimbo.si/meni/
https://pizzerijalimbo.si/kontakt/
https://pizzerijalimbo.si/my-account/
https://pizzerijalimbo.si/o-nas/
The input for the endpoint is ‘integer’, which represents the number
of simultaneous calls to the above web pages (min 1 represents all
consecutive calls, max 4 represents all simultaneous calls).
Extracts a short title text from each page and saves this text in a
common global structure (array, folder (). The program should also
count successful calls. Finally, the service should list the number of
successful calls, the number of failed calls and the saved address
texts from all web pages.
With some help I managed to do something, but I still need help with data exstraction using Jsoup or any other method.
Here is the code that I have:
import java.util.Arrays;
import java.util.List;
import java.io.IOException;
import java.net.URL;
import java.util.Scanner;
#RestController
public class APIcontroller {
#Autowired
private RestTemplate restTemplate;
List<String> websites = Arrays.asList("https://pizzerijalimbo.si/meni/",
"https://pizzerijalimbo.si/kontakt/",
"https://pizzerijalimbo.si/my-account/",
"https://pizzerijalimbo.si/o-nas/");
#GetMapping("/podatki")
public List<Object> getData(#RequestParam(required = true) int numberOfWebsites) {
List<String> websitesToScrape = websites.subList(0, numberOfWebsites);
for (String website : websitesToScrape) {
Document doc = Jsoup.connect("https://pizzerijalimbo.si/meni/").get();
log(doc.title());
Elements newsHeadlines = doc.select("#mp-itn b a");
for (Element headline : newsHeadlines) {
log("%s\n\t%s",
headline.attr("title"), headline.absUrl("href"));
}
}
}
}
I also need to do it parallel, so the calls to a secific website go on at the same time.
But the main problem now is with the log funcion which does not work properly.
What I have tried:
I tried to solve the problem using Jsoup library, but I dont seem to
undersand it well, so I got an error in the for loop which says that
the method log is undefined. I also need to do a try catch to count possible failed calls and count the calls that are successfull as you can see in the task description.

WebScrapperController.java
package com.stackovertwo.stackovertwo;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
//import org.w3c.dom.Document;
//import org.w3c.dom.DocumentFragment;
import org.jsoup.nodes.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
#RestController
public class WebScrapperController {
#GetMapping("/")
public String index() {
return "Greetings from Spring Boot!";
}
// #Autowired
// private RestTemplate restTemplate;
#Autowired
WebScrapperService webScrapperService;
List<String> websites = Arrays.asList("https://pizzerijalimbo.si/meni/",
"https://pizzerijalimbo.si/kontakt/",
"https://pizzerijalimbo.si/my-account/",
"https://pizzerijalimbo.si/o-nas/");
#GetMapping("/podatki")
public ResponseEntity<Object> getData(#RequestParam(required = true) int numberOfWebsites) throws InterruptedException, ExecutionException {
List<SiteResponse> webSitesToScrape = new ArrayList<>();
// List<String> websitesToScrape = websites.subList(0, numberOfWebsites);
List<SiteResponse> responseResults = new ArrayList<SiteResponse>();
CompletableFuture<SiteResponse> futureData1 = webScrapperService.getWebScrappedContent(websites.get(0));
CompletableFuture<SiteResponse> futureData2 = webScrapperService.getWebScrappedContent(websites.get(1));
//CompletableFuture.allOf(futureData1, futureData2).join();
webSitesToScrape.add(futureData1.get());
webSitesToScrape.add(futureData2.get());
List<SiteResponse> result = webSitesToScrape.stream().collect(Collectors.toList());
return ResponseEntity.ok().body(result);
}
}
WebScrapperService.java
package com.stackovertwo.stackovertwo;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import java.util.concurrent.CompletableFuture;
#Service
public class WebScrapperService {
#Autowired
private RestTemplate restTemplate;
Logger logger = LoggerFactory.getLogger(WebScrapperService.class);
#Async
public CompletableFuture<SiteResponse> getWebScrappedContent(String webSiteURL)
//throws InterruptedException
{
logger.info("Starting: getWebScrappedContent for webSiteURL {} with thread {}", webSiteURL, Thread.currentThread().getName());
HttpEntity<String> response = restTemplate.exchange(webSiteURL,
HttpMethod.GET, null, String.class);
//Thread.sleep(1000);
SiteResponse webSiteSummary = null ;
String resultString = response.getBody();
HttpHeaders headers = response.getHeaders();
int statusCode = ((ResponseEntity<String>) response).getStatusCode().value();
System.out.println(statusCode);
System.out.println("HEADERS"+headers);
try
{
Document doc = (Document) Jsoup.parse(resultString);
Elements header = doc.select(".elementor-inner h2.elementor-heading-title.elementor-size-default");
System.out.println(header.get(0).html());
// Return the fragment.
webSiteSummary = new SiteResponse(statusCode, header.get(0).html());
}
catch(Exception e) {
System.out.println("Exception "+e.getMessage());
}
logger.info("Complete: getWebScrappedContent for webSiteURL {} with thread {}", webSiteURL, Thread.currentThread().getName());
return CompletableFuture.completedFuture(webSiteSummary);
}
}
SpringBootApp.java
package com.stackovertwo.stackovertwo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
//import javax.net.ssl.HostnameVerifier;
//import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
//import javax.net.ssl.SSLSession;
//import javax.net.ssl.TrustManager;
//import javax.net.ssl.X509TrustManager;
//import javax.security.cert.X509Certificate;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.impl.client.*;
import org.apache.http.conn.ssl.*;
#SpringBootApplication
public class SpringBootApp
{
public static void main(String[] args)
{
SpringApplication.run(SpringBootApp.class, args);
}
#Bean
public RestTemplate restTemplate() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException {
TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true;
SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom()
.loadTrustMaterial(null, acceptingTrustStrategy)
.build();
SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
CloseableHttpClient httpClient = HttpClients.custom()
.setSSLSocketFactory(csf)
.build();
HttpComponentsClientHttpRequestFactory requestFactory =
new HttpComponentsClientHttpRequestFactory();
requestFactory.setHttpClient(httpClient);
//return new RestTemplate();
RestTemplate restTemplate = new RestTemplate(requestFactory);
return restTemplate;
}
}
Note: I disabled SSL verification while calling the webulr in resttemplate, but its not recommendd inproduction (For assignment its ok). But you need to import the keys via java keystore in case production : https://myshittycode.com/2015/12/17/java-https-unable-to-find-valid-certification-path-to-requested-target-2/

Spring Boot Controllers not working 404 error

I'm developing a Spring boot application and my endpoints are not working at all. I'm getting the next issue:
{
"timestamp": "2020-06-21T21:12:39.716+0000",
"status": 404,
"error": "Not Found",
"message": "No message available",
"path": "/prduct/jjjk"
}
This is the structure of my project:
As you can see the controller is in the same path a subpackage of the Application class, but it is not scanned.
package com.mercadolibre.mutants.controllers;
import com.mercadolibre.mutants.entities.Product;
import com.mercadolibre.mutants.repositories.ProductRepository;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
#RestController
public class ProductController {
#Autowired
ProductRepository productRepository;
#GetMapping("/prduct/{id}")
public ResponseEntity<Product> findById(#PathVariable("id") String id) {
Optional<Product> p = productRepository.findById(id);
return ResponseEntity.ok(p.orElse(null));
}
#PostMapping(value = "/products", consumes = "application/json")
public ResponseEntity<Product> save(#RequestBody Product p) {
Product result = productRepository.save(p);
return ResponseEntity.ok(result);
}
}
This is my Application class:
package com.mercadolibre.mutants;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
#SpringBootApplication(scanBasePackages = "com.mercadolibre.mutants")
public class MutantsApplication {
public static void main(String[] args) {
SpringApplication.run(MutantsApplication.class, args);
}
}
I'm not able to identify the issue or what I'm missing. Any ideas?

Cannot using "response.setService/.setMesage/.setData" in java maven springboot project

I try to build API with springboot, when i make controller and using response.response.setService/.setMesage/.setData i get error "The method setService(String) is undefined for the type Response", please help me to solve this problem.
package com.example.restapi.controller;
import com.example.restapi.entity.Hardware;
import com.example.restapi.service.HardwareService;
import com.example.restapi.util.Response;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
#RestController
#RequestMapping(value = "hardware")
public class HardwareController {
#Autowired
HardwareService hardwareService;
private String serviceString = "Hardware";
ResponseEntity<Response> create (#RequestBody #Validated Hardware hardware)
{
String nameofCurrMethod = new Throwable()
.getStackTrace()[0]
.getMethodName();
Response response = new Response();
response.setService(this.getClass().getName() + nameofCurrMethod); <==This is error code
response.setMessage("Berhasil Membuat Data"); <==This is error code
response.setData(hardwareService.create(hardware)); <==This is error code
return ResponseEntity
.status(HttpStatus.OK)
.contentType(MediaType.APPLICATION_JSON)
.body(response);
}
}

Java spring boot AuthenticationManager injection faild

I am trying to implement the JWT authentication found here. here
While running I got the following error:
Field authenticationManager in controllers.SecurityController required
a bean of type
'org.springframework.security.authentication.AuthenticationManager'
that could not be found.
Field authenticationManager in controllers.SecurityController required
a bean of type
'org.springframework.security.authentication.AuthenticationManager'
that could not be found.
Action:
Consider defining a bean of type
'org.springframework.security.authentication.AuthenticationManager' in
your configuration.
Here is my controller:
package controllers;
import common.ServiceObjectResponse;
import entity.UserEntity;
import enums.Role;
import exception.CustomException;
import org.springframework.http.HttpStatus;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.core.AuthenticationException;
import request.LoginRequest;
import request.RegisterRequest;
import security.JwtTokenProvider;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import services.IUserService;
import javax.validation.Valid;
import java.util.ArrayList;
import java.util.List;
#RestController
#Api(tags = "diak", value = "SecurityService")
public class SecurityController
{
#Autowired
private JwtTokenProvider jwtTokenProvider;
#Autowired
private AuthenticationManager authenticationManager;
#Autowired
private IUserService _userService;
#ApiOperation(value = "login", nickname = "login", tags = "login")
#PostMapping("/api/login")
#ResponseBody
public String Login(#RequestBody #Valid LoginRequest data) throws Exception
{
ServiceObjectResponse<UserEntity> request = _userService.findByCredentials(data.Email, data.Jelszo);
if(!request.getIsSuccess())
{
throw new Exception(request.getMessage());
}
else if(request.getIsSuccess() && request.getObject() == null)
{
throw new Exception("No user found with given credantials!");
}
UserEntity user = request.getObject();
try
{
UsernamePasswordAuthenticationToken authenticationData = new UsernamePasswordAuthenticationToken(data.Email, data.Jelszo);
authenticationManager.authenticate(authenticationData);
List<Role> roles = new ArrayList<>();
roles.add(enums.Role.valueOf(user.Role));
return jwtTokenProvider.createToken(user.UniqID, roles);
}
catch (AuthenticationException e)
{
throw new CustomException("Invalid username/password supplied", HttpStatus.UNPROCESSABLE_ENTITY);
}
}
return jwtTokenProvider.createToken(user.UniqID, roles);
}
}
Am not good with java. If anyone can help.
thnx

#Autowired get null value

In Login class #Autowired is not null but when I try to use postman call Logout class #Autowired get null. I copy code from Login to Logout class just change variable name all code is same pattern I don't know what happening
package xxx.api.login
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import com.fasterxml.jackson.databind.ObjectMapper;
import xxx.api.domain.logout.LoginRequest;
import xxx.api.domain.logout.LoginResponse;
import xxx.converter.FieldValidation;
import xxx.dto.log.MessageLog;
import xxx.logging.LogFactory;
#Component
public class LoginClient {
#Autowired
FieldValidation fieldValidation;
public LoginResponse login(LoginRequest loginRequest) throws Exception {
ObjectMapper mapper = new ObjectMapper();
final String url = "http://localhost:xxx/xxx/login";
LoginResponse responseObject = null;
try {
String requestData = mapper.writeValueAsString(loginRequest);
responseObject = login(url, requestData);
fieldValidation.validateResponse(responseObject, "login");
// 'fieldValidation' not null
This is my Login class. Return result as I expect.
package xxx.api.logout;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import com.fasterxml.jackson.databind.ObjectMapper;
import xxx.api.domain.logout.LogoutRequest;
import xxx.api.domain.logout.LogoutResponse;
import xxx.converter.FieldValidation;
import xxx.dto.log.MessageLog;
import xxx.logging.LogFactory;
#Component
public class LogoutClient {
#Autowired
FieldValidation fieldValidation;
public LogoutResponse logout(LogoutRequest logoutRequest) throws Exception {
ObjectMapper mapper = new ObjectMapper();
final String url = "http://localhost:xxx/xxx/logout";
LogoutResponse responseObject = null;
try {
String requestData = mapper.writeValueAsString(logoutRequest);
responseObject = logout(url, requestData);
fieldValidation.validateResponse(responseObject, "logout");
// 'fieldValidation' is null then throws nullPointerException
This is my Logout class
package xxx.api.logout;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import xxx.api.domain.logout.LogoutRequest;
import xxx.api.domain.logout.LogoutResponse;
import xxx.dto.log.MessageLog;
import xxx.exception.common.ErrorException;
import xxx.api.logout.LogoutClient;
import xxx.converter.FieldValidation;
#RestController
#RequestMapping(path = "/xxx")
public class LogoutServer {
#Autowired
FieldValidation fieldValidation;
#Autowired
WSLogFactory wsLog;
#PostMapping(value = "/logout", consumes = MediaType.APPLICATION_JSON_UTF8_VALUE)
public ResponseEntity<?> logout(#RequestHeader(value = "requestId") String requestId,
#RequestBody(required = true) LogoutRequest logoutRequest) throws ErrorException, Exception {
writeLogRequest(logoutRequest, logoutRequest.getClientIP(), "Request", "/xxx/logout");
fieldValidation.validateRequest(logoutRequest, "logout");
try {
LogoutClient logoutClient = new LogoutClient();
LogoutResponse response = logoutClient.logout(logoutRequest);
return new ResponseEntity<LogoutResponse>(response, HttpStatus.OK);
} catch (Exception e) {
throw e;
}
}
This is Logout Controller
#org.springframework.context.annotation.Configuration
#PropertySource("file:${layout.properties_file}")
public class FieldValidation {
// do somethings
}
This is my FieldValidation class.
{
"timestamp": 1550490230074,
"status": 500,
"error": "Internal Server Error",
"exception": "java.lang.NullPointerException",
"message": "No message available",
"path": "/umm/logout"
}
This is return when I call my Logout class.
This is my project structure
xxx.api.login <<< LoginClient.java
xxx.api.logout <<< LogoutClient.java
xxx.converter <<< FieldValidation.java
This is my main program
package xxx;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
#SpringBootApplication
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}
I try to replace #Component to #Configuration but still doesn't works.
I don't need 'fieldValidation' in Logout class or another class be a null value.
package xxx.converter;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.Year;
import java.time.YearMonth;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import org.springframework.context.annotation.Configuration;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.validator.routines.BigDecimalValidator;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.PropertySource;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import com.fasterxml.jackson.databind.ObjectMapper;
import xxx.exception.common.ErrorException;
#Configuration
#PropertySource("file:${layout.properties_file}")
public class FieldValidation {
public void validateObject(Object objResponse, String config) throws Exception {
try {
List<Configuration> confs = objectMapper.readValue(config,
objectMapper.getTypeFactory().constructCollectionType(List.class, Configuration.class));
for (int i = 0; i < confs.size(); i++) {
Configuration configuration = confs.get(i);
Object objValue = getValue(configuration.getFieldName(), objResponse);
Validation validation = new Validation();
BeanUtils.copyProperties(validation, configuration);
isValid(objValue, validation);
if (configuration.getType().equalsIgnoreCase("object")) {
List<Validation> validations = configuration.getValidation();
Class<?> act = Class.forName(configuration.getClassName());
Object objectConfig = act.cast(objValue);
validateObject(objectConfig, validations);
}
}
} catch (Exception e) {
throw e;
}
}
This is full import of FieldValidation class and replace #Configuration and I get this error.
configuration.getFieldName() << error The method is undefined for type Configuration
configuration.getType() << error The method is undefined for type Configuration
configuration.getValidation() << error The method is undefined for type Configuration
configuration.getClassName() << error The method is undefined for type Configuration