We Keep Coding

Elasticsearch 8 Java Client ResponseException status line [HTTP/1.1 404 Not Found] {"statusCode":404,"error":"Not Found","message":"Not Found"}

this is Michael from germany and i am reaching out to you because i have to finish my master thesis before 31.10.2022.
I had working code in july 2022 which i didn't touch since then. I used the online 2 weeks trial vom elastic.co to connect and do my stuff. Now i want to execute the code again on a new 2 weeks trial but i am facing an error, which i can't solve.
What happened in the meanwhile:
a) in july i used version 8.2.0 of the software while meanwhile Elasticsearch company released
version 8.4.0 of their software.
b) i changed password in the code, because i have a new 2 weeks trial now.
Here are some short example codes which all fail (actually my real code is bigger but these are small examples). Only the code after the line // "productive code" changes
Creating an index:
package Kapselung;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
public class create_index {
public static void main(String[] args) {
String username = "";
String password = "";
String host = "";
int port =;
RestClient restClient;
ElasticsearchClient ESClient;
// https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/connecting.html
// https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/_basic_authentication.html
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
HttpClientConfigCallback httpClientConfigCallback = new HttpClientConfigCallback() {
#Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
};
// Create the low-level client
RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(host, port, "https"));
restClientBuilder.setHttpClientConfigCallback(httpClientConfigCallback);
restClient = restClientBuilder.build();
// Create the transport with a Jackson mapper
ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
ESClient = new ElasticsearchClient(transport);
// "productive code"
try {
ESClient.indices().create(c -> c.index("products100"));
} catch (Exception e) {
e.printStackTrace();
}
}
}
Failure is:
co.elastic.clients.transport.TransportException: [es/indices.create] Failed to decode error response
at co.elastic.clients.transport.rest_client.RestClientTransport.getHighLevelResponse(RestClientTransport.java:290)
at co.elastic.clients.transport.rest_client.RestClientTransport.performRequest(RestClientTransport.java:147)
at co.elastic.clients.elasticsearch.indices.ElasticsearchIndicesClient.create(ElasticsearchIndicesClient.java:266)
at co.elastic.clients.elasticsearch.indices.ElasticsearchIndicesClient.create(ElasticsearchIndicesClient.java:282)
at Kapselung.create_index.main(create_index.java:51)
Caused by: org.elasticsearch.client.ResponseException: method [PUT], host [https://gattinger-fourth.kb.us-central1.gcp.cloud.es.io:9243], URI [/products100], status line [HTTP/1.1 404 Not Found]
... 5 more
Or posting a document:
package Kapselung;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
import com.fasterxml.jackson.databind.ObjectMapper;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
public class post_document {
public static void main(String[] args) {
String username = "elastic";
String password = "JotkB5QY5v9IN0qRtvf6d5dL";
String host = "gattinger-fourth.kb.us-central1.gcp.cloud.es.io";
int port = 9243;
RestClient restClient;
ElasticsearchClient ESClient;
// https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/connecting.html
// https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/_basic_authentication.html
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
HttpClientConfigCallback httpClientConfigCallback = new HttpClientConfigCallback() {
#Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
};
// Create the low-level client
RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(host, port, "https"));
restClientBuilder.setHttpClientConfigCallback(httpClientConfigCallback);
restClient = restClientBuilder.build();
// Create the transport with a Jackson mapper
ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
ESClient = new ElasticsearchClient(transport);
// "productive code"
try {
Request request = new Request("POST", "/products100/_doc");
String document = "{\"sku\": \"21847816\",\"name\": \"Rasierapparat\",\"price\": 23.99}";
ObjectMapper mapper = new ObjectMapper(); // Jackson Json
request.setJsonEntity(mapper.writeValueAsString(document));
Response response = restClient.performRequest(request);
} catch (Exception e) {
e.printStackTrace();
}
}
}
failure:
org.elasticsearch.client.ResponseException: method [POST], host [https://gattinger-fourth.kb.us-central1.gcp.cloud.es.io:9243], URI [/products100/_doc], status line [HTTP/1.1 404 Not Found]
{"statusCode":404,"error":"Not Found","message":"Not Found"}
at org.elasticsearch.client.RestClient.convertResponse(RestClient.java:346)
at org.elasticsearch.client.RestClient.performRequest(RestClient.java:312)
at org.elasticsearch.client.RestClient.performRequest(RestClient.java:287)
at Kapselung.post_document.main(post_document.java:59)
Or just checking the Elasticsearch version:
package Kapselung;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
public class getESClient_info_version_number {
public static void main(String[] args) {
String username = "elastic";
String password = "JotkB5QY5v9IN0qRtvf6d5dL";
String host = "gattinger-fourth.kb.us-central1.gcp.cloud.es.io";
int port = 9243;
RestClient restClient;
ElasticsearchClient ESClient;
// https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/connecting.html
// https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/_basic_authentication.html
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
HttpClientConfigCallback httpClientConfigCallback = new HttpClientConfigCallback() {
#Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
};
// Create the low-level client
RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(host, port, "https"));
restClientBuilder.setHttpClientConfigCallback(httpClientConfigCallback);
restClient = restClientBuilder.build();
// Create the transport with a Jackson mapper
ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
ESClient = new ElasticsearchClient(transport);
// "productive code"
try {
System.out.println(ESClient.info().version().number());
} catch (Exception e) {
e.printStackTrace();
}
}
}
Failure:
co.elastic.clients.transport.TransportException: [es/info] Missing [X-Elastic-Product] header. Please check that you are connecting to an Elasticsearch instance, and that any networking filters are preserving that header.
at co.elastic.clients.transport.rest_client.RestClientTransport.checkProductHeader(RestClientTransport.java:351)
at co.elastic.clients.transport.rest_client.RestClientTransport.getHighLevelResponse(RestClientTransport.java:253)
at co.elastic.clients.transport.rest_client.RestClientTransport.performRequest(RestClientTransport.java:147)
at co.elastic.clients.elasticsearch.ElasticsearchClient.info(ElasticsearchClient.java:976)
at Kapselung.getESClient_info_version_number.main(getESClient_info_version_number.java:51)
Caused by: org.elasticsearch.client.ResponseException: method [GET], host [https://gattinger-fourth.kb.us-central1.gcp.cloud.es.io:9243], URI [/], status line [HTTP/1.1 200 OK]
And this is my pom.xml:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>00003</groupId>
<artifactId>00004</artifactId>
<version>0.0.1-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>8.2.0</version>
</dependency>
<!--
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>8.2.3</version>
</dependency>
-->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.12.3</version>
</dependency>
<!-- benötigt falls ClassNotFoundException: jakarta.json.spi.JsonProvider
https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/installation.html -->
<dependency>
<groupId>jakarta.json</groupId>
<artifactId>jakarta.json-api</artifactId>
<version>2.0.1</version>
</dependency>
<!-- -->
</dependencies>
</project>
Changing dependency from 8.2.0 to 8.4.0 didn't help.
So i don't know what actually to do. Do you?
Yes i know that i spread the password, it is intentional so somebody could help me by actually executing it. It is only a 2 weeks trial so i don't care about the logindata.
Thanks - Enomine

selenium - java - finding broken links

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.interactions.Actions;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import javax.swing.*;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.sql.Driver;
import java.time.Duration;
System.setProperty("webdriver.chrome.driver", "drivers/chromedriver.exe");
WebDriver driver = new ChromeDriver();
driver.get("https://demoqa.com/");
driver.manage().window().maximize();
HttpClient client = HttpClientBuilder.create().build();
HttpGet request = new HttpGet("https://demoqa.com/");
HttpResponse response = client.execute(request);
int statusCode = response.statusCode();
System.out.println(statusCode);
hi,
first of all, my first question is when I create the client object with HttpClient, for HttpClient
it throws an error and says "Required type:
HttpClient
Provided:
It gives a warning like "CloseableHttpClient".
Secondly
The execute method in HttpClient is not working.
Can you help me ?

You're mixing apples with pears, org.apache.http with java.net.http.
Maven dependency:
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
Code sample:
package tests;
import java.io.IOException;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
public class HttpClientTest {
public static void main(String[] args) throws ClientProtocolException, IOException {
HttpClient client = HttpClientBuilder.create().build();
HttpGet request = new HttpGet("https://demoqa.com/");
HttpResponse response = client.execute(request);
System.out.println(response.getStatusLine());
System.out.println(response.getStatusLine().getStatusCode());
}
}
Output:
HTTP/1.1 200 OK
200

How do I exstract data from multiple websites using restful API and Spring? [closed]

Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 1 year ago.
Improve this question
I got a task at school in which I have to do the following:
Implement the RESTful endpoint API, which simultaneously makes calls
to the following websites:
https://pizzerijalimbo.si/meni/
https://pizzerijalimbo.si/kontakt/
https://pizzerijalimbo.si/my-account/
https://pizzerijalimbo.si/o-nas/
The input for the endpoint is ‘integer’, which represents the number
of simultaneous calls to the above web pages (min 1 represents all
consecutive calls, max 4 represents all simultaneous calls).
Extracts a short title text from each page and saves this text in a
common global structure (array, folder (). The program should also
count successful calls. Finally, the service should list the number of
successful calls, the number of failed calls and the saved address
texts from all web pages.
With some help I managed to do something, but I still need help with data exstraction using Jsoup or any other method.
Here is the code that I have:
import java.util.Arrays;
import java.util.List;
import java.io.IOException;
import java.net.URL;
import java.util.Scanner;
#RestController
public class APIcontroller {
#Autowired
private RestTemplate restTemplate;
List<String> websites = Arrays.asList("https://pizzerijalimbo.si/meni/",
"https://pizzerijalimbo.si/kontakt/",
"https://pizzerijalimbo.si/my-account/",
"https://pizzerijalimbo.si/o-nas/");
#GetMapping("/podatki")
public List<Object> getData(#RequestParam(required = true) int numberOfWebsites) {
List<String> websitesToScrape = websites.subList(0, numberOfWebsites);
for (String website : websitesToScrape) {
Document doc = Jsoup.connect("https://pizzerijalimbo.si/meni/").get();
log(doc.title());
Elements newsHeadlines = doc.select("#mp-itn b a");
for (Element headline : newsHeadlines) {
log("%s\n\t%s",
headline.attr("title"), headline.absUrl("href"));
}
}
}
}
I also need to do it parallel, so the calls to a secific website go on at the same time.
But the main problem now is with the log funcion which does not work properly.
What I have tried:
I tried to solve the problem using Jsoup library, but I dont seem to
undersand it well, so I got an error in the for loop which says that
the method log is undefined. I also need to do a try catch to count possible failed calls and count the calls that are successfull as you can see in the task description.

WebScrapperController.java
package com.stackovertwo.stackovertwo;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
//import org.w3c.dom.Document;
//import org.w3c.dom.DocumentFragment;
import org.jsoup.nodes.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
#RestController
public class WebScrapperController {
#GetMapping("/")
public String index() {
return "Greetings from Spring Boot!";
}
// #Autowired
// private RestTemplate restTemplate;
#Autowired
WebScrapperService webScrapperService;
List<String> websites = Arrays.asList("https://pizzerijalimbo.si/meni/",
"https://pizzerijalimbo.si/kontakt/",
"https://pizzerijalimbo.si/my-account/",
"https://pizzerijalimbo.si/o-nas/");
#GetMapping("/podatki")
public ResponseEntity<Object> getData(#RequestParam(required = true) int numberOfWebsites) throws InterruptedException, ExecutionException {
List<SiteResponse> webSitesToScrape = new ArrayList<>();
// List<String> websitesToScrape = websites.subList(0, numberOfWebsites);
List<SiteResponse> responseResults = new ArrayList<SiteResponse>();
CompletableFuture<SiteResponse> futureData1 = webScrapperService.getWebScrappedContent(websites.get(0));
CompletableFuture<SiteResponse> futureData2 = webScrapperService.getWebScrappedContent(websites.get(1));
//CompletableFuture.allOf(futureData1, futureData2).join();
webSitesToScrape.add(futureData1.get());
webSitesToScrape.add(futureData2.get());
List<SiteResponse> result = webSitesToScrape.stream().collect(Collectors.toList());
return ResponseEntity.ok().body(result);
}
}
WebScrapperService.java
package com.stackovertwo.stackovertwo;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import java.util.concurrent.CompletableFuture;
#Service
public class WebScrapperService {
#Autowired
private RestTemplate restTemplate;
Logger logger = LoggerFactory.getLogger(WebScrapperService.class);
#Async
public CompletableFuture<SiteResponse> getWebScrappedContent(String webSiteURL)
//throws InterruptedException
{
logger.info("Starting: getWebScrappedContent for webSiteURL {} with thread {}", webSiteURL, Thread.currentThread().getName());
HttpEntity<String> response = restTemplate.exchange(webSiteURL,
HttpMethod.GET, null, String.class);
//Thread.sleep(1000);
SiteResponse webSiteSummary = null ;
String resultString = response.getBody();
HttpHeaders headers = response.getHeaders();
int statusCode = ((ResponseEntity<String>) response).getStatusCode().value();
System.out.println(statusCode);
System.out.println("HEADERS"+headers);
try
{
Document doc = (Document) Jsoup.parse(resultString);
Elements header = doc.select(".elementor-inner h2.elementor-heading-title.elementor-size-default");
System.out.println(header.get(0).html());
// Return the fragment.
webSiteSummary = new SiteResponse(statusCode, header.get(0).html());
}
catch(Exception e) {
System.out.println("Exception "+e.getMessage());
}
logger.info("Complete: getWebScrappedContent for webSiteURL {} with thread {}", webSiteURL, Thread.currentThread().getName());
return CompletableFuture.completedFuture(webSiteSummary);
}
}
SpringBootApp.java
package com.stackovertwo.stackovertwo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
//import javax.net.ssl.HostnameVerifier;
//import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
//import javax.net.ssl.SSLSession;
//import javax.net.ssl.TrustManager;
//import javax.net.ssl.X509TrustManager;
//import javax.security.cert.X509Certificate;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.impl.client.*;
import org.apache.http.conn.ssl.*;
#SpringBootApplication
public class SpringBootApp
{
public static void main(String[] args)
{
SpringApplication.run(SpringBootApp.class, args);
}
#Bean
public RestTemplate restTemplate() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException {
TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true;
SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom()
.loadTrustMaterial(null, acceptingTrustStrategy)
.build();
SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
CloseableHttpClient httpClient = HttpClients.custom()
.setSSLSocketFactory(csf)
.build();
HttpComponentsClientHttpRequestFactory requestFactory =
new HttpComponentsClientHttpRequestFactory();
requestFactory.setHttpClient(httpClient);
//return new RestTemplate();
RestTemplate restTemplate = new RestTemplate(requestFactory);
return restTemplate;
}
}
Note: I disabled SSL verification while calling the webulr in resttemplate, but its not recommendd inproduction (For assignment its ok). But you need to import the keys via java keystore in case production : https://myshittycode.com/2015/12/17/java-https-unable-to-find-valid-certification-path-to-requested-target-2/

java.lang.NoClassDefFoundError: org/apache/http/ssl/TrustStrategy, though it is present in classpath

I am running an application on tomcat server. I am getting following error while callling a specific function :
java.lang.NoClassDefFoundError: org/apache/http/ssl/TrustStrategy
My class is :
import java.io.ByteArrayOutputStream;
import java.io.Serializable;
import java.util.Map;
import javax.net.ssl.SSLContext;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
/*import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;*/
//import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HTTP;
import org.apache.log4j.Logger;
import org.performics.air.business.common.data.HttpParam;
public class SendAndReceiveUtil implements Serializable {
//FIXME: move to suitable package
/**
*
*/
private static final long serialVersionUID = 2649891233958197253L;
private static Logger LOG = Logger.getLogger(SendAndReceiveUtil.class);
public static String httpPostWithTLS(String request,String url,Map<String,String> headerParameterMap){
String responseStr = null;
try{
// FIXME: need to handle supplier timeout and gzip
String contentType="";
String soapAction="";
boolean zipForRequest=false;
boolean acceptEncoding = false;
boolean zipForResponse=false;
if (headerParameterMap!=null){
contentType=headerParameterMap.get(HttpParam.CONTENTTYPE.toString())!=null?headerParameterMap.get(HttpParam.CONTENTTYPE.toString()):"";
zipForRequest=(headerParameterMap.containsKey(HttpParam.ZIPFORREQUEST.toString()))? new Boolean(headerParameterMap.get(HttpParam.ZIPFORREQUEST.toString())):false;
acceptEncoding=(headerParameterMap.containsKey(HttpParam.ACCEPT_ENCODING.toString()))? new Boolean(headerParameterMap.get(HttpParam.ACCEPT_ENCODING.toString())):false;
zipForResponse=(headerParameterMap.containsKey(HttpParam.ZIPFORRESPONSE.toString()))? new Boolean(headerParameterMap.get(HttpParam.ZIPFORRESPONSE.toString())):false;
soapAction=headerParameterMap.get(HttpParam.SOAPACTION.toString())!=null?headerParameterMap.get(HttpParam.SOAPACTION.toString()):"";
}
SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(null,new TrustSelfSignedStrategy()).build();
// Allow TLSv1.2 protocol only, use NoopHostnameVerifier to trust self-singed cert
SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext,new String[] { "TLSv1.2" }, null, new NoopHostnameVerifier());
//do not set connection manager
HttpClient httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).build();
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("SOAPAction", soapAction);
StringEntity mEntity = new StringEntity(request, "UTF-8");
if(StringUtils.isNotBlank(contentType)){
mEntity.setContentType(contentType);
mEntity.setContentEncoding(new BasicHeader(HTTP.CONTENT_TYPE,contentType));
}else{
mEntity.setContentType("text/xml;charset=UTF-8");
mEntity.setContentEncoding(new BasicHeader(HTTP.CONTENT_ENCODING,"gzip"));
}
httpPost.addHeader("Content-Encoding", "gzip" );
httpPost.addHeader("Accept-Encoding", "gzip,deflate" );
if(null!=headerParameterMap.get("Cookie")){
httpPost.addHeader("Cookie", headerParameterMap.get("Cookie"));
}
httpPost.setEntity(mEntity);
HttpResponse response = httpclient.execute(httpPost);
HttpEntity et=response.getEntity();
ByteArrayOutputStream os = new ByteArrayOutputStream();
et.writeTo(os);
responseStr = new String(os.toByteArray());
}catch(Exception e){
LOG.error(e.getMessage(), e);
}
return responseStr;
}
}
Error comes on calling httpPostWithTLS() function. I searched about it on net and found that class was available at compile but is not available at run time, but i am unable to correct it.
I am using following http jars:
commons-httpclient-3.1.jar
httpclient-4.5.3.jar
httpcore-4.4.6.jar
httpclient-cache-4.5.3.jar
httpclient-win-4.5.3.jar
httpmime-4.5.3.jar

Try to add the httpcore-4.4.6.jar and httpclient-cache-4.5.3.jar to server library and check.
I do face the similar issue after adding these jar file, my problem is resolved.
Thanks,

Issue with SSL certificate incorporation in JEST elastic search rest java API

I am facing problem with connecting to elasticsearch(ELK) using Jest API. I am looking for rest based java API for ELK which supports SCROLL & SCAN and also support certificate based authentication.
I found out Jest and Flummi are the available one's.
Since Jest is popular and has more support, I am trying to use it.
My ELK db needs an certificate for authentication, which I have installed in Personal certificates.
My question is how do I use Jest Client to do a Http request with the certificate? I have found some code, but it dint help me. My code is below.
package pkg;
import io.searchbox.core.*;
import com.google.gson.JsonArray;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;
import org.apache.http.nio.conn.SchemeIOSessionStrategy;
import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.security.cert.CertificateException;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import java.io.IOException;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.params.Parameters;
public class ScrollELK {
private static final String INDEX = "daivb-logs";
private static final String TYPE = "cep";
public void scroll() throws IOException, KeyManagementException, NoSuchAlgorithmException, KeyStoreException, NoSuchProviderException {
//SSL certificate incorporation
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy()
{ #java.lang.Override
public boolean isTrusted(java.security.cert.X509Certificate[] x509Certificates, java.lang.String s) throws CertificateException {
return false;}
}).build();
// skip hostname checks
HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE;
SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
SchemeIOSessionStrategy httpsIOSessionStrategy = new SSLIOSessionStrategy(sslContext, hostnameVerifier);
// Get Jest client
HttpClientConfig clientConfig = new HttpClientConfig
.Builder("https://***.net/")
.multiThreaded(true)
.connTimeout(2000)
.defaultSchemeForDiscoveredNodes("https")
.sslSocketFactory(sslSocketFactory)
.httpsIOSessionStrategy(httpsIOSessionStrategy)
.build();
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(clientConfig);
JestClient client = factory.getObject();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchAllQuery());
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(INDEX)
.addType(TYPE)
.setParameter(Parameters.SIZE, 100)
.setParameter(Parameters.SCROLL, "5m")
.setParameter(Parameters.SEARCH_TYPE, "scan")
.build();
JestResult result = client.execute(search);
JsonArray hits = result.getJsonObject().getAsJsonObject("hits").getAsJsonArray("hits");
String scrollId = result.getJsonObject().get("_scroll_id").getAsString();
int count =0;
do
{ SearchScroll scroll = new SearchScroll.Builder(scrollId, "5m")
.build();
result = client.execute(scroll);
hits = result.getJsonObject().getAsJsonObject("hits").getAsJsonArray("hits");
scrollId = result.getJsonObject().getAsJsonPrimitive("_scroll_id").getAsString();
count =result.getJsonObject().getAsJsonObject("hits").getAsJsonArray("hits").size();
System.out.println(count);
}while(count>0);
// clear a single scroll id
ClearScroll clearScroll = new ClearScroll.Builder().addScrollId(scrollId).build();
result = client.execute(clearScroll);
}
}
Pls suggest on the issue.

private SSLContext createSSLContext() {
try {
SSLContextBuilder sslBuilder = SSLContexts.custom().loadTrustMaterial(null, (x509Certificates, s) -> true);
final SSLContext sslContext = sslBuilder.build();
return sslContext;
} catch (Exception e) {
LOGGER.error("cannot create SSLContext", e);
}
return null;
}