I want to set up a client with x connections, where each connection will send a new GET request to a different URL each random time.
So my approach is to create x threads, where each will send a GET request in a loop (till the program will terminate).
I was using Apache MultiThreadedHttpConnectionManager, where I set the number of possible connections as x.
ClientHandler(int numberOfClients) {
this.numberOfClients = numberOfClients;
connManager = new PoolingHttpClientConnectionManager();
HttpClientBuilder clientBuilder = HttpClients.custom().setConnectionManager(connManager);
connManager.setMaxTotal(numberOfClients);
CloseableHttpClient httpClient = clientBuilder.build();
generateMultiThreadedClient(httpClient);
}
private void generateMultiThreadedClient(CloseableHttpClient httpClient) {
for (int i = 0; i < numberOfClients; i++) {
String clientUrl = URL + i;
HttpGet httpGet = new HttpGet(clientUrl);
ClientMultiThreaded clientMultiThreaded = new ClientMultiThreaded(httpClient,httpGet, i);
LOGGER.info(CLIENT_LOG + "A new thread for clientId " + i + " was created.");
clientMultiThreaded.start();
LOGGER.info(CLIENT_LOG + "Thread for clientId " + i + " started..");
}
}
And this is my run() method of the thread.
ClientMultiThreaded(CloseableHttpClient httpClient, HttpGet httpGet, int clientId) {
this.httpClient = httpClient;
this.clientId = clientId;
this.httpGet = httpGet;
randomGenerator = new Random();
}
#Override
public void run() {
try{
// Execute request for the first time
executeRequest(httpGet);
while (true) {
int timeToSleep = randomGenerator.nextInt(BOUND_LIMIT) + 1;
LOGGER.info(CLIENT_LOG + "Thread id " + clientId + " went to sleep for " + timeToSleep / 1000 + " seconds");
sleep(timeToSleep);
LOGGER.info(" ------- This is a test log printing for clientId: " +clientId);
executeRequest(httpGet);
}
}catch(Exception e) {
LOGGER.info(e.getMessage());
}
}
private void executeRequest(HttpGet httpGet) throws IOException {
CloseableHttpResponse response = httpClient.execute(httpGet);
LOGGER.info(CLIENT_LOG + "clientId " + clientId + " sent get request to URL " + httpGet.getURI());
statusCode = this.response.getStatusLine().getStatusCode();
LOGGER.info(CLIENT_LOG + "Status received for clientId " + clientId + " is: " + statusCode);
}
The problem is that I actually get 2 GET requests sent and then it stops.
You need to close response, use try with resources
Use following code:
private void executeRequest(HttpGet httpGet) throws IOException {
try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
LOGGER.info(CLIENT_LOG + "clientId " + clientId + " sent get request to URL " + httpGet.getURI());
statusCode = this.response.getStatusLine().getStatusCode();
LOGGER.info(CLIENT_LOG + "Status received for clientId " + clientId + " is: " + statusCode);
}
}
Related
I am trying to create a RestAPI request to fetch details of a records with a filter 'START_WITH'. But every request I send is getting rejected with the error '401 Unauthorized. Invalid Signature'. But if I don't add the filter and only send the normal request, It gives correct response back.
My code for generating signature is:
public String generateOauthHeader(
String method, UserFields userFields, String baseUrl, String offset, String limit) {
long timestamp = new Date().getTime() / 1000;
String nonce = getAlphaNumericString();
ArrayList<String> parameters = new ArrayList<>();
parameters.add(
ApplicationConstants.CONSUMER_KEY
+ ApplicationConstants.EQUAL
+ userFields.getConsumerKey());
parameters.add(ApplicationConstants.NONCE + ApplicationConstants.EQUAL + nonce);
parameters.add(
ApplicationConstants.SIGNATURE_METHOD_KEY
+ ApplicationConstants.EQUAL
+ ApplicationConstants.SIGNATURE_METHOD_VAL);
parameters.add(ApplicationConstants.TIMESTAMP + ApplicationConstants.EQUAL + timestamp);
parameters.add(
ApplicationConstants.OAUTH_TOKEN + ApplicationConstants.EQUAL + userFields.getTokenId());
parameters.add(
ApplicationConstants.VERSION_KEY
+ ApplicationConstants.EQUAL
+ ApplicationConstants.VERSION_VAL);
if (offset != null) {
parameters.add(ApplicationConstants.OFFSET + ApplicationConstants.EQUAL + offset);
}
if (limit != null) {
parameters.add(ApplicationConstants.LIMIT_CLAUSE + ApplicationConstants.EQUAL + limit);
}
parameters.add("q" + ApplicationConstants.EQUAL + "companyname START_WITH COMP1234");
Collections.sort(parameters);
StringBuffer parametersList = new StringBuffer();
for (int i = 0; i < parameters.size(); i++) {
parametersList.append(((i > 0) ? ApplicationConstants.AMPERSAND : "") + parameters.get(i));
}
String signature = null;
try {
String signatureString =
method
+ ApplicationConstants.AMPERSAND
+ URLEncoder.encode(baseUrl, StandardCharsets.UTF_8)
+ ApplicationConstants.AMPERSAND
+ URLEncoder.encode(parametersList.toString(), StandardCharsets.UTF_8);
SecretKeySpec signingKey =
new SecretKeySpec(
(userFields.getConsumerSecret()
+ ApplicationConstants.AMPERSAND
+ userFields.getTokenSecret())
.getBytes(),
ApplicationConstants.HMACSHA256);
Mac m = Mac.getInstance(ApplicationConstants.HMACSHA256);
m.init(signingKey);
m.update(signatureString.getBytes());
byte[] res = m.doFinal();
signature = Base64Coder.encodeLines(res);
} catch (Exception e) {
e.printStackTrace();
}
return ApplicationConstants.OAUTH
+ ApplicationConstants.REALM
+ ApplicationConstants.EQUAL_START_QUOTES
+ userFields.getAccountId()
+ ApplicationConstants.END_QUOTES
+ ApplicationConstants.CONSUMER_KEY
+ ApplicationConstants.EQUAL_START_QUOTES
+ userFields.getConsumerKey()
+ ApplicationConstants.END_QUOTES
+ ApplicationConstants.OAUTH_TOKEN
+ ApplicationConstants.EQUAL_START_QUOTES
+ userFields.getTokenId()
+ ApplicationConstants.END_QUOTES
+ ApplicationConstants.SIGNATURE_METHOD_KEY
+ ApplicationConstants.EQUAL_START_QUOTES
+ ApplicationConstants.SIGNATURE_METHOD_VAL
+ ApplicationConstants.END_QUOTES
+ ApplicationConstants.TIMESTAMP
+ ApplicationConstants.EQUAL_START_QUOTES
+ timestamp
+ ApplicationConstants.END_QUOTES
+ ApplicationConstants.NONCE
+ ApplicationConstants.EQUAL_START_QUOTES
+ nonce
+ ApplicationConstants.END_QUOTES
+ ApplicationConstants.VERSION_KEY
+ ApplicationConstants.EQUAL_START_QUOTES
+ ApplicationConstants.VERSION_VAL
+ ApplicationConstants.END_QUOTES
+ ApplicationConstants.SIGNATURE
+ ApplicationConstants.EQUAL_START_QUOTES
+ URLEncoder.encode(signature, StandardCharsets.UTF_8)
+ ApplicationConstants.QUOTES;
}
Url generated without filter: https ://1212112-sb1.suitetalk.api.netsuite.com/services/rest/record/v1/customer/
Url generated with filter: https ://1212112-sb1.suitetalk.api.netsuite.com/services/rest/record/v1/customer?q="companyName START_WITH COMP1234"
I am writing a bot to auto purchase items on a website (zalando).
Everything goes well from login to adding items to shopping cart but at the very end it doesn't work anyomore. It sends this error:
{ "edge_error": "halt", "ref_id": "18.57c51102.1663765843.299fc0e", "wait": 60, "feedback": { "email": true, "url": "", "recaptcha": { "enabled": false, "type": 0, "sitekey": "" } }}
I think it has something to do with their protection or just me missing a header or cookie or a delay... I honestly have no clue anymore
This is the code I use in the end (to checkout and generate a paypal link (post response)):
public void makePostJsonRequest(WebDriver driver, String eTag, String checkoutID)
{
retrieveCookiesMap(driver);
HttpClient httpClient = new DefaultHttpClient();
try {
HttpPost postRequest = new HttpPost("https://www.zalando.be/api/checkout/buy-now");
postRequest.setHeader("authority", "www.zalando.be");
postRequest.setHeader("accept", "application/json");
postRequest.setHeader("accept-language", "en-US,en;q=0.9");
postRequest.setHeader("content-type", "application/json");
postRequest.setHeader("cookie", "language-preference=nl;" +
" Zalando-Client-Id=" + cookiesMap.get("Zalando-Client-Id") + ";" +
" ncx=f;" +
" _gcl_au=" + cookiesMap.get("_gcl_au") + ";" +
" sqt_cap=" + cookiesMap.get("sqt_cap") + ";" +
" _ga=" + cookiesMap.get("_ga") + ";" +
" _gid=" + cookiesMap.get("_gid") + ";" +
" bm_sz=" + cookiesMap.get("bm_sz") + ";" +
" ak_bms=" + cookiesMap.get("ak_bms") + ";" +
" _gat_zalga=1;" +
" mpulseinject=false;" +
" frsx=" + cookiesMap.get("frsx") + ";" +
" zsa=" + cookiesMap.get("zsa") + ";" +
" zsr=" + cookiesMap.get("zsr") + ";" +
" zsi=" + cookiesMap.get("zsi") + ";" +
" bm_sv=" + cookiesMap.get("bm_sv") + ";" +
" _abck=" + cookiesMap.get("_abck") + ";");
postRequest.setHeader("origin", "https://www.zalando.be");
postRequest.setHeader("referer", "https://www.zalando.be/checkout/confirm");
postRequest.setHeader("sec-ch-ua", "\"Chromium\";v=\"104\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"104\"");
postRequest.setHeader("sec-ch-ua-mobile", "?0");
postRequest.setHeader("sec-ch-ua-platform", "\"Linux\"");
postRequest.setHeader("sec-fetch-dest", "empty");
postRequest.setHeader("sec-fetch-mode", "cors");
postRequest.setHeader("sec-fetch-site", "same-origin");
postRequest.setHeader("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36");
postRequest.setHeader("x-xsrf-token", cookiesMap.get("frsx"));
postRequest.setHeader("x-zalando-checkout-app", "web");
postRequest.setHeader("x-zalando-footer-mode", "desktop");
postRequest.setHeader("x-zalando-header-mode", "desktop");
eTag = StringUtils.chop(eTag);
eTag += "\\";
String jsonString = "{\"checkoutId\":\"" + checkoutID + "\"," +
"\"eTag\":" + "\"\\" + eTag + "\"" + "\"" + "}";
System.out.println(jsonString);
StringEntity entity = new StringEntity(jsonString);
postRequest.setEntity(entity);
long startTime = System.currentTimeMillis();
HttpResponse response = httpClient.execute(postRequest);
long elapsedTime = System.currentTimeMillis() - startTime;
System.out.println("Time taken : "+elapsedTime+"ms");
InputStream is = response.getEntity().getContent();
Reader reader = new InputStreamReader(is);
BufferedReader bufferedReader = new BufferedReader(reader);
StringBuilder builder = new StringBuilder();
while (true) {
try {
String line = bufferedReader.readLine();
if (line != null) {
builder.append(line);
} else {
break;
}
} catch (Exception e) {
e.printStackTrace();
}
}
System.out.println(builder.toString());
System.out.println("****************");
} catch (Exception ex) {
ex.printStackTrace();
}
}
This means that Akamai (the provider that zalando uses for bot protection), has detected and stopped your request because it detected you as a bot. To avoid this "stop" you MUST send a valid _abck cookie, generated by passing sensor data to the zalando akamai endpoint( you can find using chrome devtools and analyzing the requests )
Question: How do I add a authorization proxy to OkHTTP.
I have a using a proxy with ip:port:username:password, although when i run my program i get an error saying INFO: <-- HTTP FAILED: java.io.IOException: unexpected end of stream on https://www.footlocker.nl/... I used the Authentication method with Credentials.basic() but finalobject.getusername and finalobject.getpassword() return the full proxy (ip:port:username:password) instead of just the username and password. My Credentials are returning a
Basic credentialresponse like: Credentials Info: Basic dXNlci1tcGgwMV9GNVdyWC1jb3VudHJ5LW5sLXNlc3Npb23tRjhBRE5BYmc7UmFnZTFqS1ZyVGlGQTU=.
Also, proxyRandom contains an Arraylist of proxyObjects.
See image for more info:
enter image description here
Here is the code:
public Footlockerv1(String base, String keywords, String size, Billing.ProfileObject profile,
ArrayList<ProxyObjects.ProxyObject> proxyRandom) {
super();
ProxyObjects.ProxyObject object = null;
for (ProxyObjects.ProxyObject b : proxyRandom) {
Random randomGenerator = new Random();
int index = randomGenerator.nextInt(proxyRandom.size());
System.out.println("Random Proxies: " + proxyRandom.get(index));
b = proxyRandom.get(index);
object = new ProxyObjects.ProxyObject(b.getAddress(), b.getPort(), b.getUsername(), b.getPassword());
}
System.out.println("Full Proxy name: " + object +
"\nConstructed Proxy Name "
+ "Addres: " + object.getAddress() + ". "
+ "Port: " + object.getPort() + ". "
+ "Username: " + object.getUsername() + ". "
+ "Password: " + object.getPassword());
final Gson gson = new GsonBuilder()
.setLenient()
.create();
final HttpLoggingInterceptor interceptorv2 = new HttpLoggingInterceptor();
interceptorv2.setLevel(HttpLoggingInterceptor.Level.BODY);
InetSocketAddress proxyAddr = new InetSocketAddress(object.getAddress(), object.getPort());
java.net.Proxy proxyy = new java.net.Proxy(Proxy.Type.HTTP, proxyAddr);
System.out.println("\n Used proxy: " + proxyy);
ProxyObjects.ProxyObject finalObject;
finalObject = object;
System.out.println("Final object: " + finalObject);
Authenticator proxyAuthenticatorr = (route, response) -> {
String credentialTest = Credentials.basic(finalObject.getUsername(), finalObject.getPassword());
System.out.println("Credentials Info: " + credentialTest + " . Info are: " + finalObject.getUsername() + " " +
" & " + finalObject.getPassword());
System.setProperty("https.proxyUser", finalObject.getUsername());
System.setProperty("https.proxyPassword", finalObject.getPassword());
System.setProperty("jdk.https.auth.tunneling.disabledSchemes", "");
return response.request().newBuilder()
.header("Proxy-Authorization", credentialTest)
.build();
};
client = new OkHttpClient.Builder()
.proxy(proxyy)
.proxyAuthenticator(proxyAuthenticatorr)
// .authenticator(proxyAuthenticatorr)
.connectTimeout(60, TimeUnit.MILLISECONDS)
.writeTimeout(60, TimeUnit.MILLISECONDS)
.readTimeout(60, TimeUnit.MILLISECONDS)
.cookieJar(new SessionCookieJar())
.addInterceptor(chain ->
{
Request request = chain.request();
Request.Builder newRequest = request.newBuilder()
.header("Accept", "*/*")
.method("GET", null)
.header("DNT", String.valueOf(1))
.header("Accept-Language", "en-US,en;q=0.9")
.header("User-Agent", RandomUserAgent.getRandomUserAgent()
);
System.out.println("Request: " + request);
return chain.proceed(newRequest.build());
}).addInterceptor(interceptorv2)
.build();
final Retrofit retrofit = new Retrofit.Builder()
.addConverterFactory(GsonConverterFactory.create(gson))
.baseUrl(base)
.client(client)
.build();
api = retrofit.create(API_Retrofit.class);
this.proxy = proxyRandom;
this.base = base;
this.keywords = keywords;
this.size = size;
this.profile = profile;
debug = true;
}
I have two nearly identical methods in Java. The only difference is that they have different parameter types. They use generics and return the type T of the input parameter. How can I get rid of the duplicate code? Here are my two methods. Ultimately they both call Spring restTemplate.exchange() with the different types. Otherwise the methods are identical.
public <T> T send(Class<T> expectedClass) throws KenectRestClientException {
if (this.logRequest) log.info("Making request: " + this.toString());
HttpEntity<Object> entity = new HttpEntity<>(this.body, getTokenHeaders(this.token));
RestTemplate restTemplate = new RestTemplate();
String compiledUrl = UrlUtils.replacePathParamsInUrl(this.url, this.pathParams, this.uriEncode) + UrlUtils.compileRequestParamsToUrl(this.requestParams, this.uriEncode);
Object[] incompatibleStrings = extractIncompatibleStrings(compiledUrl);
try {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, expectedClass, incompatibleStrings).getBody();
} catch(RestClientResponseException e) {
log.warning(RestClientResponseException.class.getSimpleName() +
" url: " + url +
" status code: " + e.getRawStatusCode() +
" error body: " + e.getResponseBodyAsString() +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
} catch(ResourceAccessException e) {
log.warning("Resource access exception " +
" url: " + url +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
}
}
public <T> T send(ParameterizedTypeReference<T> parameterizedTypeReference) throws KenectRestClientException {
if (this.logRequest) log.info("Making request: " + this.toString());
HttpEntity<Object> entity = new HttpEntity<>(this.body, getTokenHeaders(this.token));
RestTemplate restTemplate = new RestTemplate();
String compiledUrl = UrlUtils.replacePathParamsInUrl(this.url, this.pathParams, this.uriEncode) + UrlUtils.compileRequestParamsToUrl(this.requestParams, this.uriEncode);
Object[] incompatibleStrings = extractIncompatibleStrings(compiledUrl);
try {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, parameterizedTypeReference, incompatibleStrings).getBody();
} catch(RestClientResponseException e) {
log.warning(RestClientResponseException.class.getSimpleName() +
" url: " + url +
" status code: " + e.getRawStatusCode() +
" error body: " + e.getResponseBodyAsString() +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
} catch(ResourceAccessException e) {
log.warning("Resource access exception " +
" url: " + url +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
}
}
What about passing a boolean flag, based on which you call exchange() passing either a Class<T> or a ParameterizedTypeReference<T>?
public <T> T send(T neededType, boolean flag) throws KenectRestClientException {
if (this.logRequest) log.info("Making request: " + this.toString());
HttpEntity<Object> entity = new HttpEntity<>(this.body, getTokenHeaders(this.token));
RestTemplate restTemplate = new RestTemplate();
String compiledUrl = UrlUtils.replacePathParamsInUrl(this.url, this.pathParams, this.uriEncode) + UrlUtils.compileRequestParamsToUrl(this.requestParams, this.uriEncode);
Object[] incompatibleStrings = extractIncompatibleStrings(compiledUrl);
try {
if (flag) {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, neededType.getClass(), incompatibleStrings).getBody();
} else {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, ParameterizedTypeReference.forType(neededType.getClass()), incompatibleStrings).getBody();
}
} catch(RestClientResponseException e) {
log.warning(RestClientResponseException.class.getSimpleName() +
" url: " + url +
" status code: " + e.getRawStatusCode() +
" error body: " + e.getResponseBodyAsString() +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
} catch(ResourceAccessException e) {
log.warning("Resource access exception " +
" url: " + url +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
}
}
I am using twitter home_timeline API for showing tweets.For first time I run this it works fine, but when I call it again (pull to load more), it doesn't responds. I am passing auth headers and in params I passed count, it doesn't worked too.
I don't know where am I stuck..
Here is code for pulling tweets:
if (auth != null && auth.token_type.equals("bearer")) {
HttpGet httpget = new HttpGet(TwitterStreamURL);
String oAuthConsumerKey = CONSUMER_KEY;
String oAuthConsumerSecret = CONSUMER_SECRET;
String oAuthAccessToken = HomeActivity.twitter_access_token;
String oAuthAccessTokenSecret = HomeActivity.twitter_access_token_secret;
String oAuthNonce = String.valueOf(System.currentTimeMillis());
String oAuthSignatureMethod = "HMAC-SHA1";
String oAuthTimestamp = time();
String oAuthVersion = "1.0";
String signatureBaseString1 = methods;
String signatureBaseString2 = TwitterStreamURL;
String signatureBaseString3Templ = "oauth_consumer_key=%s&oauth_nonce=%s&oauth_signature_method=%s&oauth_timestamp=%s&oauth_token=%s&oauth_version=%s";
String signatureBaseString3 = String.format(signatureBaseString3Templ,
oAuthConsumerKey,
oAuthNonce,
oAuthSignatureMethod,
oAuthTimestamp,
oAuthAccessToken,
oAuthVersion);
String signatureBaseStringTemplate = "%s&%s&%s";
try {
signatureBaseString = String.format(signatureBaseStringTemplate,
URLEncoder.encode(signatureBaseString1, "UTF-8"),
URLEncoder.encode(signatureBaseString2, "UTF-8"),
URLEncoder.encode(signatureBaseString3, "UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
try {
compositeKey = URLEncoder.encode(oAuthConsumerSecret, "UTF-8") + "&" + URLEncoder.encode(oAuthAccessTokenSecret, "UTF-8");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
try {
String oAuthSignature = computeSignature(signatureBaseString, compositeKey);
oAuthSignatureEncoded = URLEncoder.encode(oAuthSignature, "UTF-8");
String authorizationHeaderValueTempl = "OAuth oauth_consumer_key=\"%s\", oauth_nonce=\"%s\", oauth_signature=\"%s\", oauth_signature_method=\"%s\", oauth_timestamp=\"%s\", oauth_token=\"%s\", oauth_version=\"%s\"";
String authorizationHeaderValue = String.format(authorizationHeaderValueTempl,
oAuthConsumerKey,
oAuthNonce,
oAuthSignatureEncoded,
oAuthSignatureMethod,
oAuthTimestamp,
oAuthAccessToken,
oAuthVersion);
} catch (Exception e) {
e.printStackTrace();
}
String vf = "oauth_consumer_key=" + oAuthConsumerKey + ",oauth_signature_method=" + oAuthSignatureMethod + ",oauth_timestamp=" + oAuthTimestamp + ",oauth_nonce=" + oAuthNonce + ",oauth_version=" + oAuthVersion + ",oauth_token=" + oAuthAccessToken + ",oauth_signature=" + oAuthSignatureEncoded + "";
httpget.setHeader("Authorization", "OAuth " + "oauth_consumer_key=" + oAuthConsumerKey + ",oauth_signature_method=" + oAuthSignatureMethod + ",oauth_timestamp=" + oAuthTimestamp + ",oauth_nonce=" + oAuthNonce + ",oauth_version=" + oAuthVersion + ",oauth_token=" + oAuthAccessToken + ",oauth_signature=" + oAuthSignatureEncoded + "");
httpget.setHeader("Content-Type", "application/json");
// update the results with the body of the response
checkTwitRes = true;
results = getResponseBody(httpget);
}
div {
color:#000;
font-weight:800
}
p {
color:red;
background:url("mypicture.png")
}