Question: How do I add a authorization proxy to OkHTTP.
I have a using a proxy with ip:port:username:password, although when i run my program i get an error saying INFO: <-- HTTP FAILED: java.io.IOException: unexpected end of stream on https://www.footlocker.nl/... I used the Authentication method with Credentials.basic() but finalobject.getusername and finalobject.getpassword() return the full proxy (ip:port:username:password) instead of just the username and password. My Credentials are returning a
Basic credentialresponse like: Credentials Info: Basic dXNlci1tcGgwMV9GNVdyWC1jb3VudHJ5LW5sLXNlc3Npb23tRjhBRE5BYmc7UmFnZTFqS1ZyVGlGQTU=.
Also, proxyRandom contains an Arraylist of proxyObjects.
See image for more info:
enter image description here
Here is the code:
public Footlockerv1(String base, String keywords, String size, Billing.ProfileObject profile,
ArrayList<ProxyObjects.ProxyObject> proxyRandom) {
super();
ProxyObjects.ProxyObject object = null;
for (ProxyObjects.ProxyObject b : proxyRandom) {
Random randomGenerator = new Random();
int index = randomGenerator.nextInt(proxyRandom.size());
System.out.println("Random Proxies: " + proxyRandom.get(index));
b = proxyRandom.get(index);
object = new ProxyObjects.ProxyObject(b.getAddress(), b.getPort(), b.getUsername(), b.getPassword());
}
System.out.println("Full Proxy name: " + object +
"\nConstructed Proxy Name "
+ "Addres: " + object.getAddress() + ". "
+ "Port: " + object.getPort() + ". "
+ "Username: " + object.getUsername() + ". "
+ "Password: " + object.getPassword());
final Gson gson = new GsonBuilder()
.setLenient()
.create();
final HttpLoggingInterceptor interceptorv2 = new HttpLoggingInterceptor();
interceptorv2.setLevel(HttpLoggingInterceptor.Level.BODY);
InetSocketAddress proxyAddr = new InetSocketAddress(object.getAddress(), object.getPort());
java.net.Proxy proxyy = new java.net.Proxy(Proxy.Type.HTTP, proxyAddr);
System.out.println("\n Used proxy: " + proxyy);
ProxyObjects.ProxyObject finalObject;
finalObject = object;
System.out.println("Final object: " + finalObject);
Authenticator proxyAuthenticatorr = (route, response) -> {
String credentialTest = Credentials.basic(finalObject.getUsername(), finalObject.getPassword());
System.out.println("Credentials Info: " + credentialTest + " . Info are: " + finalObject.getUsername() + " " +
" & " + finalObject.getPassword());
System.setProperty("https.proxyUser", finalObject.getUsername());
System.setProperty("https.proxyPassword", finalObject.getPassword());
System.setProperty("jdk.https.auth.tunneling.disabledSchemes", "");
return response.request().newBuilder()
.header("Proxy-Authorization", credentialTest)
.build();
};
client = new OkHttpClient.Builder()
.proxy(proxyy)
.proxyAuthenticator(proxyAuthenticatorr)
// .authenticator(proxyAuthenticatorr)
.connectTimeout(60, TimeUnit.MILLISECONDS)
.writeTimeout(60, TimeUnit.MILLISECONDS)
.readTimeout(60, TimeUnit.MILLISECONDS)
.cookieJar(new SessionCookieJar())
.addInterceptor(chain ->
{
Request request = chain.request();
Request.Builder newRequest = request.newBuilder()
.header("Accept", "*/*")
.method("GET", null)
.header("DNT", String.valueOf(1))
.header("Accept-Language", "en-US,en;q=0.9")
.header("User-Agent", RandomUserAgent.getRandomUserAgent()
);
System.out.println("Request: " + request);
return chain.proceed(newRequest.build());
}).addInterceptor(interceptorv2)
.build();
final Retrofit retrofit = new Retrofit.Builder()
.addConverterFactory(GsonConverterFactory.create(gson))
.baseUrl(base)
.client(client)
.build();
api = retrofit.create(API_Retrofit.class);
this.proxy = proxyRandom;
this.base = base;
this.keywords = keywords;
this.size = size;
this.profile = profile;
debug = true;
}
Related
I am writing a bot to auto purchase items on a website (zalando).
Everything goes well from login to adding items to shopping cart but at the very end it doesn't work anyomore. It sends this error:
{ "edge_error": "halt", "ref_id": "18.57c51102.1663765843.299fc0e", "wait": 60, "feedback": { "email": true, "url": "", "recaptcha": { "enabled": false, "type": 0, "sitekey": "" } }}
I think it has something to do with their protection or just me missing a header or cookie or a delay... I honestly have no clue anymore
This is the code I use in the end (to checkout and generate a paypal link (post response)):
public void makePostJsonRequest(WebDriver driver, String eTag, String checkoutID)
{
retrieveCookiesMap(driver);
HttpClient httpClient = new DefaultHttpClient();
try {
HttpPost postRequest = new HttpPost("https://www.zalando.be/api/checkout/buy-now");
postRequest.setHeader("authority", "www.zalando.be");
postRequest.setHeader("accept", "application/json");
postRequest.setHeader("accept-language", "en-US,en;q=0.9");
postRequest.setHeader("content-type", "application/json");
postRequest.setHeader("cookie", "language-preference=nl;" +
" Zalando-Client-Id=" + cookiesMap.get("Zalando-Client-Id") + ";" +
" ncx=f;" +
" _gcl_au=" + cookiesMap.get("_gcl_au") + ";" +
" sqt_cap=" + cookiesMap.get("sqt_cap") + ";" +
" _ga=" + cookiesMap.get("_ga") + ";" +
" _gid=" + cookiesMap.get("_gid") + ";" +
" bm_sz=" + cookiesMap.get("bm_sz") + ";" +
" ak_bms=" + cookiesMap.get("ak_bms") + ";" +
" _gat_zalga=1;" +
" mpulseinject=false;" +
" frsx=" + cookiesMap.get("frsx") + ";" +
" zsa=" + cookiesMap.get("zsa") + ";" +
" zsr=" + cookiesMap.get("zsr") + ";" +
" zsi=" + cookiesMap.get("zsi") + ";" +
" bm_sv=" + cookiesMap.get("bm_sv") + ";" +
" _abck=" + cookiesMap.get("_abck") + ";");
postRequest.setHeader("origin", "https://www.zalando.be");
postRequest.setHeader("referer", "https://www.zalando.be/checkout/confirm");
postRequest.setHeader("sec-ch-ua", "\"Chromium\";v=\"104\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"104\"");
postRequest.setHeader("sec-ch-ua-mobile", "?0");
postRequest.setHeader("sec-ch-ua-platform", "\"Linux\"");
postRequest.setHeader("sec-fetch-dest", "empty");
postRequest.setHeader("sec-fetch-mode", "cors");
postRequest.setHeader("sec-fetch-site", "same-origin");
postRequest.setHeader("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36");
postRequest.setHeader("x-xsrf-token", cookiesMap.get("frsx"));
postRequest.setHeader("x-zalando-checkout-app", "web");
postRequest.setHeader("x-zalando-footer-mode", "desktop");
postRequest.setHeader("x-zalando-header-mode", "desktop");
eTag = StringUtils.chop(eTag);
eTag += "\\";
String jsonString = "{\"checkoutId\":\"" + checkoutID + "\"," +
"\"eTag\":" + "\"\\" + eTag + "\"" + "\"" + "}";
System.out.println(jsonString);
StringEntity entity = new StringEntity(jsonString);
postRequest.setEntity(entity);
long startTime = System.currentTimeMillis();
HttpResponse response = httpClient.execute(postRequest);
long elapsedTime = System.currentTimeMillis() - startTime;
System.out.println("Time taken : "+elapsedTime+"ms");
InputStream is = response.getEntity().getContent();
Reader reader = new InputStreamReader(is);
BufferedReader bufferedReader = new BufferedReader(reader);
StringBuilder builder = new StringBuilder();
while (true) {
try {
String line = bufferedReader.readLine();
if (line != null) {
builder.append(line);
} else {
break;
}
} catch (Exception e) {
e.printStackTrace();
}
}
System.out.println(builder.toString());
System.out.println("****************");
} catch (Exception ex) {
ex.printStackTrace();
}
}
This means that Akamai (the provider that zalando uses for bot protection), has detected and stopped your request because it detected you as a bot. To avoid this "stop" you MUST send a valid _abck cookie, generated by passing sensor data to the zalando akamai endpoint( you can find using chrome devtools and analyzing the requests )
I have a java requirement where I will send a message from my web server to user. I need to add any custom id in the message headers or session and when the receiver replies back i need to capture that id so that i will identify the message as a specific thread.
I tried to add the custom id in the message but when the receiver responds i am getting a null values always
I am using twilio web hook for receiving messages in java code.
Send SMS Code:
public boolean sendSMS(NotesDTO notes, HttpServletRequest request) {
int accountid = notes.getJobApplicationId();
String jobApplication = String.valueOf(accountid);
HttpSession session = request.getSession(true);
session.setAttribute("counter", notes.getCustomId());
request.setAttribute("counter", notes.getCustomId());
request.getSession().setAttribute("counter", notes.getCustomId());
Twilio.init(environment.getProperty("sendgrid.sms.account.sid"), environment.getProperty("sendgrid.sms.account.token"));
Message message = Message.creator(
new PhoneNumber("+1*********"),
new PhoneNumber("+1*********"),
"Good Morning!! \n"
+ "This is a message from ......\n"
+ "Thank You!!")
.create();
System.out.println("Request Message SID: " + message.getSid());
return true;
}
Receive Message Code:
#RequestMapping(value = "/getMessages", method = RequestMethod.POST)
public GenericResponse getTwilioMessages(HttpServletRequest request, HttpServletResponse response) throws IOException, ForbiddenException {
System.out.println(request.getParameter("Body"));
System.out.println(request.getReader());
String AccountSid = request.getParameter("AccountSid");
String From = request.getParameter("From");
String MessageSid = request.getParameter("MessageSid");
String MessageStatus = request.getParameter("MessageStatus");
String SmsSid = request.getParameter("SmsSid");
String SmsStatus = request.getParameter("SmsStatus");
String ApplicationSid = request.getParameter("ApplicationSid");
String Sid = request.getParameter("Sid");
String sid = request.getParameter("sid");
Object countObject = request.getAttribute("counter");
//String counter = countObject.toString();
System.out.println("AccountSid : " + AccountSid);
System.out.println("From : " + From);
System.out.println("MessageSid : " + MessageSid);
System.out.println("MessageStatus : " + MessageStatus);
System.out.println("SmsSid : " + SmsSid);
System.out.println("SmsStatus : " + SmsStatus);
System.out.println("Counter : " + countObject);
System.out.println("ApplicationSid : " + ApplicationSid);
System.out.println("Sid : " + Sid);
System.out.println("sid : " + sid);
BufferedReader br = new BufferedReader(request.getReader());
String s = null;
String Actualmessage = "";
while ((s=br.readLine())!=null)
{
Actualmessage = Actualmessage + s + "\n";
System.out.println(s);
}
System.out.println("Message : " + Actualmessage);
Map<String, String> map = new HashMap<String, String>();
Enumeration headerNames = request.getHeaderNames();
//Enumeration paramNames = request.getParameterNames();
while (headerNames.hasMoreElements()) {
String key = (String) headerNames.nextElement();
String value = request.getHeader(key);
System.out.println(key + ": " + value);
//map.put(key, value);
}
Body body = new Body.Builder("Hello World!").build();
Message message = new Message.Builder().body(body).build();
Redirect redirect = new Redirect
.Builder("https://demo.twilio.com/welcome/sms/").build();
MessagingResponse responseData = new MessagingResponse.Builder()
.message(message).redirect(redirect).build();
System.out.println("Response: " + responseData.toString());
Enumeration paramNames = request.getParameterNames();
while (paramNames.hasMoreElements()) {
String key = (String) paramNames.nextElement();
String value = request.getParameter(key);
System.out.println(key + ": " + value);
//map.put(key, value);
}
return new GenericResponse("success", null, null);
}```
Counter value is always coming as null
I would like to consume the twitter's API in Java but I am not able to get the authorization right. I can't use twitter4j because it doesn't include my request (Searching for places given a city name).
I already have a Twitter app developer created and I've tried a call from PostMan and it works perfectly, so the problem isn't with the lack of keys.
public void getTweets(City request, StreamObserver<Tweet> responseObserver){
System.out.print("Inside ipInfo service searching for city: " + request.getCity());
//Create a 32B nonce code in a private method
String nonce = generateNonce();
System.out.print("NONCE = " + nonce + "\n");
//Create signature
OAuthConsumer consumer = new OAuthConsumer(CONSUMER_KEY, CONSUMER_SECRET);
OAuthAccessToken accessToken = new OAuthAccessToken(ACCESS_TOKEN, ACCESS_TOKEN_KEY);
SignedRequest threeLeggedOAuthRequest = SignedRequestFactory.create(consumer, accessToken);
String Url = "https://api.twitter.com/1.1/geo/search.json";
HttpMethod method = HttpMethod.GET;
Timestamp ts = new Timestamp(System.currentTimeMillis());
long timestamp = ts.getTime();
String signature = threeLeggedOAuthRequest.getSignature(Url, method, nonce, timestamp);
System.out.print("Signature = " + signature);
// Api call
String placesUrl = String.format("https://api.twitter.com/1.1/geo/search.json?query=%s", request.getCity());
HttpResponse<JsonNode> apiResponse = null;
try {
apiResponse = Unirest.get(placesUrl)
.header( "Authorization",
"OAuth oauth_consumer_key=\""+CONSUMER_KEY+"\"," +
"oauth_token=\""+ACCESS_TOKEN+"\"," +
"oauth_signature_method=\"HMAC-SHA1\"," +
"oauth_timestamp=\""+timestamp+"\"," +
"oauth_nonce=\""+nonce+"\"," +
"oauth_version=\"1.0\"," +
"oauth_signature=\""+signature+"\""
)
.header( "Accept", "*/*" )
.header( "Host", "api.twitter.com" )
.header( "Connection", "Keep-Alive" )
.asJson();
} catch (UnirestException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.print("Api response = "+ apiResponse.getBody() + "\n");
}
private String generateNonce() {
Random gen = new Random(System.currentTimeMillis());
StringBuilder nonceBuilder = new StringBuilder("");
String base = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
int baseLength = base.length();
// Taking random word characters
for (int i = 0; i < 32; ++i) {
int position = gen.nextInt(baseLength);
nonceBuilder.append(base.charAt(position));
}
String nonce = Base64.getEncoder().encodeToString(nonceBuilder.toString().getBytes());
return nonce;
}
Result: Api response = {"errors":[{"code":215,"message":"Bad Authentication data."}]}
I want to set up a client with x connections, where each connection will send a new GET request to a different URL each random time.
So my approach is to create x threads, where each will send a GET request in a loop (till the program will terminate).
I was using Apache MultiThreadedHttpConnectionManager, where I set the number of possible connections as x.
ClientHandler(int numberOfClients) {
this.numberOfClients = numberOfClients;
connManager = new PoolingHttpClientConnectionManager();
HttpClientBuilder clientBuilder = HttpClients.custom().setConnectionManager(connManager);
connManager.setMaxTotal(numberOfClients);
CloseableHttpClient httpClient = clientBuilder.build();
generateMultiThreadedClient(httpClient);
}
private void generateMultiThreadedClient(CloseableHttpClient httpClient) {
for (int i = 0; i < numberOfClients; i++) {
String clientUrl = URL + i;
HttpGet httpGet = new HttpGet(clientUrl);
ClientMultiThreaded clientMultiThreaded = new ClientMultiThreaded(httpClient,httpGet, i);
LOGGER.info(CLIENT_LOG + "A new thread for clientId " + i + " was created.");
clientMultiThreaded.start();
LOGGER.info(CLIENT_LOG + "Thread for clientId " + i + " started..");
}
}
And this is my run() method of the thread.
ClientMultiThreaded(CloseableHttpClient httpClient, HttpGet httpGet, int clientId) {
this.httpClient = httpClient;
this.clientId = clientId;
this.httpGet = httpGet;
randomGenerator = new Random();
}
#Override
public void run() {
try{
// Execute request for the first time
executeRequest(httpGet);
while (true) {
int timeToSleep = randomGenerator.nextInt(BOUND_LIMIT) + 1;
LOGGER.info(CLIENT_LOG + "Thread id " + clientId + " went to sleep for " + timeToSleep / 1000 + " seconds");
sleep(timeToSleep);
LOGGER.info(" ------- This is a test log printing for clientId: " +clientId);
executeRequest(httpGet);
}
}catch(Exception e) {
LOGGER.info(e.getMessage());
}
}
private void executeRequest(HttpGet httpGet) throws IOException {
CloseableHttpResponse response = httpClient.execute(httpGet);
LOGGER.info(CLIENT_LOG + "clientId " + clientId + " sent get request to URL " + httpGet.getURI());
statusCode = this.response.getStatusLine().getStatusCode();
LOGGER.info(CLIENT_LOG + "Status received for clientId " + clientId + " is: " + statusCode);
}
The problem is that I actually get 2 GET requests sent and then it stops.
You need to close response, use try with resources
Use following code:
private void executeRequest(HttpGet httpGet) throws IOException {
try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
LOGGER.info(CLIENT_LOG + "clientId " + clientId + " sent get request to URL " + httpGet.getURI());
statusCode = this.response.getStatusLine().getStatusCode();
LOGGER.info(CLIENT_LOG + "Status received for clientId " + clientId + " is: " + statusCode);
}
}
I have two nearly identical methods in Java. The only difference is that they have different parameter types. They use generics and return the type T of the input parameter. How can I get rid of the duplicate code? Here are my two methods. Ultimately they both call Spring restTemplate.exchange() with the different types. Otherwise the methods are identical.
public <T> T send(Class<T> expectedClass) throws KenectRestClientException {
if (this.logRequest) log.info("Making request: " + this.toString());
HttpEntity<Object> entity = new HttpEntity<>(this.body, getTokenHeaders(this.token));
RestTemplate restTemplate = new RestTemplate();
String compiledUrl = UrlUtils.replacePathParamsInUrl(this.url, this.pathParams, this.uriEncode) + UrlUtils.compileRequestParamsToUrl(this.requestParams, this.uriEncode);
Object[] incompatibleStrings = extractIncompatibleStrings(compiledUrl);
try {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, expectedClass, incompatibleStrings).getBody();
} catch(RestClientResponseException e) {
log.warning(RestClientResponseException.class.getSimpleName() +
" url: " + url +
" status code: " + e.getRawStatusCode() +
" error body: " + e.getResponseBodyAsString() +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
} catch(ResourceAccessException e) {
log.warning("Resource access exception " +
" url: " + url +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
}
}
public <T> T send(ParameterizedTypeReference<T> parameterizedTypeReference) throws KenectRestClientException {
if (this.logRequest) log.info("Making request: " + this.toString());
HttpEntity<Object> entity = new HttpEntity<>(this.body, getTokenHeaders(this.token));
RestTemplate restTemplate = new RestTemplate();
String compiledUrl = UrlUtils.replacePathParamsInUrl(this.url, this.pathParams, this.uriEncode) + UrlUtils.compileRequestParamsToUrl(this.requestParams, this.uriEncode);
Object[] incompatibleStrings = extractIncompatibleStrings(compiledUrl);
try {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, parameterizedTypeReference, incompatibleStrings).getBody();
} catch(RestClientResponseException e) {
log.warning(RestClientResponseException.class.getSimpleName() +
" url: " + url +
" status code: " + e.getRawStatusCode() +
" error body: " + e.getResponseBodyAsString() +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
} catch(ResourceAccessException e) {
log.warning("Resource access exception " +
" url: " + url +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
}
}
What about passing a boolean flag, based on which you call exchange() passing either a Class<T> or a ParameterizedTypeReference<T>?
public <T> T send(T neededType, boolean flag) throws KenectRestClientException {
if (this.logRequest) log.info("Making request: " + this.toString());
HttpEntity<Object> entity = new HttpEntity<>(this.body, getTokenHeaders(this.token));
RestTemplate restTemplate = new RestTemplate();
String compiledUrl = UrlUtils.replacePathParamsInUrl(this.url, this.pathParams, this.uriEncode) + UrlUtils.compileRequestParamsToUrl(this.requestParams, this.uriEncode);
Object[] incompatibleStrings = extractIncompatibleStrings(compiledUrl);
try {
if (flag) {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, neededType.getClass(), incompatibleStrings).getBody();
} else {
return restTemplate.exchange(compiledUrl, this.httpMethod, entity, ParameterizedTypeReference.forType(neededType.getClass()), incompatibleStrings).getBody();
}
} catch(RestClientResponseException e) {
log.warning(RestClientResponseException.class.getSimpleName() +
" url: " + url +
" status code: " + e.getRawStatusCode() +
" error body: " + e.getResponseBodyAsString() +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
} catch(ResourceAccessException e) {
log.warning("Resource access exception " +
" url: " + url +
" stacktrace: " + ExceptionUtils.getStackTrace(e));
throw new KenectRestClientException("Failed to send request");
}
}