We Keep Coding

Prevent DOS attack with multithreaded code

Let me give an overview of my question here:
There are ~ 4000 servers and millions of URLs per server. My code needs to hit each URL and write the response code to HDFS filesystem along with the URL.
Added some part here too :
Check the number of requests sent to a webpage
I am using a producer-consumer model here with 400 threads. The code has caused a DOS attack to few web servers very recently, I am having a hard time figuring out where the issue here is :
Main class:
public void readURLS(final Path inputPath, final Path outputPath) {
LOG.info("Looking for files to download, queue size: {}, DOWNLOAD_THREADS: {}", queueSize, producerThreads);
final List<Path> files = HdfsUtils.listDirectory(inputPath, hadoopConf);
final BlockingQueue<String> queue = new LinkedBlockingQueue<>(queueSize);
final UrlConsumerWriter consumerWriter =
new UrlConsumerWriter(queue, outputPath, hadoopConf);
LOG.info("Starting download of {} files from: '{}'", files.size(), inputPath);
final ExecutorService writerPool = DownloadUtils.createWriterPool();
CompletableFuture<Void> producer = downloadFilesToQueue(files, queue)
.thenRun(consumerWriter::notifyProducersDone);
CompletableFuture<Void> consumer =
CompletableFuture.runAsync(consumerWriter, writerPool)// Cancel download workers if write worker fails
.whenComplete((result, err) -> {
if (err != null) {
LOG.error("Consumer Write worker failed!", err);
producer.cancel(true);
}
});
writerPool.shutdown();
producer.join();
consumer.join();
LOG.info("Url Validation Job Complete!!!");
}
private CompletableFuture<Void> downloadFilesToQueue(
final List<Path> files,
final BlockingQueue<String> downloadQueue
) {
final ExecutorService pool = DownloadUtils.createDownloadPool(producerThreads);
final List<CompletableFuture<Void>> workers = files.stream()
.map(file -> new UrlDownloadWorker(clock, file, hadoopConf, downloadQueue,
utils, (validatorImpl.emptyTable())))
.map(worker -> CompletableFuture.runAsync(worker, pool))
.collect(Collectors.toList());
pool.shutdown();
final CompletableFuture<Void> allDownloads = CompletableFuture.allOf(workers.toArray(new CompletableFuture[0]));
// When one worker fails, cancel all the other immediately
for (final CompletableFuture<Void> worker : workers) {
worker.whenComplete((v, err) -> {
if (err != null) {
LOG.error("Download worker failed!", err);
allDownloads.cancel(true);
}
});
}
return allDownloads;
}
PRODUCER CLASS:
#Override
public void run() {
LOG.info("Starting download worker for file: '{}'", file);
long numLines = 0;
try (BufferedReader reader = new BufferedReader(new InputStreamReader(
file.getFileSystem(hadoopConf).open(file), CHARSET))) {
String line;
while ((line = reader.readLine()) != null) {
// LOG.info("Thread {} Reading file: '{}'",Thread.currentThread().getName(), file);
if (Thread.interrupted()) {
throw new InterruptedException();
}
StringBuilder builder = new StringBuilder();
//write into database
final StatusCode statusCode = utils.validateURL(line);
if (statusCode != null) {
queue.put(builder.append(line)
.append(",")
.append(statusCode.name()).toString());
builder.setLength(0);
} else {
throw new UrlValidationException(
"Failed to validate url :'" + line + "'");
}
numLines++;
}
} catch (IOException e) {
throw new DownloadException(file, e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new DownloadException("Interrupted while downloading", file, e);
}
LOG.info("Download of {} lines complete for file: '{}'", numLines, file);
}
UrlValidationUtils CLASS:
public final class UrlValidationUtils {
private static final String WEBSITENOTCHECK = "uncheck.org";
private final Map<String, StatusCode> blockedHosts = new ConcurrentHashMap<>();
private static final int MAX_REDIRECT = 4;
public StatusCode validateURL(String url) throws IOException {
return validate(url, MAX_REDIRECT);
}
private StatusCode validate(String url, int maxRedirect) throws IOException {
URL urlValue = new URL(url);
HttpURLConnection con;
if (url.contains(WEBSITENOTCHECK)) {
blockedHosts.put(urlValue.getHost(), StatusCode.SUCCESS);
}
//first check if the host is already marked as invalid
// if (blockedHosts.containsKey(urlValue.getHost())) {
// return blockedHosts.get(urlValue.getHost());
// }
StatusCode statusCode;
con = (HttpURLConnection) urlValue.openConnection();
try {
int resCode;
con.setInstanceFollowRedirects(false);
con.setConnectTimeout(3000); //set timeout to 3 seconds
con.connect();
resCode = con.getResponseCode();
LOG.info("thread name {} connection id {} url {} ", Thread.currentThread().getName(), con.toString(), url);
if (resCode == HttpURLConnection.HTTP_OK) {
statusCode = StatusCode.SUCCESS;
} else if (resCode == HttpURLConnection.HTTP_SEE_OTHER || resCode == HttpURLConnection.HTTP_MOVED_PERM
|| resCode == HttpURLConnection.HTTP_MOVED_TEMP) {
String location = con.getHeaderField("Location");
if (location.startsWith("/")) {
location = urlValue.getProtocol() + "://" + urlValue.getHost() + location;
}
statusCode = validateRedirect(location, maxRedirect - 1, con);
} else {
blockedHosts.put(urlValue.getHost(), StatusCode.FAIL);
statusCode = StatusCode.FAIL;
}
} catch (UnknownHostException e) {
blockedHosts.put(urlValue.getHost(), StatusCode.UNKOWNHOST);
statusCode = StatusCode.UNKOWNHOST;
} catch (ConnectException e) {
blockedHosts.put(urlValue.getHost(), StatusCode.CONNECTION_ISSUE);
statusCode = StatusCode.CONNECTION_ISSUE;
} catch (IOException e) {
//if an IOException is caught possible reason is SOCKETTIMEOUT
blockedHosts.put(urlValue.getHost(), StatusCode.SOCKETTIMEOUT);
statusCode = StatusCode.SOCKETTIMEOUT;
}
con.disconnect();
LOG.info("thread name {} connection id {} url {} ", Thread.currentThread().getName(), con.toString(), url);
return statusCode;
}
private StatusCode validateRedirect(String location, int redirectCount, HttpURLConnection connection)
throws IOException {
if (redirectCount >= 0) {
connection.disconnect();
return validate(location, redirectCount);
}
return StatusCode.FAIL;
}
}

In order to not overload the servers, I suggest waiting a few milliseconds before hitting a batch of urls. For example, after hitting N urls, you could wait 20 ms, then hit the next N.. and so on. The size of the batch (N) would depend on how many requests you know your server can handle in a second. Do you have a Service Level Agreement for them in terms of performance?

Can't use google to log users into dropbox

I am building a app that allows interaction with Dropbox. I have successfully authenticated to, and Dropbox is returning a key and template using it's default login page. However, when I try to use the login with Google option for Dropbox upon authenticating Dropbox returns a user key, however it doesn't return a template id, but instead a too many templates error.
Any help would be appreciated.
Here is my code
public void authUrl(HttpServletRequest request)
{
DbxWebAuth webAuth = new DbxWebAuth(requestConfig, appInfo);
HttpSession session = request.getSession(true);
String sessionKey = "dropbox-auth-csrf-token";
DbxSessionStore csrfTokenStore = new DbxStandardSessionStore(session, sessionKey);
DbxWebAuth.Request authRequest = DbxWebAuth.newRequestBuilder()
.withRedirectUri(redirectUri, csrfTokenStore)
.build();
String authorizeUrl = webAuth.authorize(authRequest);
url =authorizeUrl;
}
public String getTemplate()
{
HttpClient httpclient = new DefaultHttpClient();
HttpPost post = new HttpPost("https://api.dropboxapi.com/2/file_properties/templates/add_for_user");
post.addHeader("Authorization","Bearer "+code);
post.addHeader("Content-Type", "application/json");
try {
HttpEntity entity = new StringEntity(json);
post.setEntity(entity);
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
HttpResponse response = httpclient.execute(post);
HttpEntity entity = response.getEntity();
System.out.println(response.getStatusLine().getStatusCode());
if (entity != null) {
InputStream instream = entity.getContent();
try {
Scanner s = new Scanner(instream).useDelimiter("\\A");
String result = s.hasNext() ? s.next() : "";
System.out.println(result);
System.err.println(result);
return result;
} finally {
instream.close();
}
}
}catch(Exception e)
{
e.printStackTrace();
}
return null;
}
public String authcode(HttpServletRequest request)
{
DbxWebAuth auth = new DbxWebAuth(requestConfig, appInfo);
String sessionKey = "dropbox-auth-csrf-token";
HttpSession session = request.getSession(true);
DbxSessionStore csrfTokenStore = new DbxStandardSessionStore(session, sessionKey);
DbxAuthFinish authFinish;
try {
authFinish = auth.finishFromRedirect(redirectUri, csrfTokenStore, request.getParameterMap());
} catch (DbxWebAuth.BadRequestException ex) {
System.out.println("On /dropbox-auth-finish: Bad request: " + ex.getMessage());
return "failed";
} catch (DbxWebAuth.BadStateException ex) {
System.out.println(ex.getMessage());
// Send them back to the start of the auth flow.
return "failed";
} catch (DbxWebAuth.CsrfException ex) {
System.out.println("On /dropbox-auth-finish: CSRF mismatch: " + ex.getMessage());
//response.sendError(403, "Forbidden.");
return "failed";
} catch (DbxWebAuth.NotApprovedException ex) {
// When Dropbox asked "Do you want to allow this app to access your
// Dropbox account?", the user clicked "No".
//...
return "failed";
} catch (DbxWebAuth.ProviderException ex) {
System.out.println("On /dropbox-auth-finish: Auth failed: " + ex.getMessage());
//response.sendError(503, "Error communicating with Dropbox.");
return "failed";
} catch (DbxException ex) {
System.out.println("On /dropbox-auth-finish: Error getting token: " + ex.getMessage());
//response.sendError(503, "Error communicating with Dropbox.");
return "failed";
}
String accessToken = authFinish.getAccessToken();
code =accessToken;
return accessToken;
}
Notes
I am using a java, running on a google app engine server to complete this authentication.
The exact error text in the logs is
{"error_summary": "too_many_templates/...", "error": {".tag":
"too_many_templates"}}
Along with a error 409.
If you need any more information feel free to ask

Volley JSONObjectRequest Request.Method.POST but received GET

I made a Custom JSONObjectRequest as below code.
Currently i made the Request.Method.POST and i had confirmed it while passing to the super class here
super(method, url, (jsonRequest == null) ? null : jsonRequest.toString(), listener, errorListener);. It was received as int 1 (POST), but when received on my server (PHP), the log was receiving as "REQUEST_METHOD":"GET"
Have anyone met this kind of problem or can help to point out what i had missed in below code.
As you can see below, i had make sure below things:
params not empty
there is function getParams() there and the params are not empty
i even put getBody() and make sure it not returning null
Refer below code for what i had tried to do
try {
JsonObjectRequest stringRequest = new JsonObjectRequest(Request.Method.POST, url, params, new Response.Listener<JSONObject>() {
#Override
public void onResponse(JSONObject jsonObject) {
if (jsonObject != null && jsonObject.length() > 0) {
try {
if (jsonObject.getInt("status") == ResponseCode.LOGIN_FAILED) {
SharedManager sharedManager = new SharedManager();
sharedManager.logoutUser(context, true);
return;
}
} catch (JSONException e) {
e.printStackTrace();
}
}
callback.onSuccessResponse(jsonObject);
}
}, new Response.ErrorListener() {
#Override
public void onErrorResponse(VolleyError error) {
callback.onErrorResponse(error);
}
}) {
/**
* Passing some request headers
* */
#Override
public Map<String, String> getHeaders() throws AuthFailureError {
return headers;
}
#Override
protected Map<String, String> getParams() throws AuthFailureError {
try {
return JSONHelperConverter.jsonToMapString(params);
} catch (JSONException e) {
e.printStackTrace();
}
return null;
}
#Override
public String getBodyContentType() {
return "application/json; charset=utf-8;";
}
#Override
public byte[] getBody() {
try {
String postBody = null;
try {
postBody = createPostBody(JSONHelperConverter.toMapString(params));
postBody = params.toString();
} catch (JSONException e) {
e.printStackTrace();
}
return postBody == null ? null : postBody.getBytes("utf-8");
} catch (NegativeArraySizeException n) {
n.printStackTrace();
return null;
} catch (UnsupportedEncodingException uee) {
uee.printStackTrace();
return null;
}
}
#Override
protected Response<JSONObject> parseNetworkResponse(NetworkResponse response) {
String responseString;
JSONObject jsonObject = null;
if (response != null) {
try {
responseString = new String(response.data, HttpHeaderParser.parseCharset(response.headers));
jsonObject = new JSONObject(responseString);
} catch (Exception e) {
e.printStackTrace();
}
}
return Response.success(jsonObject, HttpHeaderParser.parseCacheHeaders(response));
}
};
NetworkSingleton.getInstance(context).addToRequestQueue(stringRequest);
} catch (NegativeArraySizeException n) {
n.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}

I solved my own problem
Sent POST request but server says GET request in Android volley, what I am doing wrong here?
Solution:
Due to i'm requesting to PHP server, i need to point out the filename.
Currently i point the request to a URL without Filename extension, and i just add index.php to my end of URL and it worked.

BasicNetwork.performRequest: Unexpected response code 400 for https://api.infermedica.com/v2/diagnosis - Android Java

I'm testing the inter media API and I am trying it using this code below. I used volley for this:
String url = "https://api.infermedica.com/v2/diagnosis";
JSONArray evidence = new JSONArray();
JSONObject evidence1 = new JSONObject();
try {
evidence1.put("id", "s_1193");
evidence1.put("choice_id", "present");
} catch (JSONException e) {
e.printStackTrace();
}
try {
evidence.put(0,evidence1);
} catch (JSONException e) {
e.printStackTrace();
}
JSONObject postparams = new JSONObject();
try {
postparams.put("sex", "male");
postparams.put("age", 30);
postparams.put("evidence", evidence);
} catch (JSONException e) {
e.printStackTrace();
}
Log.e("DATA:", postparams.toString());
JsonObjectRequest jsonObjReq = new JsonObjectRequest(Request.Method.POST,
url, postparams,
new Response.Listener() {
#Override
public void onResponse(Object response) {
Log.e("DATA: ", response.toString());
}
},
new Response.ErrorListener() {
#Override
public void onErrorResponse(VolleyError error) {
// As of f605da3 the following should work
NetworkResponse response = error.networkResponse;
if (error instanceof ServerError && response != null) {
try {
String res = new String(response.data,
HttpHeaderParser.parseCharset(response.headers, "utf-8"));
// Now you can use any deserializer to make sense of data
JSONObject obj = new JSONObject(res);
Log.e("DATA:" , obj.toString());
} catch (UnsupportedEncodingException e1) {
// Couldn't properly decode data to string
e1.printStackTrace();
} catch (JSONException e2) {
// returned data is not JSONObject?
e2.printStackTrace();
}
}
}
}
) {
#Override
public Map<String, String> getHeaders() throws AuthFailureError {
HashMap<String, String> headers = new HashMap<String, String>();
headers.put("app-id", "MY-APP-ID");
headers.put("app-key", "MY-APP-KEY");
headers.put("authorization", "Basic Og==");
headers.put("Content-type", "application/json");
return headers;
}
};
Log.e("DATA: " , "calling volley");
RequestQueue requestQueue = Volley.newRequestQueue(this);
requestQueue.add(jsonObjReq);
After running it and checking the logs here is the result:
E/DATA:: {"sex":"male","age":30,"evidence":
[{"id":"s_1193","choice_id":"present"}]}
E/DATA:: calling volley
E/Volley: [236] BasicNetwork.performRequest: Unexpected response code 400
for https://api.infermedica.com/v2/diagnosis
E/DATA:: {"message":"bad request"}
I tried the logged json in Postman and it was accepted:
Here is the screenshot of my postman result
I've been looking for answers around here, I already added the content-type, and I made sure that my keys are correct. Am I using the volley correctly. Your help will be much appreciated.

Upload Large Video File From Android

I can not upload large video to server android?
i try to use this method
private void UploadLargeFile(String[] args) throws Exception {
CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault();
try {
httpclient.start();
File upload = new File(args[0]);
File download = new File(args[1]);
ZeroCopyPost httpost = null;
try {
httpost = new ZeroCopyPost(URLTOUploadFile", upload,
ContentType.create("video/mp4"));
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
ZeroCopyConsumer<File> consumer = null;
consumer = new ZeroCopyConsumer<File>(download) {
#Override
protected File process(final HttpResponse response,
final File file, final ContentType contentType)
throws Exception {
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
throw new ClientProtocolException("Upload failed: "
+ response.getStatusLine());
}
return file;
}
};
Future<File> future = httpclient.execute(httpost, consumer, null);
File result;
result = future.get();
System.out.println("Response file length: " + result.length());
System.out.println("Shutting down");
} finally {
httpclient.close();
}
System.out.println("Done");
}
in the first line of CloseableHttpAsyncClient i got NoSuchFieldFound error
can any one help me?