We Keep Coding

java.lang.arrayindexoutofboundsexception jsoup

I'm trying to pull all images from a website and
analyze each one using AWS image recognition API. It works for some websites, however some websites return an error saying `500 server error java.lang.arrayindexoutofboundsexception index:281 size 281.
Basically I'm scraping images using jsoup and then creating an object to store the name and image URL for each image. After that, I call the API and check each image in the ArrayList. For some reason it only works for some websites.
Can someone please explain what I'm doing wrong and how to prevent this error?
#WebServlet(name = "HelloAppEngine", urlPatterns = {
"/hello"
})
public class HelloAppEngine extends HttpServlet {
static ArrayList < ResponseData > testImages = new ArrayList < > ();
static AmazonRekognition rekognitionClient = AmazonRekognitionClientBuilder.defaultClient();
public static void getimages() throws MalformedURLException, IOException {
System.out.println("getImages called" + testImages);
int index = 0;
for (ResponseData data: testImages) {
System.err.println("open stream for:" + data.getUrl());
ByteBuffer imageBytes = null;
try (InputStream inputStream = new URL(data.getUrl()).openStream()) {
System.out.println(inputStream);
imageBytes = ByteBuffer.wrap(IOUtils.toByteArray(inputStream));
System.out.println(imageBytes);
} catch (IOException e1) {
System.err.println(e1.getMessage());
}
//
DetectLabelsRequest request = new DetectLabelsRequest().withImage(new Image().withBytes(imageBytes)); //.withMaxLabels(10).withMinConfidence(77F);
try {
DetectLabelsResult result = rekognitionClient.detectLabels(request);
List < Label > labels = result.getLabels();
//System.out.println(labels);
//System.out.println("Detected labels for " + photo+""+labels);
for (Label label: labels) {
//loop through all labels of object
//create new responsedata object for each image
//where im getting error
if (testImages.get(index) != null) {
ResponseData d = testImages.get(index);
d.setName(label.getName());
testImages.set(index, d);
//increment for making new image url and name
index++;
System.out.println(label.getName() + ": " + label.getConfidence().toString());
}
}
//
} catch (AmazonRekognitionException e) {
System.err.println(e.getMessage());
}
}
}
private static final long serialVersionUID = 1 L;
protected static final Gson GSON = new GsonBuilder().create();
// This is just a test array
ArrayList < String > list = new ArrayList < String > ();
#Override
protected final void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
resp.setContentType("text/json");
String servlet = req.getServletPath();
System.setProperty("http.proxyHost", "192.168.5.1");
System.setProperty("http.proxyPort", "1080");
log("servlet:" + servlet);
if (servlet.equalsIgnoreCase("/main")) {
log("if body start");
String urlString = java.net.URLDecoder.decode(req.getParameter("url"), "UTF-8");
// Connect to website. This can be replaced with your file loading
// implementation
Document doc = Jsoup.connect(urlString).get();
// Get all img tags
Elements img = doc.getElementsByTag("img");
Elements media = doc.select("[src]");
int counter = 0;
// Loop through img tags
for (Element src: media) {
if (src.tagName().equals("img")) {
counter++;
//create reposnsedata object for each image url
ResponseData data = new ResponseData();
//set object url to image url
data.setUrl(src.attr("abs:src"));
//set data name from aws
data.setName(" ");
testImages.add(data);
// getimages();
}
if (src.tagName().equals("link[href~=.*\\.(ico|png)]")) {
System.out.println("image is logo");
}
if (src.tagName().equals("meta[itemprop=image]")) {
System.out.println("image is logosss");
}
}
}
//log("list" + testImages);
getimages();
//
// getimages();
System.err.println(GSON.toJson(testImages));
resp.getWriter().println(GSON.toJson(testImages));
}
#Override
protected final void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
doPost(req, resp);
}
}

You're trying to get 282nd image (index=281) from testImages but there's only 281 (index=280). You're getting each image for each label and it's possible there's more labels than images.
Try displaying the amount of both of them:
System.out.println("testImages.size() is: " + testImages.size());
System.out.println("labels.size() is: " + labels.size());
To avoid getting more images than labels try replacing this condition:
if (testImages.get(index) != null) {
with
if (index < testImages.size() && testImages.get(index) != null) {

SignatureDoesNotMatch error when deleting multiple objects from S3 bucket

I m adding code to delete multiple objects from S3. POST request to delete returns 403 status. I am adding this code in legacy code and unfortunately cannot change to change AWS S3 library. Not able to figure out if I am missing a header or if the logic is incorrect.
public Response deleteMultipleObjects(Delete object) throws MalformedURLException, IOException, StorageException {
HttpURLConnection request = null;
try(ByteArrayOutputStream out = new ByteArrayOutputStream();
ObjectOutputStream os = new ObjectOutputStream(out)) {
os.writeObject(object);
Calendar now = Calendar.getInstance();
Map<String, List<String>> headers = new HashMap<>();
headers.put(AWSAuthConnection.CONTENT_TYPE_HEADER, Arrays.asList("application/octet-stream"));
headers.put(AWSAuthConnection.CONTENT_LENGTH_HEADER, Arrays.asList(Integer.toString(out.toByteArray().length)));
headers.put(AWSAuthConnection.AMAZON_DATE_HEADER, Arrays.asList(convertDateToString(now, AWSAuthConnection.TIMESTAMP_FORMAT, AWSAuthConnection.TIME_ZONE_UTC)));
headers.put(AWSAuthConnection.AMAZON_CONTENT_SHA256_HEADER, Arrays.asList(buildHash(out.toByteArray(), SHA256_HASH, EncodingType.HEX)));
headers.put(AWSAuthConnection.CONTENT_MD5_HEADER, Arrays.asList(buildHash(out.toByteArray(), AWSAuthConnection.MD5_HASH, AWSAuthConnection.EncodingType.BASE64)));
request = makeRequest("POST", bucket+"/?delete", headers, null);
request.setDoOutput(true);
request.getOutputStream().write(object == null ? new byte[]{} : out.toByteArray());
return new Response(request);
}
}
private String buildHash(final byte[] contentToHash, final String hashMethod, final AWSAuthConnection.EncodingType encodingType) throws StorageException{
try {
MessageDigest digest = MessageDigest.getInstance(hashMethod);
byte[] contentHash = digest.digest(contentToHash);
if (encodingType == AWSAuthConnection.EncodingType.HEX) {
return Hex.encodeHexString(contentHash);
} else {
return Base64.encodeBase64String(contentHash);
}
} catch (NoSuchAlgorithmException e) {
throw new StorageException(StorageException.ERROR_INVALID_STORAGE_TYPE, e);
}
}
private HttpURLConnection makeRequest(String method, String resource, Map<String, List<String>> headers, StorageObject object) throws MalformedURLException, IOException {
URL url = makeURL(resource);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod(method);
connection.setUseCaches(true);
addHeaders(connection, headers);
if (object != null) addMetadataHeaders(connection, object.metadata);
addAuthHeader(connection, method, resource);
return connection;
}
private void addAuthHeader(HttpURLConnection connection, String method, String resource) {
String canonicalString =
Utils.makeCanonicalString(method, resource, connection.getRequestProperties());
String encodedCanonical = Utils.encode(this.info, canonicalString, false);
connection.setRequestProperty("Authorization", "AWS " + this.info + ":" + encodedCanonical);
}
Error from S3:
<?xml version="1.0" encoding="UTF-8"?>
<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message>.......</Error>
Has anyone faced this issue before? If you know a way to fix, please help

How to cache traffic with littleproxy/netty?

I want to cache static files with littleproxy. So I created HashMap with key for uri and values for response body. Here's java code:
private static Map<String, FullHttpResponse> cache = new HashMap<>();
private static HttpFiltersSource getHttpFiltersSource() {
return new HttpFiltersSourceAdapter() {
#Override
public int getMaximumResponseBufferSizeInBytes() {
return 10 * 1024 * 1024;
}
public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
return new HttpFiltersAdapter(originalRequest) {
#Override
public HttpResponse clientToProxyRequest(HttpObject httpObject) {
if (httpObject instanceof HttpRequest) {
HttpRequest request = (HttpRequest) httpObject;
String requestUri = request.getUri();
if(requestUri.matches(".*[./]png.*$") ||
requestUri.matches(".*[./]jpg.*$") ||
requestUri.matches(".*[./]jpeg.*$") ||
requestUri.matches(".*[./]woff2.*$") ||
requestUri.matches(".*[./]js.*$") ) {
if (cache.containsKey(requestUri)) {
System.out.println("GOT FROM CACHE " + requestUri);
return cache.get(requestUri);
}
}
}
return null;
}
#Override
public HttpObject serverToProxyResponse(HttpObject httpObject) {
if (httpObject instanceof FullHttpResponse) {
FullHttpResponse response = (FullHttpResponse) httpObject;
String requestUri = originalRequest.getUri();
if(requestUri.matches(".*[./]png.*$") ||
requestUri.matches(".*[./]jpg.*$") ||
requestUri.matches(".*[./]jpeg.*$") ||
requestUri.matches(".*[./]woff2.*$") ||
requestUri.matches(".*[./]js.*$") ) {
cache.put(requestUri, response.retain());
System.out.println("ADDED TO CACHE " + requestUri);
}
}
return httpObject;
}
};
}
};
}
But something wrong here with response in Map. When the browser reaches static files firstly there's a debug message in console: ADDED TO CACHE. When the browser reaches static files secondly there's a message: "GOT FROM CACHE", but browser spins forever waiting for a response.
What's the right way to save and store responses from server and return it to client when time comes?

I think you need to also duplicate the FullHttpResponse to ensure you have correct writer / reader indices.
cache.put(requestUri, response.duplicate().retain());
and:
return cache.get(requestUri).duplicate();
Also ensure that you call release() once you remove something from the cache.

How to send bluemix wav stream response from java

I'd like to invoke IBM Bluemix service (say Text to Speech) from my Java code.
I've managed to get service credentials and URL but how can I invoke it after?
I've seen some example where people have used similar to below code but wondering how it works for a Text to Speech where it outputs a wav stream.
String profileString = ex.execute(profileRequest)
.handleResponse(new ResponseHandler<String>() {
#Override
public String handleResponse(HttpResponse r)
throws ClientProtocolException, IOException {
}
}
Can any one suggest on priority please?

The link below has a Java code example of how to use the Watson text-to-speech service.
https://github.com/watson-developer-cloud/text-to-speech-java
You should be looking for something like this from the DemoServlet.java class:
#Override
protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) throws ServletException, IOException {
if (req.getParameter("text") == null || req.getParameter("voice") == null) {
req.getRequestDispatcher("/index.jsp").forward(req, resp);
} else {
boolean download = false;
if (req.getParameter("download") != null && req.getParameter("download").equalsIgnoreCase("true")) {
download = true;
}
req.setCharacterEncoding("UTF-8");
try {
String queryStr = req.getQueryString();
String url = baseURL + "/v1/synthesize";
if (queryStr != null) {
url += "?" + queryStr;
}
URI uri = new URI(url).normalize();
Request newReq = Request.Get(uri);
newReq.addHeader("Accept", "audio/ogg; codecs=opus");
Executor executor = Executor.newInstance().auth(username, password);
Response response = executor.execute(newReq);
if (download)
{
resp.setHeader("content-disposition", "attachment; filename=transcript.ogg");
}
ServletOutputStream servletOutputStream = resp.getOutputStream();
response.returnResponse().getEntity()
.writeTo(servletOutputStream);
servletOutputStream.flush();
servletOutputStream.close();
} catch (Exception e) {
// Log something and return an error message
logger.log(Level.SEVERE, "got error: " + e.getMessage(), e);
resp.setStatus(HttpStatus.SC_BAD_GATEWAY);
}
}
}
Finally, the link below has instructions on how to create a Java war file and deploy to Bluemix:
https://www.ibm.com/smarterplanet/us/en/ibmwatson/developercloud/doc/getting_started/gs-full-java.shtml

Get the POST request body from HttpServletRequest

I am trying to get the whole body from the HttpServletRequest object.
The code I am following looks like this:
if ( request.getMethod().equals("POST") )
{
StringBuffer sb = new StringBuffer();
BufferedReader bufferedReader = null;
String content = "";
try {
//InputStream inputStream = request.getInputStream();
//inputStream.available();
//if (inputStream != null) {
bufferedReader = request.getReader() ; //new BufferedReader(new InputStreamReader(inputStream));
char[] charBuffer = new char[128];
int bytesRead;
while ( (bytesRead = bufferedReader.read(charBuffer)) != -1 ) {
sb.append(charBuffer, 0, bytesRead);
}
//} else {
// sb.append("");
//}
} catch (IOException ex) {
throw ex;
} finally {
if (bufferedReader != null) {
try {
bufferedReader.close();
} catch (IOException ex) {
throw ex;
}
}
}
test = sb.toString();
}
and I am testing the functionality with curl and wget as follows:
curl --header "MD5: abcd" -F "fileupload=#filename.txt http://localhost:8080/abcd.html"
wget --header="MD5: abcd" --post-data='{"imei":"351553012623446","hni":"310150","wdp":false}' http://localhost:8080/abcd.html"
But the while ( (bytesRead = bufferedReader.read(charBuffer)) != -1 ) does not return anything, and so I get nothing appended on StringBuffer.

In Java 8, you can do it in a simpler and clean way :
if ("POST".equalsIgnoreCase(request.getMethod()))
{
test = request.getReader().lines().collect(Collectors.joining(System.lineSeparator()));
}

Easy way with commons-io.
IOUtils.toString(request.getReader());
https://commons.apache.org/proper/commons-io/javadocs/api-2.5/org/apache/commons/io/IOUtils.html

Be aware, that your code is quite noisy.
I know the thread is old, but a lot of people will read it anyway.
You could do the same thing using the guava library with:
if ("POST".equalsIgnoreCase(request.getMethod())) {
test = CharStreams.toString(request.getReader());
}

If all you want is the POST request body, you could use a method like this:
static String extractPostRequestBody(HttpServletRequest request) throws IOException {
if ("POST".equalsIgnoreCase(request.getMethod())) {
Scanner s = new Scanner(request.getInputStream(), "UTF-8").useDelimiter("\\A");
return s.hasNext() ? s.next() : "";
}
return "";
}
Credit to: https://stackoverflow.com/a/5445161/1389219

This works for both GET and POST:
#Context
private HttpServletRequest httpRequest;
private void printRequest(HttpServletRequest httpRequest) {
System.out.println(" \n\n Headers");
Enumeration headerNames = httpRequest.getHeaderNames();
while(headerNames.hasMoreElements()) {
String headerName = (String)headerNames.nextElement();
System.out.println(headerName + " = " + httpRequest.getHeader(headerName));
}
System.out.println("\n\nParameters");
Enumeration params = httpRequest.getParameterNames();
while(params.hasMoreElements()){
String paramName = (String)params.nextElement();
System.out.println(paramName + " = " + httpRequest.getParameter(paramName));
}
System.out.println("\n\n Row data");
System.out.println(extractPostRequestBody(httpRequest));
}
static String extractPostRequestBody(HttpServletRequest request) {
if ("POST".equalsIgnoreCase(request.getMethod())) {
Scanner s = null;
try {
s = new Scanner(request.getInputStream(), "UTF-8").useDelimiter("\\A");
} catch (IOException e) {
e.printStackTrace();
}
return s.hasNext() ? s.next() : "";
}
return "";
}

If the request body is empty, then it simply means that it's already been consumed beforehand. For example, by a request.getParameter(), getParameterValues() or getParameterMap() call. Just remove the lines doing those calls from your code.

This will work for all HTTP method.
public class HttpRequestWrapper extends HttpServletRequestWrapper {
private final String body;
public HttpRequestWrapper(HttpServletRequest request) throws IOException {
super(request);
body = IOUtils.toString(request.getReader());
}
#Override
public ServletInputStream getInputStream() throws IOException {
final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(getBody().getBytes());
ServletInputStream servletInputStream = new ServletInputStream() {
public int read() throws IOException {
return byteArrayInputStream.read();
}
#Override
public boolean isFinished() {
return false;
}
#Override
public boolean isReady() {
return false;
}
#Override
public void setReadListener(ReadListener listener) {
}
};
return servletInputStream;
}
public String getBody() {
return this.body;
}
}

Easiest way I could think of:
request.getReader().lines().reduce("",String::concat)
However, this will be one long string which you will have to parse. IF you send a username of tim and a password of 12345. The output of the code above would look like this:
{ "username":"tim", "password": "12345"}
Please be aware
Please be aware that with the reduce() method we are performing a Mutable Reduction which does a great deal of string copying and has a runtime of O(N^2) with N being the number of characters. Please check the Mutable Reduction documentation if you need a more performant result.

I resolved that situation in this way. I created a util method that return a object extracted from request body, using the readValue method of ObjectMapper that is capable of receiving a Reader.
public static <T> T getBody(ResourceRequest request, Class<T> class) {
T objectFromBody = null;
try {
ObjectMapper objectMapper = new ObjectMapper();
HttpServletRequest httpServletRequest = PortalUtil.getHttpServletRequest(request);
objectFromBody = objectMapper.readValue(httpServletRequest.getReader(), class);
} catch (IOException ex) {
log.error("Error message", ex);
}
return objectFromBody;
}

I personnally use this code (on a dev server, not in production). Seems to work. The main difficulty is that once you read the request body, it will be lost and not transferred to the app. So you have to "cache" it first.
/* Export this filter as a jar and place it under directory ".../tomcat/lib" on your Tomcat server/
In the lib directory, also place the dependencies you need
(ex. org.apache.commons.io => commons-io-2.8.0.jar)
Once this is done, in order to activate the filter, on the Tomcat server:
o in .../tomcat/conf/server.xml, add:
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs" prefix="localhost_access_log" suffix=".txt" pattern="%h %l %u %t "%r" [%{postdata}r] %s %b"/>
=> the server will log the "postdata" attribute we generate in the Java code.
o in .../tomcat/conf/web.xml, add:
<filter>
<filter-name>post-data-dumper-filter</filter-name>
<filter-class>filters.PostDataDumperFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>post-data-dumper-filter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
Once you've done this, restart your tomcat server. You will get extra infos in file "localhost_access_log.<date>.txt"
*/
package filters;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Enumeration;
import java.util.stream.Collectors;
import javax.servlet.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import org.apache.commons.io.IOUtils;
class MultiReadHttpServletRequest extends HttpServletRequestWrapper {
private ByteArrayOutputStream cachedBytes;
public MultiReadHttpServletRequest(HttpServletRequest request) {
super(request);
}
#Override
public ServletInputStream getInputStream() throws IOException {
if (cachedBytes == null)
cacheInputStream();
return new CachedServletInputStream();
}
#Override
public BufferedReader getReader() throws IOException {
return new BufferedReader(new InputStreamReader(getInputStream()));
}
private void cacheInputStream() throws IOException {
/* Cache the inputstream in order to read it multiple times.
*/
cachedBytes = new ByteArrayOutputStream();
IOUtils.copy(super.getInputStream(), cachedBytes);
}
/* An input stream which reads the cached request body */
public class CachedServletInputStream extends ServletInputStream {
private ByteArrayInputStream input;
public CachedServletInputStream() {
/* create a new input stream from the cached request body */
input = new ByteArrayInputStream(cachedBytes.toByteArray());
}
//---------------------
#Override
public int read() throws IOException {
return input.read();
}
#Override
public boolean isFinished() {
return input.available() == 0;
}
#Override
public boolean isReady() {
return true;
}
//---------------------
#Override
public void setReadListener(ReadListener arg0) {
// TODO Auto-generated method stub
// Ex. : throw new RuntimeException("Not implemented");
}
}
}
public final class PostDataDumperFilter implements Filter {
private FilterConfig filterConfig = null;
public void destroy() {
this.filterConfig = null;
}
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
if (filterConfig == null)
return;
StringBuffer output = new StringBuffer();
output.append("PostDataDumperFilter-");
/* Wrap the request in order to be able to read its body multiple times */
MultiReadHttpServletRequest multiReadRequest = new MultiReadHttpServletRequest((HttpServletRequest) request);
// TODO : test the method in order not to log the body when receiving GET/DELETE requests ?
// I finally leave it "as it", since I've seen GET requests containing bodies (hell...).
output.append("Content-type=" + multiReadRequest.getContentType());
output.append(" - HTTP Method=" + multiReadRequest.getMethod());
output.append(" - REQUEST BODY = " + multiReadRequest.getReader().lines().collect(Collectors.joining(System.lineSeparator())));
// Log the request parameters:
Enumeration names = multiReadRequest.getParameterNames();
if (names.hasMoreElements()) {
output.append("- REQUEST PARAMS = ");
}
while (names.hasMoreElements()) {
String name = (String) names.nextElement();
output.append(name + "=");
String values[] = multiReadRequest.getParameterValues(name);
for (int i = 0; i < values.length; i++) {
if (i > 0) output.append("' ");
output.append(values[i]);
}
if (names.hasMoreElements()) output.append("&");
}
multiReadRequest.setAttribute("postdata", output);
chain.doFilter(multiReadRequest, response);
}
public void init(FilterConfig filterConfig) throws ServletException {
this.filterConfig = filterConfig;
}
}