I am creating a Blackberry application to display a full screen web view of a certain site. I have a working browserfield that displays properly but navigation from page to page is slower than that of the native browser. The browserfield does not seem to have a built in cache causing the load time to be slow. When I add the following code to manage the cache the site no longer displays properly.
BrowserFieldScreen.java:
import net.rim.device.api.browser.field2.*;
import net.rim.device.api.script.ScriptEngine;
import net.rim.device.api.system.*;
import net.rim.device.api.ui.*;
import net.rim.device.api.ui.component.*;
import net.rim.device.api.ui.container.*;
import org.w3c.dom.Document;
class BrowserFieldScreen extends MainScreen
{
BrowserField browserField;
LoadingScreen load = new LoadingScreen();;
public BrowserFieldScreen()
{
browserField = new BrowserField();
browserField.getConfig().setProperty(
BrowserFieldConfig.JAVASCRIPT_ENABLED,
Boolean.TRUE);
browserField.getConfig().setProperty(
BrowserFieldConfig.NAVIGATION_MODE,
BrowserFieldConfig.NAVIGATION_MODE_POINTER);
browserField.getConfig().setProperty(
BrowserFieldConfig.CONTROLLER,
new CacheProtocolController(browserField));
browserField.requestContent("http://www.stackoverflow.com");
add(browserField);
}
}
CacheProtocolController.java:
import javax.microedition.io.HttpConnection;
import javax.microedition.io.InputConnection;
import net.rim.device.api.browser.field2.BrowserField;
import net.rim.device.api.browser.field2.BrowserFieldRequest;
import net.rim.device.api.browser.field2.ProtocolController;
public class CacheProtocolController extends ProtocolController{
// The BrowserField instance
private BrowserField browserField;
// CacheManager will take care of cached resources
private CacheManager cacheManager;
public CacheProtocolController(BrowserField browserField) {
super(browserField);
this.browserField = browserField;
}
private CacheManager getCacheManager() {
if ( cacheManager == null ) {
cacheManager = new CacheManagerImpl();
}
return cacheManager;
}
/**
* Handle navigation requests (e.g., link clicks)
*/
public void handleNavigationRequest(BrowserFieldRequest request)
throws Exception
{
InputConnection ic = handleResourceRequest(request);
browserField.displayContent(ic, request.getURL());
}
/**
* Handle resource request
* (e.g., images, external css/javascript resources)
*/
public InputConnection handleResourceRequest(BrowserFieldRequest request)
throws Exception
{
// if requested resource is cacheable (e.g., an "http" resource),
// use the cache
if (getCacheManager() != null
&& getCacheManager().isRequestCacheable(request))
{
InputConnection ic = null;
// if requested resource is cached, retrieve it from cache
if (getCacheManager().hasCache(request.getURL())
&& !getCacheManager().hasCacheExpired(request.getURL()))
{
ic = getCacheManager().getCache(request.getURL());
}
// if requested resource is not cached yet, cache it
else
{
ic = super.handleResourceRequest(request);
if (ic instanceof HttpConnection)
{
HttpConnection response = (HttpConnection) ic;
if (getCacheManager().isResponseCacheable(response))
{
ic = getCacheManager().createCache(request.getURL(),
response);
}
}
}
return ic;
}
// if requested resource is not cacheable, load it as usual
return super.handleResourceRequest(request);
}
}
CacheManager.java:
import javax.microedition.io.HttpConnection;
import javax.microedition.io.InputConnection;
import net.rim.device.api.browser.field2.BrowserFieldRequest;
public interface CacheManager {
public boolean isRequestCacheable(BrowserFieldRequest request);
public boolean isResponseCacheable(HttpConnection response);
public boolean hasCache(String url);
public boolean hasCacheExpired(String url);
public InputConnection getCache(String url);
public InputConnection createCache(String url, HttpConnection response);
public void clearCache(String url);
}
CacheManagerImpl.java:
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.Hashtable;
import javax.microedition.io.HttpConnection;
import javax.microedition.io.InputConnection;
import net.rim.device.api.browser.field2.BrowserFieldRequest;
import net.rim.device.api.browser.field2.BrowserFieldResponse;
import net.rim.device.api.io.http.HttpHeaders;
public class CacheManagerImpl implements CacheManager {
private static final int MAX_STANDARD_CACHE_AGE = 2592000;
private Hashtable cacheTable;
public CacheManagerImpl() {
cacheTable = new Hashtable();
}
public boolean isRequestCacheable(BrowserFieldRequest request) {
// Only HTTP requests are cacheable
if (!request.getProtocol().equals("http")) {
return false;
}
// Don't cache the request whose method is not "GET".
if (request instanceof HttpConnection) {
if (!((HttpConnection) request).getRequestMethod().equals("GET"))
{
return false;
}
}
// Don't cache the request with post data.
if (request.getPostData() != null) {
return false;
}
// Don't cache authentication request.
if (request.getHeaders().getPropertyValue("Authorization") != null) {
return false;
}
return true;
}
public boolean isResponseCacheable(HttpConnection response) {
try {
if (response.getResponseCode() != 200) {
return false;
}
} catch (IOException ioe) {
return false;
}
if (!response.getRequestMethod().equals("GET")) {
return false;
}
if (containsPragmaNoCache(response)) {
return false;
}
if (isExpired(response)) {
return false;
}
if (containsCacheControlNoCache(response)) {
return false;
}
if ( response.getLength() <= 0 ) {
return false;
}
// additional checks can be implemented here to inspect
// the HTTP cache-related headers of the response object
return true;
}
private boolean isExpired(HttpConnection response) {
try
{
// getExpiration() returns 0 if not known
long expires = response.getExpiration();
if (expires > 0 && expires <= (new Date()).getTime()) {
return true;
}
return false;
} catch (IOException ioe) {
return true;
}
}
private boolean containsPragmaNoCache(HttpConnection response) {
try
{
if (response.getHeaderField("pragma") != null
&& response.getHeaderField("pragma")
.toLowerCase()
.indexOf("no-cache") >= 0)
{
return true;
}
return false;
} catch (IOException ioe) {
return true;
}
}
private boolean containsCacheControlNoCache(HttpConnection response) {
try {
String cacheControl = response.getHeaderField("cache-control");
if (cacheControl != null) {
cacheControl = removeSpace(cacheControl.toLowerCase());
if (cacheControl.indexOf("no-cache") >= 0
|| cacheControl.indexOf("no-store") >= 0
|| cacheControl.indexOf("private") >= 0
|| cacheControl.indexOf("max-age=0") >= 0) {
return true;
}
long maxAge = parseMaxAge(cacheControl);
if (maxAge > 0 && response.getDate() > 0) {
long date = response.getDate();
long now = (new Date()).getTime();
if (now > date + maxAge) {
// Already expired
return true;
}
}
}
return false;
} catch (IOException ioe) {
return true;
}
}
public InputConnection createCache(String url, HttpConnection response) {
byte[] data = null;
InputStream is = null;
try {
// Read data
int len = (int) response.getLength();
if (len > 0) {
is = response.openInputStream();
int actual = 0;
int bytesread = 0 ;
data = new byte[len];
while ((bytesread != len) && (actual != -1)) {
actual = is.read(data, bytesread, len - bytesread);
bytesread += actual;
}
}
} catch (IOException ioe) {
data = null;
} finally {
if (is != null) {
try {
is.close();
} catch (IOException ioe) {
}
}
if (response != null) {
try {
response.close();
} catch (IOException ioe) {
}
}
}
if (data == null) {
return null;
}
// Calculate expires
long expires = calculateCacheExpires(response);
// Copy headers
HttpHeaders headers = copyResponseHeaders(response);
// add item to cache
cacheTable.put(url, new CacheItem(url, expires, data, headers));
return new BrowserFieldResponse(url, data, headers);
}
private long calculateCacheExpires(HttpConnection response) {
long date = 0;
try {
date = response.getDate();
} catch (IOException ioe) {
}
if (date == 0) {
date = (new Date()).getTime();
}
long expires = getResponseExpires(response);
// If an expire date has not been specified assumes the maximum time
if ( expires == 0 ) {
return date + (MAX_STANDARD_CACHE_AGE * 1000L);
}
return expires;
}
private long getResponseExpires(HttpConnection response) {
try {
// Calculate expires from "expires"
long expires = response.getExpiration();
if (expires > 0) {
return expires;
}
// Calculate expires from "max-age" and "date"
if (response.getHeaderField("cache-control") != null) {
String cacheControl = removeSpace(response
.getHeaderField("cache-control")
.toLowerCase());
long maxAge = parseMaxAge(cacheControl);
long date = response.getDate();
if (maxAge > 0 && date > 0) {
return (date + maxAge);
}
}
} catch (IOException ioe) {
}
return 0;
}
private long parseMaxAge(String cacheControl) {
if (cacheControl == null) {
return 0;
}
long maxAge = 0;
if (cacheControl.indexOf("max-age=") >= 0) {
int maxAgeStart = cacheControl.indexOf("max-age=") + 8;
int maxAgeEnd = cacheControl.indexOf(',', maxAgeStart);
if (maxAgeEnd < 0) {
maxAgeEnd = cacheControl.length();
}
try {
maxAge = Long.parseLong(cacheControl.substring(maxAgeStart,
maxAgeEnd));
} catch (NumberFormatException nfe) {
}
}
// Multiply maxAge by 1000 to convert seconds to milliseconds
maxAge *= 1000L;
return maxAge;
}
private static String removeSpace(String s) {
StringBuffer result= new StringBuffer();
int count = s.length();
for (int i = 0; i < count; i++) {
char c = s.charAt(i);
if (c != ' ') {
result.append(c);
}
}
return result.toString();
}
private HttpHeaders copyResponseHeaders(HttpConnection response) {
HttpHeaders headers = new HttpHeaders();
try {
int index = 0;
while (response.getHeaderFieldKey(index) != null) {
headers.addProperty(response.getHeaderFieldKey(index),
response.getHeaderField(index));
index++;
}
} catch (IOException ioe) {
}
return headers;
}
public boolean hasCache(String url) {
return cacheTable.containsKey(url);
}
public boolean hasCacheExpired(String url) {
Object o = cacheTable.get(url);
if (o instanceof CacheItem) {
CacheItem ci = (CacheItem) o;
long date = (new Date()).getTime();
if (ci.getExpires() > date) {
return false;
} else {
// Remove the expired cache item
clearCache(url);
}
}
return true;
}
public void clearCache(String url) {
cacheTable.remove(url);
}
public InputConnection getCache(String url) {
Object o = cacheTable.get(url);
if (o instanceof CacheItem) {
CacheItem ci = (CacheItem) o;
return new BrowserFieldResponse(url,
ci.getData(),
ci.getHttpHeaders());
}
return null;
}
}
CacheItem.java:
import net.rim.device.api.io.http.HttpHeaders;
public class CacheItem {
private String url;
private long expires;
private byte[] data;
private HttpHeaders httpHeaders;
public CacheItem(String url,
long expires,
byte[] data,
HttpHeaders httpHeaders)
{
this.url = url;
this.expires = expires;
this.data = data;
this.httpHeaders = httpHeaders;
}
public String getUrl() {
return url;
}
public long getExpires() {
return expires;
}
public byte[] getData() {
return data;
}
public HttpHeaders getHttpHeaders() {
return httpHeaders;
}
}
Any help that can be giving towards this will be greatly appreciated. This really has me stumped. Thanks.
UPDATE: It looks like the caching only works at a certain level of the Blackberry libraries. I have added logic to check the current Software level and turn on the caching if it is supported by the device's current software level. This provides me with a good work around, but i would still like to know if there is a better way for the caching to work with all devices.
UPDATE 2 Based on comments: The site no longer displaying properly pertains to site not displaying the proper layout, images and text. It basically give a white background with links and text displaying as a bulleted list, all formatting removed.
I've been looking at your code, and the only thing I've found there's wrong with it, is you are completely ignoring the possibility of response.getLength(); returning less than zero (in CacheManagerImpl.createCache()). Although this didn't happen to me at the stackoverflow.com page, some pages use Transfer-Encoding: chunked, which means Content-Length is not present. This is, however, well handled, and should not cause the cache to fail (it would only be less effective).
I suggest testing your code on smaller problems, one step at a time. First, create cacheable page that only contains some text (like "hello") without any HTML tags. That should work pretty well, and in case it does not, it shouldn't be hard to determine where the data are getting lost. Or try to manually create cache item that does not expire and contains a webpage with no (external) stylesheet nor images, and see if it's even possible to pass it to BrowserField the way you do it. Then build on, add an image, add a style sheet so you can corner the problem.
The code is written very nicely, but at this point, it is not possible to help you because there are no evident flaws in the code and you are not explaining yourself very well, it is not clear how the error manifests itself, if it is every time or random, ... If I had a Blackberry device, I could probably try running the code for myself, but i don't.
Related
I developed an API in Laravel for reading some data with authentication. Now in my app I need to send the token in my header for API to respond. Every time i login with app, I'm getting token without problem, but i can't provide token in header and it's returning a null token response. Now every time i reopen my app everything is fine but problem appear only when i login.
Please read my code and say if you found the bug.
This is my (AppServer.kt). I use this for adding the token to my header
class AppServer private constructor(private val client: INetworkClient) : IAppServer {
private val gson = Gson()
companion object {
private var instance: AppServer? = null
fun getInstance(): AppServer {
val token = AppLoggedInUser.getInstance()?.userProfile?.userToken ?: ""
if (instance == null)
instance = AppServer(
MixedNetworkClient(
mHeaders = mutableMapOf(
Headers.CONTENT_TYPE to "application/json",
"X-Requested-With" to "XMLHttpRequest",
"Authorization" to "Bearer $token"
),
mBasePath = "https://myWebsiteURL.com/"
)
)
return instance!!
}
}
override fun getPurchasedItems(): Observable<Pair<ResponseState, List<OrderInfo>?>> {
return Observable.create<Pair<ResponseState, List<OrderInfo>?>> { emitter ->
val (items, statusCode, msg, error) =
client.get(
"api/users/${AppLoggedInUser.getInstance().userProfile.userId}/orders",
{ responseStr ->
try {
val orders = mutableListOf<OrderInfo>()
val jArray = JSONArray(responseStr)
for (i in 0 until jArray.length()) {
val jObject = jArray.getJSONObject(i)
val order =
gson.fromJson<OrderInfo>(jObject.toString(), OrderInfo::class.java)
if (order.isPaymentSuccessful)
orders.add(order)
}
if (orders.isEmpty())
emitter.onNext(pair(empty(), null))
else
emitter.onNext(pair(success(), orders))
} catch (t: Throwable) {
Timber.e(t)
emitter.onNext(pair(ResponseState.internalError(t), null))
}
})
}.attachSchedulers()
}
override fun getDiscountDetailsById(id: Int): Observable<Pair<ResponseState, DiscountDetails?>> {
return Observable.create<Pair<ResponseState, DiscountDetails?>> { emitter ->
try {
client.post("api/post", listOf("post_id" to id.toString())) { responseStr ->
val jObject = JSONObject(responseStr)
var discount: DiscountDetails? = null
if (jObject.has("ID"))
discount =
gson.fromJson<DiscountDetails>(jObject.toString(), DiscountDetails::class.java)
if (discount != null)
emitter.onNext(pair(success(), discount))
else
emitter.onNext(pair(notFound404(), null))
}
} catch (t: Throwable) {
Timber.e(t)
emitter.onNext(pair(ResponseState.internalError(t), null))
}
}.attachSchedulers()
}
}
and this is my (AppLoggedInUser.java) I use it for getting my current user token.
package com.fartaak.gilantakhfif.utilities;
import android.content.Context;
import com.fartaak.gilantakhfif.backend.server.ParsingGSON;
import com.fartaak.gilantakhfif.model.UserProfile;
public class AppLoggedInUser extends AppPreferences {
public static final String NAM_LOGIN_SdPs = "login";
public static final String KEY_USER_TOKEN = "tokenPor";
private static AppLoggedInUser mInstance;
private final String KEY_LOGIN = "prefP";
private boolean isCompletelyRegistered;
public static void init(Context context) {
if (mInstance == null) {
mInstance = new AppLoggedInUser(context);
}
}
public static AppLoggedInUser getInstance() {
if (mInstance != null) {
return mInstance;
} else {
throw new IllegalStateException(
"you should call init to initialize only once per app launch before calling getInstance");
}
}
private AppLoggedInUser(Context context) {
super(NAM_LOGIN_SdPs, context);
}
public String getUserToken() {
return getField(KEY_USER_TOKEN);
}
public void clearUserProfile() {
removeField(KEY_LOGIN);
}
public UserProfile getUserProfile() {
String data = getField(KEY_LOGIN);
if (data == null) {
return null;
}
//return new Gson().fromJson(data, UserProfile.class);
return ParsingGSON.getInstance().getParsingJSONObject(data, UserProfile.class, null);
}
public boolean isRegisterCompleted() {
if (!isCompletelyRegistered) {
UserProfile userProfile = getUserProfile();
if (userProfile.getUserName() == null) {
return false;
} else {
isCompletelyRegistered = true;
}
}
return true;
}
public boolean isUserProfile() {
return isField(KEY_LOGIN);
}
public void setUserProfile(UserProfile userProfile) {
String json = ParsingGSON.getInstance().toJSONObject(userProfile, UserProfile.class, null);
setField(KEY_LOGIN, json);
}
}
I think the problem is with my second class.
I'd like to use zuul to cache some requests. The Cache is stored in a Redis as a POJO and contains plaintext (not gzip compressed data).
For normal tests and integration tests, everything works pretty well. With a jmeter load test, some of the requests fails with
java.util.zip.ZipException: Not in GZIP format (from jmeter)
We figure out, that at this point, zuul is returning an empty response.
My PreFilter:
public class CachePreFilter extends CacheBaseFilter {
private static DynamicIntProperty INITIAL_STREAM_BUFFER_SIZE = DynamicPropertyFactory.getInstance().getIntProperty(ZuulConstants.ZUUL_INITIAL_STREAM_BUFFER_SIZE, 8192);
#Autowired
CounterService counterService;
public CachePreFilter(RedisCacheManager redisCacheManager, Properties properties) {
super(redisCacheManager, properties);
}
#Override
public Object run() {
RequestContext ctx = RequestContext.getCurrentContext();
CachedResponse data = getFromCache(ctx);
if (null != data) {
counterService.increment("counter.cached");
HttpServletResponse response = ctx.getResponse();
response.addHeader("X-Cache", "HIT");
if (null != data.getContentType()) {
response.setContentType(data.getContentType());
}
if (null != data.getHeaders()) {
for (Entry<String, String> header : data.getHeaders().entrySet()) {
if (!response.containsHeader(header.getKey())) {
response.addHeader(header.getKey(), header.getValue());
}
}
}
OutputStream outStream = null;
try {
outStream = response.getOutputStream();
boolean isGzipRequested = ctx.isGzipRequested();
if (null != data.getBody()) {
final String requestEncoding = ctx.getRequest().getHeader(ZuulHeaders.ACCEPT_ENCODING);
if (requestEncoding != null && HTTPRequestUtils.getInstance().isGzipped(requestEncoding)) {
isGzipRequested = true;
}
ByteArrayOutputStream byteArrayOutputStream = null;
ByteArrayInputStream is = null;
try {
if (isGzipRequested) {
byteArrayOutputStream = new ByteArrayOutputStream();
GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream);
gzipOutputStream.write(data.getBody().getBytes(StandardCharsets.UTF_8));
gzipOutputStream.flush();
gzipOutputStream.close();
ctx.setResponseGZipped(true);
is = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
logger.debug(String.format("Send gzip content %s", data.getBody()));
response.setHeader(ZuulHeaders.CONTENT_ENCODING, "gzip");
} else {
logger.debug(String.format("Send content %s", data.getBody()));
is = new ByteArrayInputStream(data.getBody().getBytes(StandardCharsets.UTF_8));
}
writeResponse(is, outStream);
} catch (Exception e) {
logger.error("Error at sending response " + e.getMessage(), e);
throw new RuntimeException("Failed to send content", e);
} finally {
if (null != byteArrayOutputStream) {
byteArrayOutputStream.close();
}
if (null != is) {
is.close();
}
}
}
ctx.setSendZuulResponse(false);
} catch (IOException e) {
logger.error("Cannot read from Stream " + e.getMessage(), e.getMessage());
} finally {
// don't close the outputstream
}
ctx.set(CACHE_HIT, true);
return data;
} else {
counterService.increment("counter.notcached");
}
ctx.set(CACHE_HIT, false);
return null;
}
private ThreadLocal<byte[]> buffers = new ThreadLocal<byte[]>() {
#Override
protected byte[] initialValue() {
return new byte[INITIAL_STREAM_BUFFER_SIZE.get()];
}
};
private void writeResponse(InputStream zin, OutputStream out) throws Exception {
byte[] bytes = buffers.get();
int bytesRead = -1;
while ((bytesRead = zin.read(bytes)) != -1) {
out.write(bytes, 0, bytesRead);
}
}
#Override
public int filterOrder() {
return 99;
}
#Override
public String filterType() {
return "pre";
}
}
My Post Filter
public class CachePostFilter extends CacheBaseFilter {
public CachePostFilter(RedisCacheManager redisCacheManager, Properties properties) {
super(redisCacheManager, properties);
}
#Override
public boolean shouldFilter() {
RequestContext ctx = RequestContext.getCurrentContext();
return super.shouldFilter() && !ctx.getBoolean(CACHE_HIT);
}
#Override
public Object run() {
RequestContext ctx = RequestContext.getCurrentContext();
HttpServletRequest req = ctx.getRequest();
HttpServletResponse res = ctx.getResponse();
if (isSuccess(res, ctx.getOriginResponseHeaders())) {
// Store only successful responses
String cacheKey = cacheKey(req);
if (cacheKey != null) {
String body = null;
if (null != ctx.getResponseBody()) {
body = ctx.getResponseBody();
} else if (null != ctx.getResponseDataStream()) {
InputStream is = null;
try {
is = ctx.getResponseDataStream();
final Long len = ctx.getOriginContentLength();
if (len == null || len > 0) {
if (ctx.getResponseGZipped()) {
is = new GZIPInputStream(is);
}
StringWriter writer = new StringWriter();
IOUtils.copy(is, writer, "UTF-8");
body = writer.toString();
if (null != body && !body.isEmpty()) {
ctx.setResponseDataStream(new ByteArrayInputStream(body.getBytes()));
ctx.setResponseGZipped(false);
ctx.setOriginContentLength(String.valueOf(body.getBytes().length));
} else {
ctx.setResponseBody("{}");
}
}
} catch (IOException e) {
logger.error("Cannot read body " + e.getMessage(), e);
} finally {
if (null != is) {
try {
is.close();
} catch (IOException e) {
}
}
}
saveToCache(ctx, cacheKey, body);
}
}
}
return null;
}
#Override
public int filterOrder() {
return 1;
}
#Override
public String filterType() {
return "post";
}
private boolean isSuccess(HttpServletResponse res, List<Pair<String, String>> originHeaders) {
if (res != null && res.getStatus() < 300) {
if (null != originHeaders) {
for (Pair<String, String> header : originHeaders) {
if (header.first().equals("X-CACHEABLE") && header.second().equals("1")) {
return true;
}
}
}
}
return false;
}
We test it without Redis (just store it into a local variable) and this is still the same. We logged always the response from cache (before gzip) and everything looks good.
(Posted on behalf of the question author).
Solution
We refactor our PostFilter and don't change so much in the Response for zuul. After this change, we don't see any problems any more:
Working Post Filter
public class CachePostFilter extends CacheBaseFilter {
public CachePostFilter(RedisCacheManager redisCacheManager, Properties properties) {
super(redisCacheManager, properties);
}
#Override
public boolean shouldFilter() {
RequestContext ctx = RequestContext.getCurrentContext();
return super.shouldFilter() && !ctx.getBoolean(CACHE_HIT);
}
#Override
public Object run() {
RequestContext ctx = RequestContext.getCurrentContext();
HttpServletRequest req = ctx.getRequest();
HttpServletResponse res = ctx.getResponse();
if (isSuccess(res, ctx.getOriginResponseHeaders())) {
// Store only successful responses
String cacheKey = cacheKey(req);
if (cacheKey != null) {
String body = null;
if (null != ctx.getResponseBody()) {
body = ctx.getResponseBody();
} else if (null != ctx.getResponseDataStream()) {
InputStream rawInputStream = null;
InputStream gzipByteArrayInputStream = null;
try {
rawInputStream = ctx.getResponseDataStream();
gzipByteArrayInputStream = null;
// If origin tell it's GZipped but the content is ZERO
// bytes,
// don't try to uncompress
final Long len = ctx.getOriginContentLength();
if (len == null || len > 0) {
byte[] rawData = IOUtils.toByteArray(rawInputStream);
ctx.setResponseDataStream(new ByteArrayInputStream(rawData));
if (ctx.getResponseGZipped()) {
gzipByteArrayInputStream = new GZIPInputStream(new ByteArrayInputStream(rawData));
} else {
gzipByteArrayInputStream = new ByteArrayInputStream(rawData);
}
StringWriter writer = new StringWriter();
IOUtils.copy(gzipByteArrayInputStream, writer, "UTF-8");
body = writer.toString();
}
} catch (IOException e) {
logger.error("Cannot read body " + e.getMessage(), e);
} finally {
if (null != rawInputStream) {
try {
rawInputStream.close();
} catch (IOException e) {
}
}
if (null != gzipByteArrayInputStream) {
try {
gzipByteArrayInputStream.close();
} catch (IOException e) {
}
}
}
// if we read from the stream, the other filter cannot read
// and they dont' deliver any response
// ctx.setResponseBody(body);
// ctx.setResponseGZipped(false);
saveToCache(ctx, cacheKey, body);
}
}
}
return null;
}
#Override
public int filterOrder() {
return 1;
}
#Override
public String filterType() {
return "post";
}
private boolean isSuccess(HttpServletResponse res, List<Pair<String, String>> originHeaders) {
if (res != null && res.getStatus() == 200) {
if (null != originHeaders) {
for (Pair<String, String> header : originHeaders) {
if (header.first().equals("X-CACHEABLE") && header.second().equals("1")) {
return true;
}
}
}
}
return false;
}
}
I want to cache static files with littleproxy. So I created HashMap with key for uri and values for response body. Here's java code:
private static Map<String, FullHttpResponse> cache = new HashMap<>();
private static HttpFiltersSource getHttpFiltersSource() {
return new HttpFiltersSourceAdapter() {
#Override
public int getMaximumResponseBufferSizeInBytes() {
return 10 * 1024 * 1024;
}
public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
return new HttpFiltersAdapter(originalRequest) {
#Override
public HttpResponse clientToProxyRequest(HttpObject httpObject) {
if (httpObject instanceof HttpRequest) {
HttpRequest request = (HttpRequest) httpObject;
String requestUri = request.getUri();
if(requestUri.matches(".*[./]png.*$") ||
requestUri.matches(".*[./]jpg.*$") ||
requestUri.matches(".*[./]jpeg.*$") ||
requestUri.matches(".*[./]woff2.*$") ||
requestUri.matches(".*[./]js.*$") ) {
if (cache.containsKey(requestUri)) {
System.out.println("GOT FROM CACHE " + requestUri);
return cache.get(requestUri);
}
}
}
return null;
}
#Override
public HttpObject serverToProxyResponse(HttpObject httpObject) {
if (httpObject instanceof FullHttpResponse) {
FullHttpResponse response = (FullHttpResponse) httpObject;
String requestUri = originalRequest.getUri();
if(requestUri.matches(".*[./]png.*$") ||
requestUri.matches(".*[./]jpg.*$") ||
requestUri.matches(".*[./]jpeg.*$") ||
requestUri.matches(".*[./]woff2.*$") ||
requestUri.matches(".*[./]js.*$") ) {
cache.put(requestUri, response.retain());
System.out.println("ADDED TO CACHE " + requestUri);
}
}
return httpObject;
}
};
}
};
}
But something wrong here with response in Map. When the browser reaches static files firstly there's a debug message in console: ADDED TO CACHE. When the browser reaches static files secondly there's a message: "GOT FROM CACHE", but browser spins forever waiting for a response.
What's the right way to save and store responses from server and return it to client when time comes?
I think you need to also duplicate the FullHttpResponse to ensure you have correct writer / reader indices.
cache.put(requestUri, response.duplicate().retain());
and:
return cache.get(requestUri).duplicate();
Also ensure that you call release() once you remove something from the cache.
In the Java Spring framework, how could I create an end point that starts consuming the HTTP request body in chunks before the request has finished?
It seem like the default behavior i that the end point method is not executed until the request has ended.
The following Node.js server starts consuming the request body, how to do the same with the Java Spring framework?
const http = require('http');
const server = http.createServer((request, response) => {
request.on('data', (chunk) => {
console.log('NEW CHUNK: ', chunk.toString().length);
});
request.on('end', () => {
response.end('done');
});
});
server.listen(3000);
Outputs:
NEW CHUNK: 65189
NEW CHUNK: 65536
NEW CHUNK: 65536
NEW CHUNK: 65536
NEW CHUNK: 54212
I'm not sure there is a solution for mapping chunked request with spring what i'll do is something like this:
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import javax.servlet.http.HttpServletRequest;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
#Controller
public class ChunkController {
private static final int EOS = -1;
#RequestMapping(method = RequestMethod.POST)
public ResponseEntity<Void> upload(final HttpServletRequest request, #RequestParam final int chunkSize) {
try (InputStream in = request.getInputStream()) {
byte[] readBuffer = new byte[chunkSize];
int nbByteRead = 0;
int remainingByteToChunk = chunkSize;
while ((nbByteRead = in.read(readBuffer, chunkSize - remainingByteToChunk, remainingByteToChunk)) != EOS) {
remainingByteToChunk -= nbByteRead;
if (remainingByteToChunk == 0) {
byte[] chunk = Arrays.copyOf(readBuffer, readBuffer.length);
remainingByteToChunk = readBuffer.length;
// do something with the chunk.
}
}
if (remainingByteToChunk != chunkSize) {
byte[] lastChunk = Arrays.copyOf(readBuffer, readBuffer.length - remainingByteToChunk);
// do something with the last chunk
}
return new ResponseEntity<>(HttpStatus.OK);
} catch (IOException e) {
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
}
or you can define a constant for the size of the chunk.
You can also ignore the size the chunk and just handle the result of in.read until the end of stream.
Reading like this you will need to parse data to find actual chunk sent by the client. A typical body will look like:
7\r\n
Mozilla\r\n
9\r\n
Developer\r\n
7\r\n
Network\r\n
0\r\n
\r\n
What you can do is create custom InputStream like this (adapted from Apache HttpClient ChunkedInputStream)
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Objects;
public class ChunkedInputStream extends InputStream {
public static final byte[] EMPTY = new byte[0];
private final Charset charset;
private InputStream in;
private int chunkSize;
private int pos;
private boolean bof = true;
private boolean eof = false;
private boolean closed = false;
public ChunkedInputStream(final InputStream in, Charset charset) throws IOException {
if (in == null) {
throw new IllegalArgumentException("InputStream parameter may not be null");
}
this.in = in;
this.pos = 0;
this.charset = Objects.requireNonNullElse(charset, StandardCharsets.US_ASCII);
}
public int read() throws IOException {
if (closed) {
throw new IOException("Attempted read from closed stream.");
}
if (eof) {
return -1;
}
if (pos >= chunkSize) {
nextChunk();
if (eof) {
return -1;
}
}
pos++;
return in.read();
}
public int read(byte[] b, int off, int len) throws IOException {
if (closed) {
throw new IOException("Attempted read from closed stream.");
}
if (eof) {
return -1;
}
if (pos >= chunkSize) {
nextChunk();
if (eof) {
return -1;
}
}
len = Math.min(len, chunkSize - pos);
int count = in.read(b, off, len);
pos += count;
return count;
}
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
public byte[] readChunk() throws IOException {
if (eof) {
return EMPTY;
}
if (pos >= chunkSize) {
nextChunk();
if (eof) {
return EMPTY;
}
}
byte[] chunk = new byte[chunkSize];
int nbByteRead = 0;
int remainingByteToChunk = chunkSize;
while (remainingByteToChunk > 0 && !eof) {
nbByteRead = read(chunk, chunkSize - remainingByteToChunk, remainingByteToChunk);
remainingByteToChunk -= nbByteRead;
}
if (remainingByteToChunk == 0) {
return chunk;
} else {
return Arrays.copyOf(chunk, chunk.length - remainingByteToChunk);
}
}
private void readCRLF() throws IOException {
int cr = in.read();
int lf = in.read();
if ((cr != '\r') || (lf != '\n')) {
throw new IOException(
"CRLF expected at end of chunk: " + cr + "/" + lf);
}
}
private void nextChunk() throws IOException {
if (!bof) {
readCRLF();
}
chunkSize = getChunkSizeFromInputStream(in);
bof = false;
pos = 0;
if (chunkSize == 0) {
eof = true;
}
}
private int getChunkSizeFromInputStream(final InputStream in)
throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
// States: 0=normal, 1=\r was scanned, 2=inside quoted string, -1=end
int state = 0;
while (state != -1) {
int b = in.read();
if (b == -1) {
throw new IOException("chunked stream ended unexpectedly");
}
switch (state) {
case 0:
switch (b) {
case '\r':
state = 1;
break;
case '\"':
state = 2;
/* fall through */
default:
baos.write(b);
}
break;
case 1:
if (b == '\n') {
state = -1;
} else {
// this was not CRLF
throw new IOException("Protocol violation: Unexpected"
+ " single newline character in chunk size");
}
break;
case 2:
switch (b) {
case '\\':
b = in.read();
baos.write(b);
break;
case '\"':
state = 0;
/* fall through */
default:
baos.write(b);
}
break;
default:
throw new RuntimeException("assertion failed");
}
}
//parse data
String dataString = baos.toString(charset);
int separator = dataString.indexOf(';');
dataString = (separator > 0)
? dataString.substring(0, separator).trim()
: dataString.trim();
int result;
try {
result = Integer.parseInt(dataString.trim(), 16);
} catch (NumberFormatException e) {
throw new IOException("Bad chunk size: " + dataString);
}
return result;
}
public void close() throws IOException {
if (!closed) {
try {
if (!eof) {
exhaustInputStream(this);
}
} finally {
eof = true;
closed = true;
}
}
}
static void exhaustInputStream(InputStream inStream) throws IOException {
// read and discard the remainder of the message
byte[] buffer = new byte[1024];
while (inStream.read(buffer) >= 0) {
;
}
}
}
In the controller you can keep the same controller code but wrap the request.getInputStream() with this but you still won't get the actual client chunk. That's why I add the readChunk() method
#PostMapping("/upload")
public ResponseEntity<Void> upload(final HttpServletRequest request, #RequestHeader HttpHeaders headers) {
Charset charset = StandardCharsets.US_ASCII;
if (headers.getContentType() != null) {
charset = Objects.requireNonNullElse(headers.getContentType().getCharset(), charset);
}
try (ChunkedInputStream in = new ChunkedInputStream(request.getInputStream(), charset)) {
byte[] chunk;
while ((chunk = in.readChunk()).length > 0) {
// do something with the chunk.
System.out.println(new String(chunk, Objects.requireNonNullElse(charset, StandardCharsets.US_ASCII)));
}
return new ResponseEntity<>(HttpStatus.OK);
} catch (IOException e) {
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
I want the simplest logger there is to simply log errors, mostly exceptions, to a Log file in the android file system. for exemple, the easiest and most convinient (in my opinion at least) way that i used to log to a file on PC java was by simply printing all exceptions to console and redirecting system out to both console and my file, this doesnt really suffice on android as far as i know i guess its because of how Android OS is designed, so what is the simplest way of doing it in Android?
Note that the project has already lot of code in it and i really wouldnt like to go over it and add log calls on catch blocks or whatever to log my exceptions, as little i need to do for logging those exceptions is best for my use case...
Thanks ahead!
It's not finished but works quite stable. It saves human readable json array with exception name, time, stack trace and additional data. Also you can save logcat's logs.
Using:
ExceptionWriter ew = new ExceptionWriter(new File(Environment.getExternalStorageDirectory(), "debug.txt"));
ew.w(new IllegalArgumentException("some msg"), "additional message");
Source:
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
* User: elevenetc
* Date: 10/9/13
* Time: 12:52 PM
*/
public class ExceptionWriter {
private final StringBuilder sb;
private final ExceptionWriter.WriteExceptionTask writeExceptionTask;
private final SimpleDateFormat dataFormat;
private int totalExceptions;
private StringBuilder stackBuilder = new StringBuilder();
public int getTotalExceptions(){return totalExceptions;}
public ExceptionWriter(File file) throws IOException {
if(file != null){
writeExceptionTask = new WriteExceptionTask(file);
sb = new StringBuilder();
dataFormat = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss");
new Thread(writeExceptionTask).start();
}else{
sb = null;
writeExceptionTask = null;
dataFormat = null;
}
}
public synchronized int wLogcat(){
try {
writeExceptionTask.addStreamToRead(Runtime.getRuntime().exec("logcat -d -v time").getInputStream());
} catch (IOException e) {
e.printStackTrace();
}
return 0;
}
public int w(Exception debugException, String caughtMessage){
return w(debugException, caughtMessage, null);
}
public synchronized int w(Exception debugException, String caughtMessage, String additionalData){
if(writeExceptionTask == null) return -1;
sb.setLength(0);
StackTraceElement[] stackTrace = debugException == null ? null : debugException.getStackTrace();
sb.append("{\"date\":\"");sb.append(getTime());
sb.append("\",\"exceptionClassName\":\"");sb.append(debugException == null ? null : debugException.getClass());
sb.append("\",\"exceptionMessage:\":\"");sb.append(debugException == null ? null : debugException.getMessage());
sb.append("\",\"caughtMessage:\":\"");sb.append(caughtMessage);
if(additionalData != null) {sb.append("\",\"data:\":\"");sb.append(additionalData);}
sb.append("\",\"stack\":");sb.append(stackToString(stackTrace));
sb.append("},");
writeExceptionTask.stringQueue.add(sb.toString());
totalExceptions++;
return 0;
}
public void destroy() {
if(writeExceptionTask != null) {
writeExceptionTask.stop();
}
}
private String getTime(){
return dataFormat.format(System.currentTimeMillis());
}
private String stackToString(StackTraceElement[] stackTrace){
if(stackTrace == null) return null;
stackBuilder.setLength(0);
stackBuilder.append("[");
for (int i = 0; i < stackTrace.length; i++) {
StackTraceElement e = stackTrace[i];
stackBuilder.append("{\"");
stackBuilder.append(e.getLineNumber());
stackBuilder.append("\":\"");
stackBuilder.append(e.getClassName());
stackBuilder.append(".");
stackBuilder.append(e.getMethodName());
stackBuilder.append("\"}");
if(i != stackTrace.length -1) stackBuilder.append(",");
}
stackBuilder.append("]");
return stackBuilder.toString();
}
///////////////////////////////////////////////
/// Static classes
///////////////////////////////////////////////
private class WriteExceptionTask implements Runnable {
private final File file;
private boolean running;
private final ConcurrentLinkedQueue<String> stringQueue;
private final ConcurrentLinkedQueue<InputStream> isQueue;
private final FileWriter writer;
private WriteExceptionTask(File file) throws IOException {
this.file = file;
writer = new FileWriter(this.file, true);
stringQueue = new ConcurrentLinkedQueue<String>();
isQueue = new ConcurrentLinkedQueue<InputStream>();
running = true;
}
public void addStreamToRead(InputStream is){
if(is != null){
isQueue.add(is);
}
}
#Override
public void run() {
while(running){
if(!stringQueue.isEmpty()){
//TODO check file existence
try {
writer.append(stringQueue.poll());
writer.flush();
} catch (IOException e) {
e.printStackTrace();
running = false;
}
}
if(!isQueue.isEmpty()){
InputStream is = isQueue.poll();
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
StringBuilder builder = new StringBuilder("{\"catLog\":\"");
String aux;
try {
while ((aux = reader.readLine()) != null) {
//TODO view like array or \n
builder.append(aux);
}
} catch (IOException e) {
e.printStackTrace();
}
builder.append("\"},");
stringQueue.add(builder.toString());
}
}
try {
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public void stop() {
running = false;
}
}
}