We Keep Coding

503 Service unavailable while uploading file on SharePoint online in Finish Upload method

I am trying to upload the file on SharePoint larger than 250MB. I have divided data of the file in small chunks say 100MB and used Start upload, continue Upload and Finish Upload of SharePoint. I am getting 503 service unavailable in finish upload method if my file size is 250MB or greater. However, The below code runs successfully for the file size up to 249MB. Any leads or help is much appreciated.
Thanks,
Bharti Gulati
package test;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.json.JSONObject;
public class SampleFileUpload1 {
private final static int chunkSize = 100 * 1024 * 1024;
public static void main(String args[]) throws IOException {
SampleFileUpload1 fileUpload = new SampleFileUpload1();
File file = new File("C:\\users\\bgulati\\Desktop\\twofivefive.txt");
fileUpload.genereateAndUploadChunks(file);
}
private static void executeRequest(HttpPost httpPost, String urlString) {
try {
HttpClient client = new DefaultHttpClient();
HttpResponse response = client.execute(httpPost);
System.out.println("Response Code: " + response.getStatusLine().getStatusCode());
System.out.println("Response getReasonPhrase: " + response.getStatusLine().getReasonPhrase());
System.out.println("Response getReasonPhrase: " + response.getEntity().getContent().toString());
BufferedReader br = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
while (true) {
String s = br.readLine();
if (s == null)
break;
System.out.println(s);
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IllegalStateException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void executeMultiPartRequest(String urlString, byte[] fileByteArray) throws IOException {
HttpPost postRequest = new HttpPost(urlString);
postRequest = addHeader(postRequest, "accessToken");
try {
postRequest.setEntity(new ByteArrayEntity(fileByteArray));
} catch (Exception ex) {
ex.printStackTrace();
}
executeRequest(postRequest, urlString);
}
private static HttpPost addHeader(HttpPost httpPost, String accessToken) {
httpPost.addHeader("Accept", "application/json;odata=verbose");
httpPost.setHeader("Authorization", "Bearer " + accessToken);
httpPost.setHeader("Content-Type", "application/json;odata=verbose;charset=utf-8");
return httpPost;
}
private static String getUniqueId(HttpResponse response, String key) throws ParseException, IOException {
if (checkResponse(response)) {
String responseString = EntityUtils.toString(response.getEntity(), "UTF-8");
JSONObject json = new JSONObject(responseString);
return json.getJSONObject("d").getString(key);
}
return "";
}
private static boolean checkResponse(HttpResponse response) throws ParseException, IOException {
if (response.getStatusLine().getStatusCode() == 200 || (response.getStatusLine().getStatusCode() == 201)) {
return true;
}
return false;
}
private String createDummyFile(String relativePath, String fileName) throws ClientProtocolException, IOException {
String urlString = "https://siteURL/_api/web/GetFolderByServerRelativeUrl('"+relativePath+"')/Files/add(url='" +fileName+"',overwrite=true)";
HttpPost postRequest = new HttpPost(urlString);
postRequest = addHeader(postRequest, "access_token");
HttpClient client = new DefaultHttpClient();
HttpResponse response = client.execute(postRequest);
return getUniqueId(response, "UniqueId");
}
private void genereateAndUploadChunks(File file) throws IOException {
String relativePath = "/relativePath";
String fileName = file.getName();
String gUid = createDummyFile(relativePath, fileName);
String endpointUrlS = "https://siteURL/_api/web/GetFileByServerRelativeUrl('"+relativePath+"/"+fileName+"')/savebinarystream";
HttpPost postRequest = new HttpPost(endpointUrlS);
postRequest = addHeader(postRequest, "access_token");
HttpClient client = new DefaultHttpClient();
HttpResponse response = client.execute(postRequest);
long fileSize = file.length();
if (fileSize <= chunkSize) {
}
else {
byte[] buffer = new byte[(int) fileSize <= chunkSize ? (int) fileSize : chunkSize];
long count = 0;
if (fileSize % chunkSize == 0)
count = fileSize / chunkSize;
else
count = (fileSize / chunkSize) + 1;
// try-with-resources to ensure closing stream
try (FileInputStream fis = new FileInputStream(file);
BufferedInputStream bis = new BufferedInputStream(fis)) {
int bytesAmount = 0;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
int i = 0;
String startUploadUrl = "";
int k = 0;
while ((bytesAmount = bis.read(buffer)) > 0) {
baos.write(buffer, 0, bytesAmount);
byte partialData[] = baos.toByteArray();
if (i == 0) {
startUploadUrl = "https://siteURL/_api/web/GetFileByServerRelativeUrl('"+relativePath+"/"+fileName+"')/StartUpload(uploadId=guid'"+gUid+"')";
executeMultiPartRequest(startUploadUrl, partialData);
System.out.println("first worked");
// StartUpload call
} else if (i == count-1) {
String finishUploadUrl = "https://siteURL/_api/web/GetFileByServerRelativeUrl('"+relativePath+"/"+fileName+"')/FinishUpload(uploadId=guid'"+gUid+"',fileOffset="+i+")";
executeMultiPartRequest(finishUploadUrl, partialData);
System.out.println("FinishUpload worked");
// FinishUpload call
} else {
String continueUploadUrl = "https://siteURL/_api/web/GetFileByServerRelativeUrl('"+relativePath+"/"+fileName+"')/ContinueUpload(uploadId=guid'"+gUid+"',fileOffset="+i+")";
executeMultiPartRequest(continueUploadUrl, partialData);
System.out.println("continue worked");
}
i++;
}
}
}
}
}

Is there a way to read a file from GitHub and print it in the console using Java?

I want to store a file in Github as a string in java and process it later in the code. How can I do that ?

We can execute curl commands in Java and use Basic authentication to access files.
URL url;String username="username",password="password",file="";
try {
url = new URL("https://www.bitbucket.com/raw-file-url");
URLConnection uc;
uc = url.openConnection();
uc.setRequestProperty("X-Requested-With", "Curl");
ArrayList<String> list=new ArrayList<String>();
String userpass = username + ":" + password;
String basicAuth = "Basic " + new String(new Base64().encode(userpass.getBytes()));//needs Base64 encoder, apache.commons.codec
uc.setRequestProperty("Authorization", basicAuth);
BufferedReader reader = new BufferedReader(new InputStreamReader(uc.getInputStream()));
String line = null;
while ((line = reader.readLine()) != null)
file=file+line+"\n";
System.out.println(file);
return file;
} catch (IOException e) {
System.out.println("Wrong username and password");
return null;
}

This is plain java way of doing this...
I used standard java.net.URL api and Base64 class to connect github and print a json like the below example :
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Base64;
public class GitConnect {
public static void main(String... args) throws Exception {
java.net.URL url = null;
String username = "user";
String password = "gitpwd";
String file = "";
try {
url = new java.net.URL("https://raw.githubusercontent.com/lrjoshi/webpage/master/public/post/c159s.csv");
java.net.URLConnection uc;
uc = url.openConnection();
uc.setRequestProperty("X-Requested-With", "Curl");
java.util.ArrayList<String> list = new java.util.ArrayList<String>();
String userpass = username + ":" + password;
String basicAuth = "Basic " + new String(Base64.getEncoder().encode(userpass.getBytes()));//needs Base64 encoder, apache.commons.codec
uc.setRequestProperty("Authorization", basicAuth);
BufferedReader reader = new BufferedReader(new InputStreamReader(uc.getInputStream()));
String line = null;
while ((line = reader.readLine()) != null)
file = file + line + "\n";
System.out.println(file);
} catch (IOException e) {
System.out.println("Wrong username and password");
}
}
}

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import java.util.Map;
public static String getTextFromGithub(String link) {
URL Url = null;
try {
Url = new URL(link);
} catch (MalformedURLException e1) {
e1.printStackTrace();
}
HttpURLConnection Http = null;
try {
Http = (HttpURLConnection) Url.openConnection();
} catch (IOException e1) {
e1.printStackTrace();
}
Map<String, List<String>> Header = Http.getHeaderFields();
for (String header : Header.get(null)) {
if (header.contains(" 302 ") || header.contains(" 301 ")) {
link = Header.get("Location").get(0);
try {
Url = new URL(link);
} catch (MalformedURLException e) {
e.printStackTrace();
}
try {
Http = (HttpURLConnection) Url.openConnection();
} catch (IOException e) {
e.printStackTrace();
}
Header = Http.getHeaderFields();
}
}
InputStream Stream = null;
try {
Stream = Http.getInputStream();
} catch (IOException e) {
e.printStackTrace();
}
String Response = null;
try {
Response = GetStringFromStream(Stream);
} catch (IOException e) {
e.printStackTrace();
}
return Response;
}
private static String GetStringFromStream(InputStream Stream) throws IOException {
if (Stream != null) {
Writer Writer = new StringWriter();
char[] Buffer = new char[2048];
try {
Reader Reader = new BufferedReader(new InputStreamReader(Stream, "UTF-8"));
int counter;
while ((counter = Reader.read(Buffer)) != -1) {
Writer.write(Buffer, 0, counter);
}
} finally {
Stream.close();
}
return Writer.toString();
} else {
return "No Contents";
}
}

FileNotFoundException for URL that works in browser

I am trying to use an API from https://us.mc-api.net/ for a project and I have made this as a test.
public static void main(String[] args){
try {
URL url = new URL("http://us.mc-api.net/v3/uuid/193nonaxishsl/csv/");
BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
String line;
while ((line = in.readLine()) != null) {
System.out.println(line);
}
in.close();
}
catch (MalformedURLException e) {
e.printStackTrace();
}
catch (IOException e) {
e.printStackTrace();
System.out.println("I/O Error");
}
}
}
And this is giving me an IOException error but when ever I open the same page in my web browser I get
false,Unknown-Username
which is what I want to get from the code. I am new and don't really know why it is happening or why.
EDIT: StackTrace
java.io.FileNotFoundException: http://us.mc-api.net/v3/uuid/193nonaxishsl/csv/
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(Unknown Source)
at java.net.URL.openStream(Unknown Source)
at com.theman1928.Test.Main.main(Main.java:13)

The URL is returning status code 404 and therefore the input stream (mild guess here) is not being created and therefore is null. Sort the status code and you should be OK.
Ran it with this CSV and it is fine: other csv
If the error code is important to you then you can use HttpURLConnection:
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
System.out.println("code:"+conn.getResponseCode());
In that way you can process the response code before proceeding with a quick if-then-else check.

I tried it with the Apache HTTP libraries. The API endpoint seems to return a status code of 404, hence your error. Code I used is below.
public static void main(String[] args) throws URISyntaxException, ClientProtocolException, IOException {
HttpClient httpclient = HttpClients.createDefault();
URIBuilder builder = new URIBuilder("http://us.mc-api.net/v3/uuid/193nonaxishsl/csv/");
URI uri = builder.build();
HttpGet request = new HttpGet(uri);
HttpResponse response = httpclient.execute(request);
System.out.println(response.getStatusLine().getStatusCode()); // 404
}
Switching out the http://us.mc-api.net/v3/uuid/193nonaxishsl/csv/ with www.example.com or whatever returns a status code of 200, which further proves an error with the API endpoint. You can take a look at [Apache HTTP Components] library here.

This has to do with how the wire protocols are working in comparison with the java.net classes and an actual browser. A browser is going to be much more sophisticated than the simple java.net API you are using.
If you want to get the equivalent response value in Java, then you need to use a richer HTTP API.
This code will give you the same response as the browser; however, you need to download the Apache HttpComponents jars
The code:
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import javax.net.ssl.HttpsURLConnection;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.HttpClients;
public class TestDriver
{
public static void main(String[] args)
{
try
{
String url = "http://us.mc-api.net/v3/uuid/193nonaxishsl/csv";
HttpGet httpGet = new HttpGet(url);
getResponseFromHTTPReq(httpGet, url);
}
catch (Throwable e)
{
e.printStackTrace();
}
}
private static String getResponseFromHTTPReq(HttpUriRequest httpReq, String url)
{
HttpClient httpclient = HttpClients.createDefault();
// Execute and get the response.
HttpResponse response = null;
HttpEntity entity = null;
try
{
response = httpclient.execute(httpReq);
entity = response.getEntity();
}
catch (IOException ioe)
{
throw new RuntimeException(ioe);
}
if (entity == null)
{
String errMsg = "No response entity back from " + url;
throw new RuntimeException(errMsg);
}
String returnRes = null;
InputStream is = null;
BufferedReader buf = null;
try
{
is = entity.getContent();
buf = new BufferedReader(new InputStreamReader(is, "UTF-8"));
System.out.println("Response Code : " + response.getStatusLine().getStatusCode());
StringBuilder sb = new StringBuilder();
String s = null;
while (true)
{
s = buf.readLine();
if (s == null || s.length() == 0)
{
break;
}
sb.append(s);
}
returnRes = sb.toString();
System.out.println("Response: [" + returnRes + "]");
}
catch (UnsupportedOperationException | IOException e)
{
throw new RuntimeException(e);
}
finally
{
if (buf != null)
{
try
{
buf.close();
}
catch (IOException e)
{
}
}
if (is != null)
{
try
{
is.close();
}
catch (IOException e)
{
}
}
}
return returnRes;
}
}
Outputs:
Response Code : 404
Response: [false,Unknown-Username]

parsing json data in API url using jsonparser

I have a JSON data in this URL : http://api.pemiluapi.org/calonpresiden/api/caleg/jk?apiKey=56513c05217f73e6be82d5542368ae4f
when I try parsing using this jsonparser code :
package percobaan;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.List;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.DefaultHttpClient;
import org.json.JSONException;
import org.json.JSONObject;
public class JSONParser {
static InputStream is = null;
static JSONObject jObj = null;
static String json = "";
// constructor
public JSONParser() {
}
public String makeHttpRequest(String url, String method) {
return this.makeHttpRequest(url, method, null);
}
public JSONObject getJSONFromUrl(String url, List<NameValuePair> params) {
// Making HTTP request
try {
// defaultHttpClient
DefaultHttpClient httpClient = new DefaultHttpClient();
HttpPost httpPost = new HttpPost(url);
httpPost.setEntity(new UrlEncodedFormEntity(params));
HttpResponse httpResponse = httpClient.execute(httpPost);
HttpEntity httpEntity = httpResponse.getEntity();
is = httpEntity.getContent();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(
is, "iso-8859-1"), 8);
StringBuilder sBuilder = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null) {
sBuilder.append(line + "\n");
}
is.close();
json = sBuilder.toString();
} catch (Exception exception) {
exception.printStackTrace();
}
// try parse the string to a JSON object
try {
jObj = new JSONObject(json);
} catch (JSONException e) {
}
// return JSON String
return jObj;
}
// function get json from url
// by making HTTP POST or GET mehtod
public String makeHttpRequest(String url, String method,
List<NameValuePair> params) {
// Making HTTP request
try {
DefaultHttpClient httpClient = new DefaultHttpClient();
// check for request method
if (method == "POST") {
HttpPost httpPost = new HttpPost(url);
if (params != null) {
httpPost.setEntity(new UrlEncodedFormEntity(params));
}
HttpResponse httpResponse = httpClient.execute(httpPost);
HttpEntity httpEntity = httpResponse.getEntity();
is = httpEntity.getContent();
} else if (method == "GET") {
if (params != null) {
String paramString = URLEncodedUtils
.format(params, "utf-8");
url += "?" + paramString;
}
HttpGet httpGet = new HttpGet(url);
// DefaultHttpClient httpClient = new
// DefaultHttpClient(httpParameters);
HttpResponse httpResponse = httpClient.execute(httpGet);
HttpEntity httpEntity = httpResponse.getEntity();
is = httpEntity.getContent();
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(
is, "iso-8859-1"), 8);
StringBuilder sb = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null) {
sb.append(line + "\n");
}
is.close();
json = sb.toString();
} catch (Exception e) {
}
// return JSON String
return json;
}
}
why the output just says :
{"data":[]}
this is my code :
package percobaan;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.apache.http.NameValuePair;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
/**
*
* #author nafian
*/
public class Coba {
public static void main(String[] args) {
ArrayList<HashMap<String, String>> daftar_d = new ArrayList<HashMap<String, String>>();
JSONParser jsonParser = new JSONParser();
String link_url = "http://api.pemiluapi.org/calonpresiden/api/caleg/jk?apiKey=56513c05217f73e6be82d5542368ae4f";
List<NameValuePair> params = new ArrayList<NameValuePair>();
String parsing = jsonParser.makeHttpRequest(link_url, "POST",
params);
System.out.print(parsing);
// try {
// JSONObject json = new JSONObject(parsing).getJSONObject("data").getJSONObject("results");
// JSONArray caleg = json.getJSONArray("caleg");
//
// for (int i = 0; i < caleg.length(); i++) {
// HashMap<String, String> map = new HashMap<String, String>();
// JSONObject ar = caleg.getJSONObject(i);
// String nama = ar.getString("nama");
// String calon = ar.getString("role");
//
// JSONArray riwayat = ar.getJSONArray("riwayat_pendidikan");
// for (int j = 0; j < riwayat.length(); j++) {
// JSONObject ringkasan = riwayat.getJSONObject(j);
// String ringkasan_p = ringkasan.getString("ringkasan");
// map.put("pendidikan_r", ringkasan_p);
// }
//
// map.put("nama", nama);
// map.put("calon", calon);
// daftar_d.add(map);
//
// }
// } catch (JSONException ex) {
// ex.printStackTrace();
// }
// for (int i = 0; i < daftar_d.size(); i++) {
//
// System.out.println(daftar_d.get(i).get("pendidikan_r").toString());
// }
}
}
Am I missing something?

I suggest you use JSON-SIMPLE, it will literally simplify your life.
https://code.google.com/p/json-simple/
Here is a small example for the given URL.
Please note that's I'm using Jersey for establishing the connection, but you can pretty much use anything you like instead.
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
...
String callString = "http://api.pemiluapi.org/calonpresiden/api/caleg/jk?apiKey=56513c05217f73e6be82d5542368ae4f";
Client client = Client.create();
WebResource webResource = client.resource(callString);
ClientResponse clientResponse = webResource.accept("application/json").get(ClientResponse.class);
if (clientResponse.getStatus() != 200) {
throw new RuntimeException("Failed"+ clientResponse.toString());
}
JSONObject resObj = (JSONObject)new JSONParser().parse(clientResponse.getEntity(String.class));
JSONObject data_obj = (JSONObject) resObj.get("data");
JSONObject results_obj = (JSONObject) data_obj.get("results");
JSONArray caleg_array = (JSONArray) results_obj.get("caleg");

DefaultHttpClient response code 400

http://acm.hdu.edu.cn/showproblem.php/../../../data/images/con208-1004-1.JPG
This is the url, if you copy it to the brower, it works well. But when I use the apache HttpClient to download this img.
HttpClient client = new DefaultHttpClient();
I can't download it. I think it's because of the "../" , But in my program, I have to use this url.

Response 400 means you didn't ask in the right way to httpClient...
i put next a simple attempt:
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
public class a {
public static void main(String[] args) throws ClientProtocolException, IOException{
HttpClient client = new DefaultHttpClient();
HttpGet request = new HttpGet("http://acm.hdu.edu.cn/data/images/con208-1004-1.JPG");
HttpResponse response = client.execute(request);
// Get the response
InputStream inputStream = response.getEntity().getContent();
OutputStream outputStream = null;
try {
// read this file into InputStream
// write the inputStream to a FileOutputStream
outputStream =
new FileOutputStream(new File("C:/test.holder-new.jpg"));
int read = 0;
byte[] bytes = new byte[1024];
while ((read = inputStream.read(bytes)) != -1) {
outputStream.write(bytes, 0, read);
}
System.out.println("Done!");
} catch (IOException e) {
e.printStackTrace();
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (outputStream != null) {
try {
// outputStream.flush();
outputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
pls also notice that DefaultHttpClient is deprecated. You can replace this:
HttpClient client = new DefaultHttpClient();
with:
HttpClient client = HttpClientBuilder.create().build();
this works to me, let me know if it helps.