Note: Original Question Was Deleted
Original Question is https://stackoverflow.com/questions/15240011/get-the-download-url-for-youtube-video-android-java/15240012#15240012
here the answer is outdated, not working so i will post a new question and answering my self
OLDCODE
new YouTubePageStreamUriGetter().execute("https://www.youtube.com/watch?v=4GuqB1BQVr4");
class Meta {
public String num;
public String type;
public String ext;
Meta(String num, String ext, String type) {
this.num = num;
this.ext = ext;
this.type = type;
}
}
class Video {
public String ext = "";
public String type = "";
public String url = "";
Video(String ext, String type, String url) {
this.ext = ext;
this.type = type;
this.url = url;
}
}
public ArrayList<Video> getStreamingUrisFromYouTubePage(String ytUrl)
throws IOException {
if (ytUrl == null) {
return null;
}
// Remove any query params in query string after the watch?v=<vid> in
// e.g.
// http://www.youtube.com/watch?v=0RUPACpf8Vs&feature=youtube_gdata_player
int andIdx = ytUrl.indexOf('&');
if (andIdx >= 0) {
ytUrl = ytUrl.substring(0, andIdx);
}
// Get the HTML response
String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0.1)";
HttpClient client = new DefaultHttpClient();
client.getParams().setParameter(CoreProtocolPNames.USER_AGENT,
userAgent);
HttpGet request = new HttpGet(ytUrl);
HttpResponse response = client.execute(request);
String html = "";
InputStream in = response.getEntity().getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
StringBuilder str = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null) {
str.append(line.replace("\\u0026", "&"));
}
in.close();
html = str.toString();
// Parse the HTML response and extract the streaming URIs
if (html.contains("verify-age-thumb")) {
CLog.w("YouTube is asking for age verification. We can't handle that sorry.");
return null;
}
if (html.contains("das_captcha")) {
CLog.w("Captcha found, please try with different IP address.");
return null;
}
Pattern p = Pattern.compile("stream_map\": \"(.*?)?\"");
// Pattern p = Pattern.compile("/stream_map=(.[^&]*?)\"/");
Matcher m = p.matcher(html);
List<String> matches = new ArrayList<String>();
while (m.find()) {
matches.add(m.group());
}
if (matches.size() != 1) {
CLog.w("Found zero or too many stream maps.");
return null;
}
String urls[] = matches.get(0).split(",");
HashMap<String, String> foundArray = new HashMap<String, String>();
for (String ppUrl : urls) {
String url = URLDecoder.decode(ppUrl, "UTF-8");
Pattern p1 = Pattern.compile("itag=([0-9]+?)[&]");
Matcher m1 = p1.matcher(url);
String itag = null;
if (m1.find()) {
itag = m1.group(1);
}
Pattern p2 = Pattern.compile("sig=(.*?)[&]");
Matcher m2 = p2.matcher(url);
String sig = null;
if (m2.find()) {
sig = m2.group(1);
}
Pattern p3 = Pattern.compile("url=(.*?)[&]");
Matcher m3 = p3.matcher(ppUrl);
String um = null;
if (m3.find()) {
um = m3.group(1);
}
if (itag != null && sig != null && um != null) {
foundArray.put(itag, URLDecoder.decode(um, "UTF-8") + "&"
+ "signature=" + sig);
}
}
if (foundArray.size() == 0) {
CLog.w("Couldn't find any URLs and corresponding signatures");
return null;
}
HashMap<String, Meta> typeMap = new HashMap<String, Meta>();
typeMap.put("13", new Meta("13", "3GP", "Low Quality - 176x144"));
typeMap.put("17", new Meta("17", "3GP", "Medium Quality - 176x144"));
typeMap.put("36", new Meta("36", "3GP", "High Quality - 320x240"));
typeMap.put("5", new Meta("5", "FLV", "Low Quality - 400x226"));
typeMap.put("6", new Meta("6", "FLV", "Medium Quality - 640x360"));
typeMap.put("34", new Meta("34", "FLV", "Medium Quality - 640x360"));
typeMap.put("35", new Meta("35", "FLV", "High Quality - 854x480"));
typeMap.put("43", new Meta("43", "WEBM", "Low Quality - 640x360"));
typeMap.put("44", new Meta("44", "WEBM", "Medium Quality - 854x480"));
typeMap.put("45", new Meta("45", "WEBM", "High Quality - 1280x720"));
typeMap.put("18", new Meta("18", "MP4", "Medium Quality - 480x360"));
typeMap.put("22", new Meta("22", "MP4", "High Quality - 1280x720"));
typeMap.put("37", new Meta("37", "MP4", "High Quality - 1920x1080"));
typeMap.put("33", new Meta("38", "MP4", "High Quality - 4096x230"));
ArrayList<Video> videos = new ArrayList<ARViewer.Video>();
for (String format : typeMap.keySet()) {
Meta meta = typeMap.get(format);
if (foundArray.containsKey(format)) {
Video newVideo = new Video(meta.ext, meta.type,
foundArray.get(format));
videos.add(newVideo);
CLog.d("YouTube Video streaming details: ext:" + newVideo.ext
+ ", type:" + newVideo.type + ", url:" + newVideo.url);
}
}
return videos;
}
private class YouTubePageStreamUriGetter extends
AsyncTask<String, String, String> {
ProgressDialog progressDialog;
#Override
protected void onPreExecute() {
super.onPreExecute();
progressDialog = ProgressDialog.show(ARViewer.this, "",
"Connecting to YouTube...", true);
}
#Override
protected String doInBackground(String... params) {
String url = params[0];
try {
ArrayList<Video> videos = getStreamingUrisFromYouTubePage(url);
if (videos != null && !videos.isEmpty()) {
String retVidUrl = null;
for (Video video : videos) {
if (video.ext.toLowerCase().contains("mp4")
&& video.type.toLowerCase().contains("medium")) {
retVidUrl = video.url;
break;
}
}
if (retVidUrl == null) {
for (Video video : videos) {
if (video.ext.toLowerCase().contains("3gp")
&& video.type.toLowerCase().contains(
"medium")) {
retVidUrl = video.url;
break;
}
}
}
if (retVidUrl == null) {
for (Video video : videos) {
if (video.ext.toLowerCase().contains("mp4")
&& video.type.toLowerCase().contains("low")) {
retVidUrl = video.url;
break;
}
}
}
if (retVidUrl == null) {
for (Video video : videos) {
if (video.ext.toLowerCase().contains("3gp")
&& video.type.toLowerCase().contains("low")) {
retVidUrl = video.url;
break;
}
}
}
return retVidUrl;
}
} catch (Exception e) {
CLog.e("Couldn't get YouTube streaming URL", e);
}
CLog.w("Couldn't get stream URI for " + url);
return null;
}
#Override
protected void onPostExecute(String streamingUrl) {
super.onPostExecute(streamingUrl);
progressDialog.dismiss();
if (streamingUrl != null) {
/* Do what ever you want with streamUrl */
}
}
}
This Code is Not Working
Edit 3
You can use the Lib : https://github.com/HaarigerHarald/android-youtubeExtractor
Ex :
String youtubeLink = "http://youtube.com/watch?v=xxxx";
new YouTubeExtractor(this) {
#Override
public void onExtractionComplete(SparseArray<YtFile> ytFiles, VideoMeta vMeta) {
if (ytFiles != null) {
int itag = 22;
String downloadUrl = ytFiles.get(itag).getUrl();
}
}
}.extract(youtubeLink, true, true);
They decipherSignature using :
private boolean decipherSignature(final SparseArray<String> encSignatures) throws IOException {
// Assume the functions don't change that much
if (decipherFunctionName == null || decipherFunctions == null) {
String decipherFunctUrl = "https://s.ytimg.com/yts/jsbin/" + decipherJsFileName;
BufferedReader reader = null;
String javascriptFile;
URL url = new URL(decipherFunctUrl);
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestProperty("User-Agent", USER_AGENT);
try {
reader = new BufferedReader(new InputStreamReader(urlConnection.getInputStream()));
StringBuilder sb = new StringBuilder("");
String line;
while ((line = reader.readLine()) != null) {
sb.append(line);
sb.append(" ");
}
javascriptFile = sb.toString();
} finally {
if (reader != null)
reader.close();
urlConnection.disconnect();
}
if (LOGGING)
Log.d(LOG_TAG, "Decipher FunctURL: " + decipherFunctUrl);
Matcher mat = patSignatureDecFunction.matcher(javascriptFile);
if (mat.find()) {
decipherFunctionName = mat.group(1);
if (LOGGING)
Log.d(LOG_TAG, "Decipher Functname: " + decipherFunctionName);
Pattern patMainVariable = Pattern.compile("(var |\\s|,|;)" + decipherFunctionName.replace("$", "\\$") +
"(=function\\((.{1,3})\\)\\{)");
String mainDecipherFunct;
mat = patMainVariable.matcher(javascriptFile);
if (mat.find()) {
mainDecipherFunct = "var " + decipherFunctionName + mat.group(2);
} else {
Pattern patMainFunction = Pattern.compile("function " + decipherFunctionName.replace("$", "\\$") +
"(\\((.{1,3})\\)\\{)");
mat = patMainFunction.matcher(javascriptFile);
if (!mat.find())
return false;
mainDecipherFunct = "function " + decipherFunctionName + mat.group(2);
}
int startIndex = mat.end();
for (int braces = 1, i = startIndex; i < javascriptFile.length(); i++) {
if (braces == 0 && startIndex + 5 < i) {
mainDecipherFunct += javascriptFile.substring(startIndex, i) + ";";
break;
}
if (javascriptFile.charAt(i) == '{')
braces++;
else if (javascriptFile.charAt(i) == '}')
braces--;
}
decipherFunctions = mainDecipherFunct;
// Search the main function for extra functions and variables
// needed for deciphering
// Search for variables
mat = patVariableFunction.matcher(mainDecipherFunct);
while (mat.find()) {
String variableDef = "var " + mat.group(2) + "={";
if (decipherFunctions.contains(variableDef)) {
continue;
}
startIndex = javascriptFile.indexOf(variableDef) + variableDef.length();
for (int braces = 1, i = startIndex; i < javascriptFile.length(); i++) {
if (braces == 0) {
decipherFunctions += variableDef + javascriptFile.substring(startIndex, i) + ";";
break;
}
if (javascriptFile.charAt(i) == '{')
braces++;
else if (javascriptFile.charAt(i) == '}')
braces--;
}
}
// Search for functions
mat = patFunction.matcher(mainDecipherFunct);
while (mat.find()) {
String functionDef = "function " + mat.group(2) + "(";
if (decipherFunctions.contains(functionDef)) {
continue;
}
startIndex = javascriptFile.indexOf(functionDef) + functionDef.length();
for (int braces = 0, i = startIndex; i < javascriptFile.length(); i++) {
if (braces == 0 && startIndex + 5 < i) {
decipherFunctions += functionDef + javascriptFile.substring(startIndex, i) + ";";
break;
}
if (javascriptFile.charAt(i) == '{')
braces++;
else if (javascriptFile.charAt(i) == '}')
braces--;
}
}
if (LOGGING)
Log.d(LOG_TAG, "Decipher Function: " + decipherFunctions);
decipherViaWebView(encSignatures);
if (CACHING) {
writeDeciperFunctToChache();
}
} else {
return false;
}
} else {
decipherViaWebView(encSignatures);
}
return true;
}
Now with use of this library High Quality Videos Lossing Audio so i use the MediaMuxer for Murging Audio and Video for Final Output
Edit 1
https://stackoverflow.com/a/15240012/9909365
Why the previous answer not worked
Pattern p2 = Pattern.compile("sig=(.*?)[&]");
Matcher m2 = p2.matcher(url);
String sig = null;
if (m2.find()) {
sig = m2.group(1);
}
As of November 2016, this is a little rough around the edges, but
displays the basic principle. The url_encoded_fmt_stream_map today
does not have a space after the colon (better make this optional) and
"sig" has been changed to "signature"
and while i am debuging the code i found the new keyword its
signature&s in many video's URL
here edited answer
private static final HashMap<String, Meta> typeMap = new HashMap<String, Meta>();
initTypeMap(); call first
class Meta {
public String num;
public String type;
public String ext;
Meta(String num, String ext, String type) {
this.num = num;
this.ext = ext;
this.type = type;
}
}
class Video {
public String ext = "";
public String type = "";
public String url = "";
Video(String ext, String type, String url) {
this.ext = ext;
this.type = type;
this.url = url;
}
}
public ArrayList<Video> getStreamingUrisFromYouTubePage(String ytUrl)
throws IOException {
if (ytUrl == null) {
return null;
}
// Remove any query params in query string after the watch?v=<vid> in
// e.g.
// http://www.youtube.com/watch?v=0RUPACpf8Vs&feature=youtube_gdata_player
int andIdx = ytUrl.indexOf('&');
if (andIdx >= 0) {
ytUrl = ytUrl.substring(0, andIdx);
}
// Get the HTML response
/* String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0.1)";*/
/* HttpClient client = new DefaultHttpClient();
client.getParams().setParameter(CoreProtocolPNames.USER_AGENT,
userAgent);
HttpGet request = new HttpGet(ytUrl);
HttpResponse response = client.execute(request);*/
String html = "";
HttpsURLConnection c = (HttpsURLConnection) new URL(ytUrl).openConnection();
c.setRequestMethod("GET");
c.setDoOutput(true);
c.connect();
InputStream in = c.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
StringBuilder str = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null) {
str.append(line.replace("\\u0026", "&"));
}
in.close();
html = str.toString();
// Parse the HTML response and extract the streaming URIs
if (html.contains("verify-age-thumb")) {
Log.e("Downloader", "YouTube is asking for age verification. We can't handle that sorry.");
return null;
}
if (html.contains("das_captcha")) {
Log.e("Downloader", "Captcha found, please try with different IP address.");
return null;
}
Pattern p = Pattern.compile("stream_map\":\"(.*?)?\"");
// Pattern p = Pattern.compile("/stream_map=(.[^&]*?)\"/");
Matcher m = p.matcher(html);
List<String> matches = new ArrayList<String>();
while (m.find()) {
matches.add(m.group());
}
if (matches.size() != 1) {
Log.e("Downloader", "Found zero or too many stream maps.");
return null;
}
String urls[] = matches.get(0).split(",");
HashMap<String, String> foundArray = new HashMap<String, String>();
for (String ppUrl : urls) {
String url = URLDecoder.decode(ppUrl, "UTF-8");
Log.e("URL","URL : "+url);
Pattern p1 = Pattern.compile("itag=([0-9]+?)[&]");
Matcher m1 = p1.matcher(url);
String itag = null;
if (m1.find()) {
itag = m1.group(1);
}
Pattern p2 = Pattern.compile("signature=(.*?)[&]");
Matcher m2 = p2.matcher(url);
String sig = null;
if (m2.find()) {
sig = m2.group(1);
} else {
Pattern p23 = Pattern.compile("signature&s=(.*?)[&]");
Matcher m23 = p23.matcher(url);
if (m23.find()) {
sig = m23.group(1);
}
}
Pattern p3 = Pattern.compile("url=(.*?)[&]");
Matcher m3 = p3.matcher(ppUrl);
String um = null;
if (m3.find()) {
um = m3.group(1);
}
if (itag != null && sig != null && um != null) {
Log.e("foundArray","Adding Value");
foundArray.put(itag, URLDecoder.decode(um, "UTF-8") + "&"
+ "signature=" + sig);
}
}
Log.e("foundArray","Size : "+foundArray.size());
if (foundArray.size() == 0) {
Log.e("Downloader", "Couldn't find any URLs and corresponding signatures");
return null;
}
ArrayList<Video> videos = new ArrayList<Video>();
for (String format : typeMap.keySet()) {
Meta meta = typeMap.get(format);
if (foundArray.containsKey(format)) {
Video newVideo = new Video(meta.ext, meta.type,
foundArray.get(format));
videos.add(newVideo);
Log.d("Downloader", "YouTube Video streaming details: ext:" + newVideo.ext
+ ", type:" + newVideo.type + ", url:" + newVideo.url);
}
}
return videos;
}
private class YouTubePageStreamUriGetter extends AsyncTask<String, String, ArrayList<Video>> {
ProgressDialog progressDialog;
#Override
protected void onPreExecute() {
super.onPreExecute();
progressDialog = ProgressDialog.show(webViewActivity.this, "",
"Connecting to YouTube...", true);
}
#Override
protected ArrayList<Video> doInBackground(String... params) {
ArrayList<Video> fVideos = new ArrayList<>();
String url = params[0];
try {
ArrayList<Video> videos = getStreamingUrisFromYouTubePage(url);
/* Log.e("Downloader","Size of Video : "+videos.size());*/
if (videos != null && !videos.isEmpty()) {
for (Video video : videos)
{
Log.e("Downloader", "ext : " + video.ext);
if (video.ext.toLowerCase().contains("mp4") || video.ext.toLowerCase().contains("3gp") || video.ext.toLowerCase().contains("flv") || video.ext.toLowerCase().contains("webm")) {
ext = video.ext.toLowerCase();
fVideos.add(new Video(video.ext,video.type,video.url));
}
}
return fVideos;
}
} catch (Exception e) {
e.printStackTrace();
Log.e("Downloader", "Couldn't get YouTube streaming URL", e);
}
Log.e("Downloader", "Couldn't get stream URI for " + url);
return null;
}
#Override
protected void onPostExecute(ArrayList<Video> streamingUrl) {
super.onPostExecute(streamingUrl);
progressDialog.dismiss();
if (streamingUrl != null) {
if (!streamingUrl.isEmpty()) {
//Log.e("Steaming Url", "Value : " + streamingUrl);
for (int i = 0; i < streamingUrl.size(); i++) {
Video fX = streamingUrl.get(i);
Log.e("Founded Video", "URL : " + fX.url);
Log.e("Founded Video", "TYPE : " + fX.type);
Log.e("Founded Video", "EXT : " + fX.ext);
}
//new ProgressBack().execute(new String[]{streamingUrl, filename + "." + ext});
}
}
}
}
public void initTypeMap()
{
typeMap.put("13", new Meta("13", "3GP", "Low Quality - 176x144"));
typeMap.put("17", new Meta("17", "3GP", "Medium Quality - 176x144"));
typeMap.put("36", new Meta("36", "3GP", "High Quality - 320x240"));
typeMap.put("5", new Meta("5", "FLV", "Low Quality - 400x226"));
typeMap.put("6", new Meta("6", "FLV", "Medium Quality - 640x360"));
typeMap.put("34", new Meta("34", "FLV", "Medium Quality - 640x360"));
typeMap.put("35", new Meta("35", "FLV", "High Quality - 854x480"));
typeMap.put("43", new Meta("43", "WEBM", "Low Quality - 640x360"));
typeMap.put("44", new Meta("44", "WEBM", "Medium Quality - 854x480"));
typeMap.put("45", new Meta("45", "WEBM", "High Quality - 1280x720"));
typeMap.put("18", new Meta("18", "MP4", "Medium Quality - 480x360"));
typeMap.put("22", new Meta("22", "MP4", "High Quality - 1280x720"));
typeMap.put("37", new Meta("37", "MP4", "High Quality - 1920x1080"));
typeMap.put("33", new Meta("38", "MP4", "High Quality - 4096x230"));
}
Edit 2:
Some time This Code Not worked proper
Same-origin policy
https://en.wikipedia.org/wiki/Same-origin_policy
https://en.wikipedia.org/wiki/Cross-origin_resource_sharing
problem of Same-origin policy. Essentially, you cannot download this file from www.youtube.com because they are different domains. A workaround of this problem is [CORS][1].
Ref : https://superuser.com/questions/773719/how-do-all-of-these-save-video-from-youtube-services-work/773998#773998
url_encoded_fmt_stream_map // traditional: contains video and audio stream
adaptive_fmts // DASH: contains video or audio stream
Each of these is a comma separated array of what I would call "stream objects". Each "stream object" will contain values like this
url // direct HTTP link to a video
itag // code specifying the quality
s // signature, security measure to counter downloading
Each URL will be encoded so you will need to decode them. Now the tricky part.
YouTube has at least 3 security levels for their videos
unsecured // as expected, you can download these with just the unencoded URL
s // see below
RTMPE // uses "rtmpe://" protocol, no known method for these
The RTMPE videos are typically used on official full length movies, and are protected with SWF Verification Type 2. This has been around since 2011 and has yet to be reverse engineered.
The type "s" videos are the most difficult that can actually be downloaded. You will typcially see these on VEVO videos and the like. They start with a signature such as
AA5D05FA7771AD4868BA4C977C3DEAAC620DE020E.0F421820F42978A1F8EAFCDAC4EF507DB5
Then the signature is scrambled with a function like this
function mo(a) {
a = a.split("");
a = lo.rw(a, 1);
a = lo.rw(a, 32);
a = lo.IC(a, 1);
a = lo.wS(a, 77);
a = lo.IC(a, 3);
a = lo.wS(a, 77);
a = lo.IC(a, 3);
a = lo.wS(a, 44);
return a.join("")
}
This function is dynamic, it typically changes every day. To make it more difficult the function is hosted at a URL such as
http://s.ytimg.com/yts/jsbin/html5player-en_US-vflycBCEX.js
this introduces the problem of Same-origin policy. Essentially, you cannot download this file from www.youtube.com because they are different domains. A workaround of this problem is CORS. With CORS, s.ytimg.com could add this header
Access-Control-Allow-Origin: http://www.youtube.com
and it would allow the JavaScript to download from www.youtube.com. Of course they do not do this. A workaround for this workaround is to use a CORS proxy. This is a proxy that responds with the following header to all requests
Access-Control-Allow-Origin: *
So, now that you have proxied your JS file, and used the function to scramble the signature, you can use that in the querystring to download a video.
please help, i have some code to show a distance. before i use static code for get lat and long. but i need to get lat and long from user..need help how i can get this latitude and longitude from place_id /.
public void setDirectionMap()
ArrayList list = MainActivity.arrJsonTargetLocation;
JSONObject startLocation = MainActivity.jsonStartLocation;
ArrayList targetLocation = MainActivity.arrJsonTargetLocation;
if( list != null )
{
JSONObject targetLocationList = (JSONObject) list.get(list.size() - 1);
}
LatLng start_point = new LatLng(Double.parseDouble(startLocation.getString(``lat``)),
double.parseDouble(startLocation.getString(``lng``)));
LatLng target_point = new LatLng(Double.parseDouble(targetLocation.indexOf(``lat``)),
double.parseDouble(targetLocation.indexOf(``lng``)));
GoDelivery.geoPoints = new ArrayList<LatLng>();
Log.i(TAG, ``#FROM: `` + start_point.toString() + `` -- #TO : `` + target_point.toString());
System.out.println(``=======================ORDER==================================``);
System.out.println(``#FROM: `` + start_point.toString() + `` -- #TO : `` + target_point.toString());
HttpURLConnection conn = null;
StringBuilder jsonResults = new StringBuilder();
StringBuilder jsonResultsDetail = new StringBuilder();
try {
StringBuilder sb = new StringBuilder(DIRECTION_API_BASE + OUT_JSON);
//sb.append(``&components=country:gr``);
sb.append(``?key=`` + API_KEY);
sb.append(``&origin=`` +
start_point.latitude + ``,`` +
start_point.longitude +
``&destination=`` +
target_point.latitude + ``,`` +
target_point.longitude +
``&avoid=tolls``);
Log.i(TAG, ``Query String: `` + sb.toString());
URL url = new URL(sb.toString());
conn = (HttpURLConnection) url.openConnection();
InputStreamReader in = new InputStreamReader(conn.getInputStream());
// Load the results into a StringBuilder
int read;
char[] buff = new char[1024];
while ((read = in.read(buff)) != -1) {
jsonResults.append(buff, 0, read);
}
//Log.i(TAG,``===============DIRECTIONS===============``);
//Log.i(TAG,jsonResults.toString());
if (conn != null) {
conn.disconnect();
}
} catch (MalformedURLException e) {
Log.e(LOG_TAG, ``MalformedURL ``, e);
} catch (IOException e) {
Log.e(LOG_TAG, ``IOException ``, e);
} catch () {
Log.e(LOG_TAG, ``Throwable``, e);
}
try {
// double place_id = JSONArray.getDouble(``latitude``);
//double place_id = JSONArray.getDouble(``longitude``);
JSONObject jsonObj = new JSONObject(jsonResults.toString());
JSONObject jsonLeg = jsonObj.getJSONArray(``routes``).getJSONObject(0).getJSONArray(``legs``).getJSONObject(0);
JSONObject jsonSumDistance = jsonLeg.getJSONObject(``distance``);
JSONObject jsonSumDuration = jsonLeg.getJSONObject(``duration``);
JSONArray jsonSteps = jsonLeg.getJSONArray(``steps``);
JSONObject jsonObjlat = new JSONObject().getJSONObject(``lat``);
JSONObject jsonObjlng = new JSONObject().getJSONObject(``lng``);
Don't worry dude , It is not that to tough
Just pass your place id and your key in this url
This url will return json containing all the info of the place.
once you got json then you need to parse the json string to get the lat and lang .
I tried to request a large amount of data from freebase. But I got error message "HTTP response code: 403". Did anyone have similar problem before?
Here is my code
private static final String service_url = "https://www.googleapis.com/freebase/v1/mqlread";
public static void main(String[] args)
{
try
{
String cursor = "";
String urlStr_initial = service_url + "?query=" + URLEncoder.encode(getQuery(), "UTF-8") + "&cursor";
URL url = new URL(urlStr_initial);
List<Freebase> list = new ArrayList<Freebase>();
Freebase f_b;
do
{
HttpURLConnection url_con = (HttpURLConnection)url.openConnection();
url_con.addRequestProperty("User-Agent", "Mozilla/4.76");
StringBuilder str_builder = new StringBuilder();
BufferedReader reader = new BufferedReader(new InputStreamReader(url_con.getInputStream()));
String line;
while((line = reader.readLine()) != null)
{
str_builder.append(line);
}
String response = str_builder.toString();
JSONObject j_object = new JSONObject(response);
if(j_object.has("result"))
{
JSONArray j_array = j_object.getJSONArray("result");
for(int i = 0; i < j_array.length(); i++)
{
JSONObject j_o = j_array.getJSONObject(i);
if(j_o.has("id") == true && j_o.has("name"))
{
String id = j_o.getString("id");
String name = j_o.getString("name");
System.out.println("ID: " + id + " / Name:" + name);
f_b = new Freebase(id, name);
if(f_b != null)
{
list.add(f_b);
}
else
{
System.out.println("Null value in Freebase");
}
}
}
}
else
{
System.out.println("There is no \"result\" key in JASON object");
}
if(j_object.has("cursor"))
{
cursor = j_object.get("cursor").toString();
}
else
{
cursor = "false";
}
String urlStr = urlStr_initial + "=" + cursor;
url = new URL(urlStr);
}while( !cursor.equalsIgnoreCase("false"));
if(list != null)
{
TextFile tf = new TextFile();
tf.writeToFile(list);
}
}
catch(Exception e)
{
e.printStackTrace();
}
}
private static String getQuery()
{
return "[{"
+ "\"name\": null,"
+ "\"id\": null,"
+ "\"type\":\"/people/person\","
+ "\"limit\": 500"
+ "}]";
}
You don't say what "large" is, but the API isn't designed for bulk downloads. You should be using the data dumps for that.
There's usually more detailed error message included with the HTTP response code. If, for example, it says 403 - Forbidden - API key required, it means you didn't include your API key (I don't see where you include it in your code). If it says 403 - Forbidden - quota exceeded it means you've exceeded your request quota (you can look on the API console to see how much quota you have remaining).
I am wondering how to find the maven repository close to my home?
Actually, the listing at http://docs.codehaus.org/display/MAVENUSER/Mirrors+Repositories is problematic. Is Central in San Francisco, California or in St. Louis, Missouri? (I guess this is an inadvertent lesson on why code, or even xml, should not be commented). To make things more confusing, the owner of the domain (according to http://whois.net/whois/maven.org) is in Fulton, MD.
Maybe the question was not "Where are the repositories?" (which esaj answered), but really:
"How can I find the maven repository closest to my home?"
That question bugged me, too.
On the other hand, does it really matter, given how fast packets travel across the world?
But it still bugged me.
There are two ways to find out which host is the closest:
Open a browser, go to http://freegeoip.net, and type in the hostname. If you know where you are, then you can go to GoogleMaps, get directions, and check the distance.
Open a terminal window, and ping the hostname to find the average round trip transit time in milliseconds. This will give you the electronic distance, which is really more important for file transfer times.
Repeat for each hostname (i.e. about 40 times, if you're looking at the download sites for Maven itself)
Sort by the distance measured by the GoogleMaps directions, or by the transit time.
For 40 hosts you could probably do that manually in 20 tedious minutes, but a true professional would rather spend a few hours writing the Selenium and java code that could do it in 5 minutes.
In my copious amounts of spare time, I threw together only half the job (i.e. no Selenium):
public class ClosestUrlFinder {
/**
* Find out where a hostname is.
* Adapted from http://www.mkyong.com/java/how-to-send-http-request-getpost-in-java/
* Send HTTP GET requests to: freegeoip.net/{format}/{ip_or_hostname}
* The API supports both HTTP and HTTPS.
* Supported formats are csv, xml or json.
* E.G. http://freegeoip.net/xml/www.poolsaboveground.com
* returns <Response><Ip>207.58.184.93</Ip><CountryCode>US</CountryCode>
* <CountryName>United States</CountryName><RegionCode>VA</RegionCode><RegionName>Virginia</RegionName>
* <City>Mclean</City><ZipCode>22101</ZipCode>
* <Latitude>38.9358</Latitude><Longitude>-77.1621</Longitude>
* <MetroCode>511</MetroCode><AreaCode>703</AreaCode></Response>
* #param urlOrHostname
* #return
*/
public String getUrlLocation(String urlOrHostname) {
String USER_AGENT = "Mozilla/5.0";
BufferedReader in = null;
URL urlObject;
StringBuilder response = null;
try {
urlObject = new URL("http://freegeoip.net/xml/" + urlOrHostname);
HttpURLConnection connection = (HttpURLConnection) urlObject.openConnection();
connection.setRequestMethod("GET");
connection.setRequestProperty("User-Agent", USER_AGENT);
int responseCode = connection.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + urlOrHostname);
System.out.println("Response Code : " + responseCode);
in = new BufferedReader( new InputStreamReader(connection.getInputStream()));
String inputLine;
response = new StringBuilder();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (in != null) {
try { in.close(); }
catch (IOException e) {
e.printStackTrace();
}
}
}
//System.out.println(response.toString());
return response.toString();
}
/*
* Fixed version of code from http://stackoverflow.com/questions/11506321/java-code-to-ping-an-ip-address
*/
public String runDosCommand(List<String> command)
{
String string = "", result = "";
ProcessBuilder pBuilder = new ProcessBuilder(command);
Process process;
try {
process = pBuilder.start();
BufferedReader stdInput = new BufferedReader(new InputStreamReader(process.getInputStream()));
BufferedReader stdError = new BufferedReader(new InputStreamReader(process.getErrorStream()));
//System.out.println("Standard output of the command:");
while ((string = stdInput.readLine()) != null)
{
System.out.println(string);
result = result + "\n" + string;
}
while ((string = stdError.readLine()) != null)
{
System.out.println("stdError: "+ string);
result = result + "\nstdError: " + string;
}
} catch (IOException e) {
System.out.println("Error with command "+command);
e.printStackTrace();
}
return result;
}
public String pingUrl(String url) {
List<String> command = new ArrayList<String>();
String averagePingTime = "0", geoText = "", loss = "";
command.add("ping");
command.add("-n");
command.add("2");
url = url.replace("ftp://", "");
url = Utils.first(url.replace("/", " "));
url = Utils.first(url.replace(":", " "));
command.add(url);
System.out.println("command is: "+command);
String pingResult = runDosCommand(command);
String timeoutString = "Request timed out";
String timeout = Utils.grep(timeoutString, pingResult);
String noHostString = "Ping request could not find host";
String noHost = Utils.grep(noHostString, pingResult);
String unreachableString = "Destination net unreachable";
String unreachable = Utils.grep(unreachableString, pingResult);
if (Utils.isNullOrEmptyString(timeout) && Utils.isNullOrEmptyString(noHost)
&& Utils.isNullOrEmptyString(unreachable)) {
String lostString = "Lost =";
loss = Utils.grep(lostString, pingResult);
int index = loss.indexOf(lostString);
loss = loss.substring(index);
if (!loss.equals("Lost = 0 (0% loss),")) {
System.out.println("Non-zero loss for " + url);
averagePingTime = "0";
} else {
String replyString = "Reply from";
String replyFrom = Utils.grep(replyString, pingResult);
String ipAddress = Utils.nth(replyFrom, 3);
System.out.println("reply Ip="+ipAddress.replace(":", ""));
String averageString = "Average =";
averagePingTime = Utils.grep(averageString, pingResult);
index = averagePingTime.indexOf(averageString);
averagePingTime = averagePingTime.substring(index);
averagePingTime = Utils.last(averagePingTime).replace("ms", "");
String xml = getUrlLocation(url);
//System.out.println("xml="+xml);
geoText = extractTextFromXML(xml);
System.out.println("geoText="+geoText);
}
} else {
averagePingTime = "0";
System.out.println("Error. Either could not find host, Request timed out, or unreachable network for " + url);;
}
System.out.println("Results for " + url + " are: " + loss + " " + averagePingTime + " miliseconds.");
return url + " " + averagePingTime + " " + geoText ;
}
public ArrayList<Entry<String,Integer>> pingManyUrls() {
ArrayList<Entry<String,Integer>> mirrorTimeList = new ArrayList<>();
String[] urls = Utils.readTextFile("resources/MavenUrls.txt").split("\\s+");
System.out.println("Start pingManyUrls with "+urls.length + " urls.");
for (String url : urls) {
url = url.trim();
System.out.println("************************************\npingManyUrls: Check Url " + url);
if (arrayListContainsKey(url, mirrorTimeList)) {
System.out.println("Key " + url + " already in array.");
} else {
String pingInfo = pingUrl(url);
String averagePingString = Utils.nth(pingInfo, 2);
if (!averagePingString.equals("0")) {
int averagePingMilliseconds = Integer.parseInt(averagePingString);
pingInfo = Utils.rest(pingInfo); //chop the first term (the url)
pingInfo = Utils.rest(pingInfo); // chop the 2nd term (the time)
url = url + " " + pingInfo;
System.out.println(" Adding Key " + url + " " + averagePingMilliseconds + " to array.");
Entry<String,Integer> urlTimePair = new java.util.AbstractMap.SimpleEntry<>(url, averagePingMilliseconds);
mirrorTimeList.add(urlTimePair);
} else { System.out.println("Url " + url + " has a problem and therefore will be not be included."); }
}
}
Collections.sort(mirrorTimeList, new Comparator<Entry<String,Integer>>() {
#Override
public int compare (Entry<String,Integer> pair1, Entry<String,Integer> pair2) {
return pair1.getValue().compareTo(pair2.getValue());
}
});
return mirrorTimeList;
}
public boolean arrayListContainsKey(String key, ArrayList<Entry<String,Integer>> arrayList) {
for (Entry<String,Integer> keyValuePair : arrayList) {
//System.out.println(keyValuePair.getKey() + " =? " + key);
if (key.equalsIgnoreCase(keyValuePair.getKey())) { return true; }
}
return false;
}
public String extractFirstTagContents(Element parentElement, String tagname) {
NodeList list = parentElement.getElementsByTagName(tagname);
if (list != null && list.getLength()>0) {
String contents = list.item(0).getTextContent();
System.out.println("Tagname=" + tagname + " contents="+contents);
return contents;
}
return "";
}
public String extractTextFromXML(String xml) {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder;
String content = "";
try {
builder = factory.newDocumentBuilder();
Document document = builder.parse(new InputSource(new StringReader(xml)));
Element rootElement = document.getDocumentElement();
content = rootElement.getTextContent();
//System.out.println("content="+content);
// String city = extractFirstTagContents(rootElement, "City");
// String state = extractFirstTagContents(rootElement, "RegionName");
// String zipCode = extractFirstTagContents(rootElement, "ZipCode");
// String country = extractFirstTagContents(rootElement, "CountryName");
// String latitude = extractFirstTagContents(rootElement, "Latitude");
// String longitude = extractFirstTagContents(rootElement, "Longitude");
} catch (ParserConfigurationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (SAXException saxex) {
// TODO Auto-generated catch block
saxex.printStackTrace();
} catch (IOException ioex) {
// TODO Auto-generated catch block
ioex.printStackTrace();
}
return content;
}
public static void main(String[] args) {
System.out.println("Starting ClosestUrlFinder");
ClosestUrlFinder ping = new ClosestUrlFinder();
ArrayList<Entry<String,Integer>> mirrorList = ping.pingManyUrls();
System.out.println("Final Results, sorted by trip travel time (out of "+mirrorList.size()+" successful pings).");
for (Entry<String,Integer> urlTime : mirrorList) {
System.out.println(urlTime.getKey() + " " + urlTime.getValue());
}
}
} // End of Class
Where the data in MavenUrls.txt is:
apache.claz.org
apache.cs.utah.edu
apache.mesi.com.ar
apache.mirrors.hoobly.com
apache.mirrors.lucidnetworks.net
apache.mirrors.pair.com
apache.mirrors.tds.net
apache.osuosl.org
apache.petsads.us
apache.spinellicreations.com
apache.tradebit.com
download.nextag.com
ftp.osuosl.org
ftp://mirror.reverse.net/pub/apache/
mirror.cc.columbia.edu/pub/software/apache/
mirror.cogentco.com/pub/apache/
mirror.metrocast.net/apache/
mirror.nexcess.net/apache/
mirror.olnevhost.net/pub/apache/
mirror.reverse.net/pub/apache/
mirror.sdunix.com/apache/
mirror.symnds.com/software/Apache/
mirror.tcpdiag.net/apache/
mirrors.gigenet.com/apache/
mirrors.ibiblio.org/apache/
mirrors.sonic.net/apache/
psg.mtu.edu/pub/apache/
supergsego.com/apache/
www.bizdirusa.com
www.bizdirusa.com/mirrors/apache/
www.carfab.com/apachesoftware/
www.dsgnwrld.com/am/
www.eng.lsu.edu/mirrors/apache/
www.gtlib.gatech.edu/pub/apache/
www.interior-dsgn.com/apache/
www.motorlogy.com/apache/
www.picotopia.org/apache/
www.poolsaboveground.com/apache/
www.trieuvan.com/apache/
www.webhostingjams.com/mirror/apache/
And my Utils class includes the following:
/**
* Given a string of words separated by spaces, returns the first word.
* #param string
* #return
*/
public static String first(String string) {
if (isNullOrEmptyString(string)) { return ""; }
string = string.trim();
int index = string.indexOf(" "); //TODO: shouldn't this be "\\s+" to handle double spaces and tabs? That means regexp version of indexOf
if (index<0) { return string; }
return string.substring(0, index);
}
/**
* Given a string of words separated by spaces, returns the rest of the string after the first word.
* #param string
* #return
*/
public static String rest(String string) {
if (isNullOrEmptyString(string)) { return ""; }
string = string.trim();
int index = string.indexOf(" "); //TODO: shouldn't this be "\\s+" to handle double spaces and tabs? That means regexp version of indexOf
if (index<0) { return ""; }
return string.substring(index+1);
}
public static String grep(String regexp, String multiLineStringToSearch) {
String result = "";
//System.out.println("grep for '"+ regexp + "'");
String[] lines = multiLineStringToSearch.split("\\n");
//System.out.println("grep input string contains "+ lines.length + " lines.");
Pattern pattern = Pattern.compile(regexp);
for (String line : lines) {
Matcher matcher = pattern.matcher(line);
if (matcher.find()) {
//System.out.println("grep found match="+ line);
result = result + "\n" + line;
}
}
return result.trim();
}
/**
* Given the path and name of a plain text (ascii) file,
* reads it and returns the contents as a string.
* #param filePath
* #return
*/
public static String readTextFile(String filePath) {
BufferedReader reader;
String result = "";
int counter = 0;
try {
reader = new BufferedReader(new FileReader(filePath));
String line = "";
StringBuilder stringBuilder = new StringBuilder();
String lineSeparator = System.getProperty("line.separator");
while ((line = reader.readLine()) != null) {
counter = counter + 1;
stringBuilder.append(line);
stringBuilder.append(lineSeparator);
}
reader.close();
result = stringBuilder.toString();
logger.info("readTextFile: Read "+filePath+" with "+ counter + " lines, "+result.length()+" characters.");
return result;
} catch (FileNotFoundException e) {
logger.fatal("readTextFile: Could not find file "+filePath+".");
e.printStackTrace();
} catch (IOException e) {
logger.fatal("readTextFile: Could not read file "+filePath+".");
e.printStackTrace();
}
return "";
}
public static boolean isNullOrEmptyString(String s) {
return (s==null || s.equals(""));
}
/**
* Given a string of words separated by one or more whitespaces, returns the Nth word.
* #param string
* #return
*/
public static String nth(String string, int n) {
string = string.trim();
String[] splitstring = string.split("\\s+");
String result = "";
if (splitstring.length<n) { return ""; }
else {
result = splitstring[n - 1];
}
return result.trim();
}
/**
* Given a string of words separated by spaces, returns the last word.
* #param string
* #return
*/
public static String last(String string) {
return string.substring(string.trim().lastIndexOf(" ")+1, string.trim().length());
}
Enjoy!
Here's there official central repository mirror listing (as xml-file) and here's another list of central repository mirrors.