I'm doing the Project Hyperlink Crawler for inspecting broken link. This is my code. www.utem.edu.my/portal/portal. So that the link give 404 error. I think my code for split URL is something wrong. Help me please.
public class HInterface extends JFrame {
// Declaring variables to be used as components in the interface
private JLabel lblURL;
private JTextField inputSearch;
private JButton btnSearch;
private JEditorPane outputLinks;
public HInterface() {
super("Hyperlink Crawler");
setType(Type.POPUP);
setResizable(false);
getContentPane().setBackground(Color.BLACK);
setTitle("Web Link Crawler For Inspecting Broken Link");
FlowLayout flowLayout = new FlowLayout();
flowLayout.setAlignment(FlowLayout.LEFT);
getContentPane().setLayout(flowLayout);
// Creates a label for displaying a text
lblURL = new JLabel("\r\nENTER URL : ");
lblURL.setLocation(new Point(13, 9));
lblURL.setDoubleBuffered(true);
lblURL.setAlignmentY(Component.BOTTOM_ALIGNMENT);
lblURL.setAlignmentX(Component.RIGHT_ALIGNMENT);
lblURL.setVerticalAlignment(SwingConstants.TOP);
lblURL.setForeground(Color.WHITE);
lblURL.setFont(new Font("Tw Cen MT Condensed", Font.BOLD, 20));
getContentPane().add(lblURL);
// Creates text field for URL input
inputSearch = new JTextField();
inputSearch.setText("http://");
inputSearch.setPreferredSize(new Dimension(400, 32));
inputSearch.setFont(new Font("SansSerif", Font.BOLD, 17));
getContentPane().add(inputSearch);
// Creates a search button
btnSearch = new JButton(" Search ");
btnSearch.setPreferredSize(new Dimension(100, 32));
btnSearch.setFont(new Font("SansSerif", Font.BOLD, 13));
getContentPane().add(btnSearch);
// Adds the results text area to a scroll-able pane
JScrollPane scrollOutput = new JScrollPane (JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_NEVER);
scrollOutput.setBackground(Color.GRAY);
scrollOutput.setPreferredSize(new Dimension(1100, 670));
getContentPane().add(scrollOutput);
outputLinks = new JEditorPane();
outputLinks.addHyperlinkListener(new HyperlinkListener() {
public void hyperlinkUpdate(HyperlinkEvent e) {
if (HyperlinkEvent.EventType.ACTIVATED.equals(e.getEventType())) {
System.out.println(e.getURL());
Desktop desktop = Desktop.getDesktop();
try {
desktop.browse(e.getURL().toURI());
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
});
outputLinks.setText("RESULT");
outputLinks.setContentType("text/html");
outputLinks.setEditable(true);
scrollOutput.setColumnHeaderView(outputLinks);
outputLinks.setEditable(false);
// Add event handler to search button click
HandleEvents theEventHandler = new HandleEvents();
inputSearch.addActionListener(theEventHandler);
btnSearch.addActionListener(theEventHandler);
}
private class HandleEvents implements ActionListener {
public void actionPerformed(ActionEvent event) { // Called when elements are triggered
// Preparing output variable string
String strOutput = "No results were found!";
if (event.getSource() == btnSearch || event.getSource() == inputSearch) {
if (!inputSearch.getText().equals(""))
strOutput = crawlURL(inputSearch.getText());
else
strOutput = "Please enter URL to crawl it's hyperlinks";
}
// Prints out the results
outputLinks.setText(strOutput);
}
public String pullURL(String strUrl) {
String resutls = "";
URLConnection connection = null;
try {
connection = new URL(strUrl).openConnection();
#SuppressWarnings("resource")
Scanner scanner = new Scanner(connection.getInputStream());
scanner.useDelimiter("\\Z");
if(scanner.hasNext())
resutls = scanner.next();
} catch ( Exception ex ) {
ex.printStackTrace();
}
return resutls;
}
public String crawlURL(String strUrl) {
String results = ""; // For return
String protocol = "http://";
// Assigns the input to the inURL variable and checks to add http
String inURL = strUrl;
if (!inURL.toLowerCase().contains("http://".toLowerCase()) &&
!inURL.toLowerCase().contains("https://".toLowerCase())) {
inURL = protocol + inURL;
}
// Pulls URL contents from the web
String contectURL = pullURL(inURL);
if (contectURL == "") { // If it fails, then try with https
protocol = "https://";
inURL = protocol + inURL.split("http://")[1];
contectURL = pullURL(inURL);
}
// Declares some variables to be used inside loop
String aTagAttr = "";
String href = "";
String msg = "";
// Finds A tag and stores its href value into output var
String bodyTag = contectURL.split("<body")[1]; // Find 1st <body>
String[] aTags = bodyTag.split(">"); // Splits on every tag
//To show link different from one another
int index = 0;
for (String s: aTags) {
// Process only if it is A tag and contains href
if (s.toLowerCase().contains("<a") && s.toLowerCase().contains("href")) {
aTagAttr = s.split("href")[1]; // Split on href
// Split on space if it contains it
if (aTagAttr.toLowerCase().contains("\\s"))
aTagAttr = aTagAttr.split("\\s")[2];
// Splits on the link and deals with " or ' quotes
href = aTagAttr.split(
((aTagAttr.toLowerCase().contains("\""))? "\"" : "\'")
)[1];
if (!results.toLowerCase().contains(href))
//results += "~~~ " + href + "\r\n";
/*
* Last touches to URl before display
* Adds http(s):// if not exist
* Adds base url if not exist
*/
if(results.toLowerCase().indexOf("about") != -1) {
//Contains 'about'
}
if (!href.toLowerCase().contains("http://") &&
!href.toLowerCase().contains("https://")) {
// http:// + baseURL + href
if (!href.toLowerCase().contains(inURL.split("://")[1]))
href = protocol + inURL.split("://")[1] + href;
else
href = protocol + href;
}
System.out.println(href);//debug
try {
msg = URLheker(href);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// Store the link in output var
if (!results.toLowerCase().contains(href)){
results += "<a href=\"";
results += href;
results += "\">";
results += "Link" + (index + 1)+ " : "+ href ;
results += "</a>";
results += " : ";
results += msg;
results += "<br>";
index++;
}
}
}
System.out.println(results);
return results;
}
}
public String URLheker(String href) throws Exception {
String msg = "";
int code = 0;
URL url = new URL(href);
URLConnection connection = url.openConnection();
if(connection instanceof HttpURLConnection) {
HttpURLConnection httpconn=(HttpURLConnection)connection;
code = httpconn.getResponseCode();
msg = httpconn.getResponseMessage();
if(code == HttpURLConnection.HTTP_OK )
System.out.println("Return normal response :"+msg);
else
System.out.println(code);
}
msg = msg+" [" + Integer.toString(code) + "]";
return msg;
}
}
I'm not sure if it resolves your problem, but you can check response code before getting input stream from connection:
public String pullURL(String strUrl) {
String resutls = "";
HttpURLConnection connection = null;
try {
connection = (HttpURLConnection)new URL(strUrl).openConnection();
//connection.getResponseCode() <- CHECK YOUR RESPONSE CODE
#SuppressWarnings("resource") Scanner scanner = new Scanner(connection.getInputStream());
scanner.useDelimiter("\\Z");
if (scanner.hasNext())
resutls = scanner.next();
} catch (Exception ex) {
ex.printStackTrace();
}
return resutls;
}
Firstly, yes I'm calling this from a web browser. It's quite a long piece of code but I've tried shortening it as much as possible.
Basically, I need to wait let's say 1 second for every iteration in the loop. Tried pretty much everything (.sleep() etc.) but it just doesn't seem to be pausing. The reason why I need to do this is because the SimpleSocketClient is calling a socket which has a low limit per second allowed.
#Override
public String execute(HttpServletRequest request, HttpServletResponse response) {
String forwardToJsp = null;
HttpSession session = request.getSession();
String allUrls = request.getParameter("domains");
ArrayList domainList = new ArrayList<String>();
Scanner sc = new Scanner(allUrls);
while (sc.hasNextLine()) {
String line = sc.nextLine();
domainList.add(line);
// process the line
}
sc.close();
String pageHtml = null;
String domain = "";
String status = "";
String registrant = "";
String dates = "";
String tag = "";
String email = "";
ArrayList domains = new ArrayList<Domain>();
Domain theDomain;
String ipAddress = request.getHeader("X-FORWARDED-FOR");
if (ipAddress == null) {
ipAddress = request.getRemoteAddr();
}
for (int i = 0; i < domainList.size(); i++) {
//NEED TO WAIT 1 SECOND HERE / ANYWHERE IN LOOP
String singleDomain = domainList.get(i).toString();
SimpleSocketClient tester = new SimpleSocketClient(singleDomain,ipAddress);
pageHtml = tester.getResult();
try {
String whoIs2 = ipAddress + " " + ipAddress + " " + singleDomain + "\r\n";
byte[] data = whoIs2.getBytes();
//details of each domain
//domain name
domain = singleDomain;
//status
status = "FTR";
//registrant
registrant = "N/A";
//dates
dates = "N/A";
//tag
tag = "N/A";
//email
email = "N/A";
}
} catch (Exception e) {
Logger.getLogger("ip is " + ipAddress + bulkWhoIsCommand.class.getName()).log(Level.SEVERE, null, e);
forwardToJsp = "index.jsp";
return forwardToJsp;
}
//single one
theDomain = new Domain(domain,status,registrant,dates,tag,email);
//now add to arrayList
domains.add(theDomain);
// try {
// Thread.sleep(230000);
// } catch (InterruptedException ex) {
// Logger.getLogger(bulkWhoIsCommand.class.getName()).log(Level.SEVERE, null, ex);
// }
// try {
// pause.poll(100 * 300, TimeUnit.MILLISECONDS);
// } catch (InterruptedException ex) {
// Logger.getLogger(bulkWhoIsCommand.class.getName()).log(Level.SEVERE, null, ex);
// }
}
EDIT - Friend recommended to use ajax to poll updates but surely there's a way of just using java.
Your can try to set a while-loop in the while-loop, to pause it. Should like this:
while(!done)
{
long start = System.currentTimeMillis();
while(System.currentTimeMillis() - start < 1000L){}
}
Didn't test it but the approach counts. I had the idea to do a combination of both. So every time Thread.Sleep() crashes, you have to take the loop. Something like this:
while(!done)
{
long start = System.currentTimeMillis();
try {
Thread.sleep(1000);
} catch (InterruptedException ex) {
System.err.println(e);
}
while(System.currentTimeMillis() - start < 1000L){}
}
When Thread.Sleep() worked it just get called once. Otherwise you need some CPU time. Could be the cpu economical version.
I've been trying to use the Forecast.io API and the JAR that was provided by their website for my application. But when making web API calls it looks like the data that is being returned by the site isn't fully downloaded.
I try it print the data and it appears that it is not all the information.
I'm using this code:
HttpClient client = new DefaultHttpClient();
URI website = new URI(requestURL);
HttpGet request = new HttpGet();
request.setURI(website);
HttpResponse response = client.execute(request);
scanner = new BufferedReader(new inputStreamReader(response.getEntity()
.getContent()));
String availalbe;
while ((availalbe = scanner.readLine()) != null) {
res += availalbe;
}
Data that is printed out (it stops partway through what is expected):
{"latitude":51.7589177,"longitude":-0.2342903,"timezone":"Europe/London","offset":1,"currently":{"time":1370612854,"summary":"Partly Cloudy","icon":"partly-cloudy-day","precipIntensity":0,"temperature":20.65,"dewPoint":10.56,"windSpeed":9.92,"windBearing":59,"cloudCover":0.34,"humidity":0.5,"pressure":1023.91,"visibility":10.75,"ozone":356.06},"minutely":{"summary":"Light rain in 30 min.","icon":"rain","data":[{"time":1370612820,"precipIntensity":0},{"time":1370612880,"precipIntensity":0},{"time":1370612940,"precipIntensity":0},{"time":1370613000,"precipIntensity":0},{"time":1370613060,"precipIntensity":0},{"time":1370613120,"precipIntensity":0},{"time":1370613180,"precipIntensity":0},{"time":1370613240,"precipIntensity":0},{"time":1370613300,"precipIntensity":0},{"time":1370613360,"precipIntensity":0},{"time":1370613420,"precipIntensity":0},{"time":1370613480,"precipIntensity":0},{"time":1370613540,"precipIntensity":0},{"time":1370613600,"precipIntensity":0},{"time":1370613660,"precipIntensity":0.107,"precipIntensityError":0.055,"precipProbability":0.01,"precipType":"rain"},{"time":1370613720,"precipIntensity":0.111,"precipIntensityError":0.057,"precipProbability":0.01,"precipType":"rain"},{"time":1370613780,"precipIntensity":0.132,"precipIntensityError":0.065,"precipProbability":0.01,"precipType":"rain"},{"time":1370613840,"precipIntensity":0.137,"precipIntensityError":0.062,"precipProbability":0.03,"precipType":"rain"},{"time":1370613900,"precipIntensity":0.142,"precipIntensityError":0.065,"precipProbability":0.03,"precipType":"rain"},{"time":1370613960,"precipIntensity":0.161,"precipIntensityError":0.072,"precipProbability":0.04,"precipType":"rain"},{"time":1370614020,"precipIntensity":0.174,"precipIntensityError":0.074,"precipProbability":0.04,"precipType":"rain"},{"time":1370614080,"precipIntensity":0.187,"precipIntensityError":0.077,"precipProbability":0.08,"precipType":"rain"},{"time":1370614140,"precipIntensity":0.207,"precipIntensityError":0.084,"precipProbability":0.09,"precipType":"rain"},{"time":1370614200,"precipIntensity":0.223,"precipIntensityError":0.088,"precipProbability":0.1,"precipType":"rain"},{"time":1370614260,"precipIntensity":0.224,"precipIntensityError":0.094,"precipProbability":0.15,"precipType":"rain"},{"time":1370614320,"precipIntensity":0.243,"precipIntensityError":0.102,"precipProbability":0.16,"precipType":"rain"},{"time":1370614380,"precipIntensity":0.259,"precipIntensityError":0.108,"precipProbability":0.17,"precipType":"rain"},{"time":1370614440,"precipIntensity":0.262,"precipIntensityError":0.108,"precipProbability":0.24,"precipType":"rain"},{"time":1370614500,"precipIntensity":0.28,"precipIntensityError":0.115,"precipProbability":0.25,"precipType":"rain"},{"time":1370614560,"precipIntensity":0.3,"precipIntensityError":0.12,"precipProbability":0.25,"precipType":"rain"},{"time":1370614620,"precipIntensity":0.322,"precipIntensityError":0.125,"precipProbability":0.26,"precipType":"rain"},{"time":1370614680,"precipIntensity":0.33,"precipIntensityError":0.125,"precipProbability":0.33,"precipType":"rain"},{"time":1370614740,"precipIntensity":0.352,"precipIntensityError":0.131,"precipProbability":0.34,"precipType":"rain"},{"time":1370614800,"precipIntensity":0.375,"precipIntensityError":0.136,"precipProbability":0.34,"precipType":"rain"},{"time":1370614860,"precipIntensity":0.38,"precipIntensityError":0.14,"precipProbability":0.42,"precipType":"rain"},{"time":1370614920,"precipIntensity":0.402,"precipIntensityError":0.147,"precipProbability":0.42,"precipType":"rain"},{"time":1370614980,"precipIntensity":0.425,"precipIntensityError":0.154,"precipProbability":0.42,"precipType":"rain"},{"time":1370615040,"precipIntensity":0.432,"precipIntensityError":0.157,"precipProbability":0.5,"precipType":"rain"},{"time":1370615100,"precipIntensity":0.454,"precipIntensityError":0.164,"precipProbability":0.5,"precipType":"rain"},{"time":1370615160,"precipIntensity":0.477,"precipIntensityError":0.168,"precipProbability":0.5,"precipType":"rain"},{"time":1370615220,"precipIntensit
Method calling the Forecast Api test class
public void weatherLike()
{
StrictMode.enableDefaults();
MyLocation myLocation = new MyLocation();
myLocation.getLocation(MyService.this, new LocationResult() {
ForecastIO fio = null;
#Override
public void gotLocation(Location location) {
try {
double latitude = location.getLatitude();
double longitude = location.getLongitude();
fio= new ForecastIO("[API_KEY]");
} catch (Exception e) {
speakOut(user + ", I am not able to locate you");
e.printStackTrace();
}finally
{
System.out.println("Latitude: "+fio.getLatitude());
System.out.println("Longitude: "+fio.getLongitude());
System.out.println("Timezone: "+fio.getTimezone());
System.out.println("Offset: "+fio.offsetValue());
System.out.println("\n");
}
}
});
}
Thanks to all the Answers. i found out that My LogCat was trucating long messages no wonder why i never showed a full reply.
in case any one fell in the same problem.
Split the String reply to pieces using this code
if (sb.length() > 4000) {
Log.v("length", "sb.length = " + sb.length());
int chunkCount = sb.length() / 4000; // integer division
for (int i = 0; i <= chunkCount; i++) {
int max = 4000 * (i + 1);
if (max >= sb.length()) {
Log.v("1st", "chunk " + i + " of " + chunkCount + ":" + sb.substring(4000 * i));
} else {
Log.v("2nd", "chunk " + i + " of " + chunkCount + ":" + sb.substring(4000 * i, max));
}
}
}
BufferedReader use 'buffer' so U have to use 'flush method' (or close method )
refer link : http://docs.oracle.com/javase/tutorial/essential/io/buffers.html
try this
public static String getResponseText(String stringUrl) throws IOException
{
StringBuilder response = new StringBuilder();
System.out.println("webservice 1");
URL url = new URL(stringUrl);
HttpURLConnection httpconn = (HttpURLConnection)url.openConnection();
if (httpconn.getResponseCode() == HttpURLConnection.HTTP_OK)
{
System.out.println("webservice 2");
BufferedReader input = new BufferedReader(new InputStreamReader(httpconn.getInputStream()),8192);
String strLine = null;
while ((strLine = input.readLine()) != null)
{
response.append(strLine);
}
input.close();
}
return response.toString();
}
I tried to request a large amount of data from freebase. But I got error message "HTTP response code: 403". Did anyone have similar problem before?
Here is my code
private static final String service_url = "https://www.googleapis.com/freebase/v1/mqlread";
public static void main(String[] args)
{
try
{
String cursor = "";
String urlStr_initial = service_url + "?query=" + URLEncoder.encode(getQuery(), "UTF-8") + "&cursor";
URL url = new URL(urlStr_initial);
List<Freebase> list = new ArrayList<Freebase>();
Freebase f_b;
do
{
HttpURLConnection url_con = (HttpURLConnection)url.openConnection();
url_con.addRequestProperty("User-Agent", "Mozilla/4.76");
StringBuilder str_builder = new StringBuilder();
BufferedReader reader = new BufferedReader(new InputStreamReader(url_con.getInputStream()));
String line;
while((line = reader.readLine()) != null)
{
str_builder.append(line);
}
String response = str_builder.toString();
JSONObject j_object = new JSONObject(response);
if(j_object.has("result"))
{
JSONArray j_array = j_object.getJSONArray("result");
for(int i = 0; i < j_array.length(); i++)
{
JSONObject j_o = j_array.getJSONObject(i);
if(j_o.has("id") == true && j_o.has("name"))
{
String id = j_o.getString("id");
String name = j_o.getString("name");
System.out.println("ID: " + id + " / Name:" + name);
f_b = new Freebase(id, name);
if(f_b != null)
{
list.add(f_b);
}
else
{
System.out.println("Null value in Freebase");
}
}
}
}
else
{
System.out.println("There is no \"result\" key in JASON object");
}
if(j_object.has("cursor"))
{
cursor = j_object.get("cursor").toString();
}
else
{
cursor = "false";
}
String urlStr = urlStr_initial + "=" + cursor;
url = new URL(urlStr);
}while( !cursor.equalsIgnoreCase("false"));
if(list != null)
{
TextFile tf = new TextFile();
tf.writeToFile(list);
}
}
catch(Exception e)
{
e.printStackTrace();
}
}
private static String getQuery()
{
return "[{"
+ "\"name\": null,"
+ "\"id\": null,"
+ "\"type\":\"/people/person\","
+ "\"limit\": 500"
+ "}]";
}
You don't say what "large" is, but the API isn't designed for bulk downloads. You should be using the data dumps for that.
There's usually more detailed error message included with the HTTP response code. If, for example, it says 403 - Forbidden - API key required, it means you didn't include your API key (I don't see where you include it in your code). If it says 403 - Forbidden - quota exceeded it means you've exceeded your request quota (you can look on the API console to see how much quota you have remaining).
I am wondering how to find the maven repository close to my home?
Actually, the listing at http://docs.codehaus.org/display/MAVENUSER/Mirrors+Repositories is problematic. Is Central in San Francisco, California or in St. Louis, Missouri? (I guess this is an inadvertent lesson on why code, or even xml, should not be commented). To make things more confusing, the owner of the domain (according to http://whois.net/whois/maven.org) is in Fulton, MD.
Maybe the question was not "Where are the repositories?" (which esaj answered), but really:
"How can I find the maven repository closest to my home?"
That question bugged me, too.
On the other hand, does it really matter, given how fast packets travel across the world?
But it still bugged me.
There are two ways to find out which host is the closest:
Open a browser, go to http://freegeoip.net, and type in the hostname. If you know where you are, then you can go to GoogleMaps, get directions, and check the distance.
Open a terminal window, and ping the hostname to find the average round trip transit time in milliseconds. This will give you the electronic distance, which is really more important for file transfer times.
Repeat for each hostname (i.e. about 40 times, if you're looking at the download sites for Maven itself)
Sort by the distance measured by the GoogleMaps directions, or by the transit time.
For 40 hosts you could probably do that manually in 20 tedious minutes, but a true professional would rather spend a few hours writing the Selenium and java code that could do it in 5 minutes.
In my copious amounts of spare time, I threw together only half the job (i.e. no Selenium):
public class ClosestUrlFinder {
/**
* Find out where a hostname is.
* Adapted from http://www.mkyong.com/java/how-to-send-http-request-getpost-in-java/
* Send HTTP GET requests to: freegeoip.net/{format}/{ip_or_hostname}
* The API supports both HTTP and HTTPS.
* Supported formats are csv, xml or json.
* E.G. http://freegeoip.net/xml/www.poolsaboveground.com
* returns <Response><Ip>207.58.184.93</Ip><CountryCode>US</CountryCode>
* <CountryName>United States</CountryName><RegionCode>VA</RegionCode><RegionName>Virginia</RegionName>
* <City>Mclean</City><ZipCode>22101</ZipCode>
* <Latitude>38.9358</Latitude><Longitude>-77.1621</Longitude>
* <MetroCode>511</MetroCode><AreaCode>703</AreaCode></Response>
* #param urlOrHostname
* #return
*/
public String getUrlLocation(String urlOrHostname) {
String USER_AGENT = "Mozilla/5.0";
BufferedReader in = null;
URL urlObject;
StringBuilder response = null;
try {
urlObject = new URL("http://freegeoip.net/xml/" + urlOrHostname);
HttpURLConnection connection = (HttpURLConnection) urlObject.openConnection();
connection.setRequestMethod("GET");
connection.setRequestProperty("User-Agent", USER_AGENT);
int responseCode = connection.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + urlOrHostname);
System.out.println("Response Code : " + responseCode);
in = new BufferedReader( new InputStreamReader(connection.getInputStream()));
String inputLine;
response = new StringBuilder();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (in != null) {
try { in.close(); }
catch (IOException e) {
e.printStackTrace();
}
}
}
//System.out.println(response.toString());
return response.toString();
}
/*
* Fixed version of code from http://stackoverflow.com/questions/11506321/java-code-to-ping-an-ip-address
*/
public String runDosCommand(List<String> command)
{
String string = "", result = "";
ProcessBuilder pBuilder = new ProcessBuilder(command);
Process process;
try {
process = pBuilder.start();
BufferedReader stdInput = new BufferedReader(new InputStreamReader(process.getInputStream()));
BufferedReader stdError = new BufferedReader(new InputStreamReader(process.getErrorStream()));
//System.out.println("Standard output of the command:");
while ((string = stdInput.readLine()) != null)
{
System.out.println(string);
result = result + "\n" + string;
}
while ((string = stdError.readLine()) != null)
{
System.out.println("stdError: "+ string);
result = result + "\nstdError: " + string;
}
} catch (IOException e) {
System.out.println("Error with command "+command);
e.printStackTrace();
}
return result;
}
public String pingUrl(String url) {
List<String> command = new ArrayList<String>();
String averagePingTime = "0", geoText = "", loss = "";
command.add("ping");
command.add("-n");
command.add("2");
url = url.replace("ftp://", "");
url = Utils.first(url.replace("/", " "));
url = Utils.first(url.replace(":", " "));
command.add(url);
System.out.println("command is: "+command);
String pingResult = runDosCommand(command);
String timeoutString = "Request timed out";
String timeout = Utils.grep(timeoutString, pingResult);
String noHostString = "Ping request could not find host";
String noHost = Utils.grep(noHostString, pingResult);
String unreachableString = "Destination net unreachable";
String unreachable = Utils.grep(unreachableString, pingResult);
if (Utils.isNullOrEmptyString(timeout) && Utils.isNullOrEmptyString(noHost)
&& Utils.isNullOrEmptyString(unreachable)) {
String lostString = "Lost =";
loss = Utils.grep(lostString, pingResult);
int index = loss.indexOf(lostString);
loss = loss.substring(index);
if (!loss.equals("Lost = 0 (0% loss),")) {
System.out.println("Non-zero loss for " + url);
averagePingTime = "0";
} else {
String replyString = "Reply from";
String replyFrom = Utils.grep(replyString, pingResult);
String ipAddress = Utils.nth(replyFrom, 3);
System.out.println("reply Ip="+ipAddress.replace(":", ""));
String averageString = "Average =";
averagePingTime = Utils.grep(averageString, pingResult);
index = averagePingTime.indexOf(averageString);
averagePingTime = averagePingTime.substring(index);
averagePingTime = Utils.last(averagePingTime).replace("ms", "");
String xml = getUrlLocation(url);
//System.out.println("xml="+xml);
geoText = extractTextFromXML(xml);
System.out.println("geoText="+geoText);
}
} else {
averagePingTime = "0";
System.out.println("Error. Either could not find host, Request timed out, or unreachable network for " + url);;
}
System.out.println("Results for " + url + " are: " + loss + " " + averagePingTime + " miliseconds.");
return url + " " + averagePingTime + " " + geoText ;
}
public ArrayList<Entry<String,Integer>> pingManyUrls() {
ArrayList<Entry<String,Integer>> mirrorTimeList = new ArrayList<>();
String[] urls = Utils.readTextFile("resources/MavenUrls.txt").split("\\s+");
System.out.println("Start pingManyUrls with "+urls.length + " urls.");
for (String url : urls) {
url = url.trim();
System.out.println("************************************\npingManyUrls: Check Url " + url);
if (arrayListContainsKey(url, mirrorTimeList)) {
System.out.println("Key " + url + " already in array.");
} else {
String pingInfo = pingUrl(url);
String averagePingString = Utils.nth(pingInfo, 2);
if (!averagePingString.equals("0")) {
int averagePingMilliseconds = Integer.parseInt(averagePingString);
pingInfo = Utils.rest(pingInfo); //chop the first term (the url)
pingInfo = Utils.rest(pingInfo); // chop the 2nd term (the time)
url = url + " " + pingInfo;
System.out.println(" Adding Key " + url + " " + averagePingMilliseconds + " to array.");
Entry<String,Integer> urlTimePair = new java.util.AbstractMap.SimpleEntry<>(url, averagePingMilliseconds);
mirrorTimeList.add(urlTimePair);
} else { System.out.println("Url " + url + " has a problem and therefore will be not be included."); }
}
}
Collections.sort(mirrorTimeList, new Comparator<Entry<String,Integer>>() {
#Override
public int compare (Entry<String,Integer> pair1, Entry<String,Integer> pair2) {
return pair1.getValue().compareTo(pair2.getValue());
}
});
return mirrorTimeList;
}
public boolean arrayListContainsKey(String key, ArrayList<Entry<String,Integer>> arrayList) {
for (Entry<String,Integer> keyValuePair : arrayList) {
//System.out.println(keyValuePair.getKey() + " =? " + key);
if (key.equalsIgnoreCase(keyValuePair.getKey())) { return true; }
}
return false;
}
public String extractFirstTagContents(Element parentElement, String tagname) {
NodeList list = parentElement.getElementsByTagName(tagname);
if (list != null && list.getLength()>0) {
String contents = list.item(0).getTextContent();
System.out.println("Tagname=" + tagname + " contents="+contents);
return contents;
}
return "";
}
public String extractTextFromXML(String xml) {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder;
String content = "";
try {
builder = factory.newDocumentBuilder();
Document document = builder.parse(new InputSource(new StringReader(xml)));
Element rootElement = document.getDocumentElement();
content = rootElement.getTextContent();
//System.out.println("content="+content);
// String city = extractFirstTagContents(rootElement, "City");
// String state = extractFirstTagContents(rootElement, "RegionName");
// String zipCode = extractFirstTagContents(rootElement, "ZipCode");
// String country = extractFirstTagContents(rootElement, "CountryName");
// String latitude = extractFirstTagContents(rootElement, "Latitude");
// String longitude = extractFirstTagContents(rootElement, "Longitude");
} catch (ParserConfigurationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (SAXException saxex) {
// TODO Auto-generated catch block
saxex.printStackTrace();
} catch (IOException ioex) {
// TODO Auto-generated catch block
ioex.printStackTrace();
}
return content;
}
public static void main(String[] args) {
System.out.println("Starting ClosestUrlFinder");
ClosestUrlFinder ping = new ClosestUrlFinder();
ArrayList<Entry<String,Integer>> mirrorList = ping.pingManyUrls();
System.out.println("Final Results, sorted by trip travel time (out of "+mirrorList.size()+" successful pings).");
for (Entry<String,Integer> urlTime : mirrorList) {
System.out.println(urlTime.getKey() + " " + urlTime.getValue());
}
}
} // End of Class
Where the data in MavenUrls.txt is:
apache.claz.org
apache.cs.utah.edu
apache.mesi.com.ar
apache.mirrors.hoobly.com
apache.mirrors.lucidnetworks.net
apache.mirrors.pair.com
apache.mirrors.tds.net
apache.osuosl.org
apache.petsads.us
apache.spinellicreations.com
apache.tradebit.com
download.nextag.com
ftp.osuosl.org
ftp://mirror.reverse.net/pub/apache/
mirror.cc.columbia.edu/pub/software/apache/
mirror.cogentco.com/pub/apache/
mirror.metrocast.net/apache/
mirror.nexcess.net/apache/
mirror.olnevhost.net/pub/apache/
mirror.reverse.net/pub/apache/
mirror.sdunix.com/apache/
mirror.symnds.com/software/Apache/
mirror.tcpdiag.net/apache/
mirrors.gigenet.com/apache/
mirrors.ibiblio.org/apache/
mirrors.sonic.net/apache/
psg.mtu.edu/pub/apache/
supergsego.com/apache/
www.bizdirusa.com
www.bizdirusa.com/mirrors/apache/
www.carfab.com/apachesoftware/
www.dsgnwrld.com/am/
www.eng.lsu.edu/mirrors/apache/
www.gtlib.gatech.edu/pub/apache/
www.interior-dsgn.com/apache/
www.motorlogy.com/apache/
www.picotopia.org/apache/
www.poolsaboveground.com/apache/
www.trieuvan.com/apache/
www.webhostingjams.com/mirror/apache/
And my Utils class includes the following:
/**
* Given a string of words separated by spaces, returns the first word.
* #param string
* #return
*/
public static String first(String string) {
if (isNullOrEmptyString(string)) { return ""; }
string = string.trim();
int index = string.indexOf(" "); //TODO: shouldn't this be "\\s+" to handle double spaces and tabs? That means regexp version of indexOf
if (index<0) { return string; }
return string.substring(0, index);
}
/**
* Given a string of words separated by spaces, returns the rest of the string after the first word.
* #param string
* #return
*/
public static String rest(String string) {
if (isNullOrEmptyString(string)) { return ""; }
string = string.trim();
int index = string.indexOf(" "); //TODO: shouldn't this be "\\s+" to handle double spaces and tabs? That means regexp version of indexOf
if (index<0) { return ""; }
return string.substring(index+1);
}
public static String grep(String regexp, String multiLineStringToSearch) {
String result = "";
//System.out.println("grep for '"+ regexp + "'");
String[] lines = multiLineStringToSearch.split("\\n");
//System.out.println("grep input string contains "+ lines.length + " lines.");
Pattern pattern = Pattern.compile(regexp);
for (String line : lines) {
Matcher matcher = pattern.matcher(line);
if (matcher.find()) {
//System.out.println("grep found match="+ line);
result = result + "\n" + line;
}
}
return result.trim();
}
/**
* Given the path and name of a plain text (ascii) file,
* reads it and returns the contents as a string.
* #param filePath
* #return
*/
public static String readTextFile(String filePath) {
BufferedReader reader;
String result = "";
int counter = 0;
try {
reader = new BufferedReader(new FileReader(filePath));
String line = "";
StringBuilder stringBuilder = new StringBuilder();
String lineSeparator = System.getProperty("line.separator");
while ((line = reader.readLine()) != null) {
counter = counter + 1;
stringBuilder.append(line);
stringBuilder.append(lineSeparator);
}
reader.close();
result = stringBuilder.toString();
logger.info("readTextFile: Read "+filePath+" with "+ counter + " lines, "+result.length()+" characters.");
return result;
} catch (FileNotFoundException e) {
logger.fatal("readTextFile: Could not find file "+filePath+".");
e.printStackTrace();
} catch (IOException e) {
logger.fatal("readTextFile: Could not read file "+filePath+".");
e.printStackTrace();
}
return "";
}
public static boolean isNullOrEmptyString(String s) {
return (s==null || s.equals(""));
}
/**
* Given a string of words separated by one or more whitespaces, returns the Nth word.
* #param string
* #return
*/
public static String nth(String string, int n) {
string = string.trim();
String[] splitstring = string.split("\\s+");
String result = "";
if (splitstring.length<n) { return ""; }
else {
result = splitstring[n - 1];
}
return result.trim();
}
/**
* Given a string of words separated by spaces, returns the last word.
* #param string
* #return
*/
public static String last(String string) {
return string.substring(string.trim().lastIndexOf(" ")+1, string.trim().length());
}
Enjoy!
Here's there official central repository mirror listing (as xml-file) and here's another list of central repository mirrors.