I have to display a webpage inside my Java software.( you can think it is one the page of a Wireless Device configuration)
But when Im gonna go to that page in browser,browser displays a user and password popup and I enter the user and password and go to that page.( notice that username is always root, but password can be different)
Now I am gonna display the page from Java software, I could link the page and open the page via Java, But the host inside the wireless device displays :
401 Unauthorized, The URL of the page I want to display is http://192.168.1.2/Wireless
I used Fiddler to monitor its behavior, that URL is continuously reloading.
And here is the Header of the Get method of that :
GET /Wireless HTTP/1.1
Host: 192.168.1.2
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0
Accept: */*
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Referer: http://192.168.1.2/Wireless
Authorization: Basic xxxxxxxxx
Connection: keep-alive
How can I do that? With HttpClient ?
Does anybody knows how to do that? And give me a solution?
Or a sample or anything can help me?|
Thanks.
EDIT :
Here is my code to display the webview :
myFrame.setSize(mainJTabbed.getSize());
myFrame.setLocationRelativeTo(mainJTabbed);
myFrame.setVisible(true);
myFrame.add(myFXPanel);
Platform.runLater(() -> {
BorderPane borderPane = new BorderPane();
WebView webComponent = new WebView();
webComponent.getEngine().load("myURL");
borderPane.setCenter(webComponent);
Scene scene = new Scene(borderPane,450,450);
myFXPanel.setScene(scene);
});
And even it is a GET method, the user and password does not post via URL.
I did it in this way, I created a class with a constructor, the all is in constructor, notice that my constructor what I wanted, you can change it, but the algorithm is this :
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.Authenticator;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.PasswordAuthentication;
import java.net.URL;
import javafx.application.Platform;
import javafx.embed.swing.JFXPanel;
import javafx.scene.Scene;
import javafx.scene.layout.BorderPane;
import javafx.scene.web.WebView;
import javax.swing.JFrame;
public class WebViewSample {
public WebViewSample(String urlString, JFrame myFrame, JFXPanel myFXPanel) {
try {
// Sets the authenticator that will be used by the networking code
// when a proxy or an HTTP server asks for authentication.
Authenticator.setDefault(new CustomAuthenticator());
URL url = new URL("http://" +urlString + "/wireless");
Platform.runLater(() -> {
BorderPane borderPane = new BorderPane();
WebView webComponent = new WebView();
webComponent.getEngine().load(url.toString());
borderPane.setCenter(webComponent);
Scene scene = new Scene(borderPane,450,450);
myFXPanel.setScene(scene);
});
}
catch (MalformedURLException e) {
System.out.println("Malformed URL: " + e.getMessage());
}
catch (IOException e) {
System.out.println("I/O Error: " + e.getMessage());
}
}
public static class CustomAuthenticator extends Authenticator {
// Called when password authorization is needed
protected PasswordAuthentication getPasswordAuthentication() {
// Get information about the request
String prompt = getRequestingPrompt();
String hostname = getRequestingHost();
InetAddress ipaddr = getRequestingSite();
int port = getRequestingPort();
String username = "myUserName";
String password = "myPassword";
// Return the information (a data holder that is used by Authenticator)
return new PasswordAuthentication(username, password.toCharArray());
}
}
}
Related
So I'm trying to read a JSON file in from a website (fortniteapi.com), every time I try to download the file to my local computer it does not download. I've been at this for about a week and I just can't figure out why it won't work.
also i'm using Gson
Here is my code so far:
package sample;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import javafx.application.Application;
import javafx.fxml.FXMLLoader;
import javafx.scene.Parent;
import javafx.scene.Scene;
import javafx.stage.Stage;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
public class Main extends Application {
#Override
public void start(Stage primaryStage) throws Exception{
Parent root = FXMLLoader.load(getClass().getResource("sample.fxml"));
primaryStage.setTitle("Fortnite");
primaryStage.setScene(new Scene(root, 300, 275));
primaryStage.show();
ReadJson();
}
public static void main(String[] args) {
launch(args);
}
public void ReadJson()
{
try {
// read url
String sURL = "https://fortnite-public-api.theapinetwork.com/prod09/users/id?username=Ninja"; //just a string
// Connect to the URL using java's native library
URL url = new URL(sURL);
URLConnection request = url.openConnection();
request.connect();
// Convert to a JSON object
JsonParser jp = new JsonParser(); //from gson
JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent())); //Convert the input stream to a json element
JsonObject rootobj = root.getAsJsonObject();
String output = rootobj.get("username").getAsString(); //just grab the username value
// print out the result/output
System.out.println(output);
} catch (IOException e) {
System.out.println("Unexpected Error.");
// JOptionPane.showMessageDialog(null, "Oh no something went wrong.", "Unexpected Error", JOptionPane.ERROR_MESSAGE);
System.exit(1);
}
}
}
The error
After reading the errorStream() of the request (after casting it to HttpURLConnection) HTML is printed and states:
Access denied | fortnite-public-api.theapinetwork.com used Cloudflare
to restrict access
and
The owner of this website (fortnite-public-api.theapinetwork.com) has
banned your access based on your browser's signature
(mybrowsersignature).
What does this mean
Cloudflare states that that error means that:
the domain owner is blocking this request based on the client's web
browser signature.
and that the feature is called "Browser Integrity Check", from there we can find What does the Browser Integrity Check do?:
Cloudflare's Browser Integrity Check (BIC) is similar to Bad Behavior
and looks for common HTTP headers abused most commonly by spammers and
denies access to your page. It will also challenge visitors that do
not have a user agent or a non standard user agent (also commonly used
by abuse bots, crawlers or visitors).
Solution
We can change the User-Agent of request to something that should be valid before request.connect(); like so (user agent copied from User-Agent | MDN):
request.setRequestProperty("User-Agent",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0");
The expected output is printed:
Ninja
Ok so I wrote a piece of code testing ability of my java to connect to internet. It is supposed to fetch html from www.google.com and display the contents in a JFrame's JTextArea object.
Here's the code, so you can have clear picture:
import java.awt.Color;
import java.awt.Dimension;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import javax.swing.JFrame;
import javax.swing.JTextArea;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
public class JSoupFetchTest extends JFrame{
String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:37.0) Gecko/20100101 Firefox/37.0";
boolean jsoupcond = true;
String address = "http://www.google.com";
JTextArea text;
public JSoupFetchTest(){
text = new JTextArea();
text.setPreferredSize(new Dimension(500, 500));
text.setBackground(Color.BLACK);
text.setForeground(Color.WHITE);
text.setVisible(true);
text.setLineWrap(true);
this.add(text);
this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
this.setVisible(true);
this.pack();
gogo();
}
private void gogo() {
if(jsoupcond){
text.setText(text.getText() +"\nstart...");
try {
text.setText(text.getText() +"\nConnecting to " +address+ "...");
Document doc = Jsoup.connect(address).userAgent(userAgent).get();
text.setText(text.getText() +"\nConverting page document into text");
String s = doc.toString();
text.setText(text.getText() +"\nText: \n" +s);
System.out.println();
} catch (Exception e) {
text.setText(text.getText() +"\n" +e.toString());
e.printStackTrace();
}
text.setText(text.getText() +"\nEnd.");
}
String html = downloadHtml(address);
text.setText(text.getText() +"\nDownloading HTML...");
text.setText(text.getText() +"\nHTML:");
text.setText(text.getText() +"\n" +html);
}
private String downloadHtml(String path) {
text.setText(text.getText() +"\ndownloadHtml entry point...");
InputStream is = null;
try {
text.setText(text.getText() +"\ntry block entered...");
String result = "";
String line;
URL url = new URL(path);
text.setText(text.getText() +"\nabout to open url stream...");
is = url.openStream(); // throws an IOException
text.setText(text.getText() +"\nurl stream opened...");
BufferedReader br = new BufferedReader(new InputStreamReader(is));
text.setText(text.getText() +"\nstarting to read lines...");
while ((line = br.readLine()) != null) {
result += line;
}
text.setText(text.getText() +"\nreading lines finished...");
return result;
} catch (IOException ioe) {
ioe.printStackTrace();
} finally {
try {
if (is != null) is.close();
} catch (IOException ioe) { }
}
return "";
}
public static void main(String[] args) {
new JSoupFetchTest();
}
}
I should also add that:
1. My eclipse (cause that's what I'm using) can't connect to marketplace nor can't fetch updates.
2. Eclipse's web browser works fine.
3. My system's browser (Mozilla Firefox) connects fine
4. I exported JSoupFetchTest into a runnable jar and tried to run it from system's level, with no effect
5. I am running Windows 7 Professional MSDN version
6. I contacted eclipse support and they concluded it is not eclipse's fault and suggested that I'm behind a proxy.
7. I contacted my ISP to see if I indeed am and they said I am not.
8. I changed my JAVA's network settings so now it connects "directly". Before the setting was "use browser settings" and it didn't work either.
9. My eclipse's Window -> Preferences -> General -> Network Connections active provider is set to "Native", I also tried "Direct"
10. Method downloadHtml(String path) stops at "is = url.openStream();" and goes on forever...
The exception I get from JSoup is:
java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:150)
at java.net.SocketInputStream.read(SocketInputStream.java:121)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:703)
at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:647)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1534)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1439)
at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480)
at org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:453)
at org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:434)
at org.jsoup.helper.HttpConnection.execute(HttpConnection.java:181)
at org.jsoup.helper.HttpConnection.get(HttpConnection.java:170)
at JSoupFetchTest.gogo(JSoupFetchTest.java:42)
at JSoupFetchTest.<init>(JSoupFetchTest.java:32)
at JSoupFetchTest.main(JSoupFetchTest.java:92)
I also tried to set JSoup.connect's timeout to infinity. Then it goes on forever.
Before you guys say that my question is a duplicate, or delegate me to other, external possible solutions to my problem, believe me - either the question is mine or I was there - I browse internet in search for solution for weeks now and I feel like pulling my hair out...
Please help if you can cause it prevents me from installing stuff in my eclipse and from developing anything else than stand alone apps...
You need a socket number after the URL -- "http:/www.google.com:80" works. JSoup likely uses defaults for that, but opening the URL as a stream in Java does not.
The following program works for me. So Java and JSoup are working. It has to be some sort of local configuration problem with your network. Check your firewall, routers, gateway, and Java permissions. Do a clean rebuild of your project. Etc. Comment out lines until it does work and then put the lines back one at a time until you find the problem. Etc.
package stuff;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.net.URLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
public class SocketTest
{
public static void main( String[] args ) throws Exception
{
URL url = new URL( "http://www.google.com" );
URLConnection sock = url.openConnection();
InputStream ins = sock.getInputStream();
BufferedReader reader = new BufferedReader( new InputStreamReader(ins, "UTF-8" ) );
for( String line; (line = reader.readLine()) != null; ) {
System.out.println( line );
}
ins.close();
Document doc = Jsoup.connect( "http://www.google.com" ).get();
System.out.println( doc.toString() );
String userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:37.0) Gecko/20100101 Firefox/37.0";
Document doc2 = Jsoup.connect( "http://www.google.com" ).userAgent(userAgent).get();
System.out.println( doc2.toString() );
}
}
I am trying to downloada picture from a certain url, but cant do so because I somehow have to give the right userclient to the website.I am sure the problem is that I cant give the user client while using the Url class, because the page can be accesed via browser. I tried using proxy and Urlconnection but couldnt get it to work. Please share your toughts on the matter!
My code is the following:
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.SocketAddress;
import java.net.URL;
import java.net.URLConnection;
import javax.imageio.ImageIO;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.highgui.Highgui;
public class KepLetolto {
public static void main(String[] args) throws IOException {
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
File file = new File("tempjpeg");
SocketAddress address = new java.net.InetSocketAddress("xyz.com", 8080);
// Create an HTTP Proxy using the above SocketAddress.
Proxy proxy = new Proxy(Proxy.Type.HTTP, address);
URL url_kep =new URL("http://www.theouthousers.com/images/templates/thumbnails/128058/bayfinger_size3.png");
ImageIO.write(ImageIO.read(url_kep), "jpeg", file);
Mat uj = Highgui.imread("temp.jpeg" ,Highgui.CV_LOAD_IMAGE_COLOR);
}
}
Instead of using ImageIO.read(URL), which limits you to the default behavior of the URL's underlying URLConnection, use ImageIO.read(InputStream).
This allows you to use any HTTP client library - including the basic HttpURLConnection, which you can get from (HttpURLConnection)url_kep.openConnection(). Using that, you can set headers such as User-Agent, if that's the header required by the site, or other headers such as Referer which are sometimes used to prevent deep-linking.
Once you set up all the headers and any other request options, you can get an InputStream from the client object, and pass that to ImageIO.
This Solution Worked For Me:
URLConnection openConnection = new URL("YOUR_IMAGE_URL").openConnection();
openConnection.addRequestProperty("User-Agent", "YOUR USER AGENT");
InputStream is = openConnection.getInputStream();
BufferedImage saveImage = ImageIO.read(is);
ImageIO.write(saveImage, "png", new File("PATH\\TO\\IMAGE\\FILE.PNG"));
I am trying to come up with a Android app that needs some information on the university inner website. I have been trying to use Jsoup to login the website programmatically. Here is the code I have now:
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Jsoup;
//import org.jsoup.helper.Validate;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
//import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.Map;
public class Test {
public static void main(String[] args) {
Document doc;
try {
Connection.Response res = Jsoup
.connect(
"https://sso.bris.ac.uk/sso/login?service=https%3A%2F%2Fwww.cs.bris.ac.uk%2FTeaching%2Fsecure%2Funit-list.jsp%3Flist%3Dmine")
.execute();
Map<String, String> cookies = res.cookies();
System.out.println(cookies.keySet());
Document fakepage = res.parse();
Element fakelt = fakepage.select("input[name=lt]").get(0);
Element fakeexecution = fakepage.select("input[name=execution]")
.get(0);
Element fake_eventID = fakepage.select("input[name=_eventId]").get(
0);
System.out.println("Hello World!");
System.out.println(fakelt.attr("value"));
System.out.println(fakeexecution.toString());
System.out.println(fake_eventID.toString());
// System.out.println(cookies.get("JSESSIONID"));
String url="https://sso.bris.ac.uk/sso/login?service=https%3A%2F%2Fwww.cs.bris.ac.uk%2FTeaching%2Fsecure%2Funit-list.jsp%3Flist%3Dmine";
System.out.println(url);
Connection newreq = Jsoup
.connect(url)
.cookies(cookies).data("lt", fakelt.attr("value")).followRedirects(true).header("Connection", "keep-alive")
.header("Refer", " https://sso.bris.ac.uk/sso/login?service=https%3A%2F%2Fwww.cs.bris.ac.uk%2FTeaching%2Fsecure%2Funit-list.jsp%3Flist%3Dmine")
.header("Content-Type","application/x-www-form-urlencoded;charset=UTF-8")
.userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0")
.data("lt",fakelt.attr("value"))
.data("execution", fakeexecution.attr("value"))
.data("_eventID", fake_eventID.attr("value"))
.data("username", "aabbcc").data("password", "ddeeff")
.data("submit", "").method(Method.POST);
Connection.Response newres = newreq.execute();
doc = newres.parse();
System.out.println(doc.toString());
System.out.println(newres.statusCode());
Map<String,String> newcookies = newres.cookies();
doc = Jsoup.connect("https://www.cs.bris.ac.uk/Teaching/secure/unit-list.jsp?list=mine").cookies(newcookies).get();
System.out.println(doc.toString());
// System.out.println(doc.toString());
} catch (IOException e) {
System.out.println("Excepiton:");
System.out.println(e.getMessage());
}
}
}
I completely faked a form to submit use Jsoup, and to get around the security cookies I first request the website once and then use the cookies it sent me to request the website again. The form has some hidden fields so I use the ones I got on my first request to fake it when I request it again. However this does not work. Is it possible to do it or the server has some advanced preventer against me doing so?
Do not use Jsoup to do this, it needs you to handle all the cookies yourself, instead, use Httpclient, if you use something from 4.0 onward it handle the cookies automatically. Much eaiser to work with.
I am trying to implement OAuth 2.0 implicit grant from java desktop client application using SkyDrive REST API. I use the following code:
Desktop.getDesktop().browse(new URL(st.toString()).toURI());
JOptionPane.showMessageDialog(null, "Press ok to continue once you have authenticated.");
my code opens web browser and ask users to signin and then SkyDrive sends access token to the browser url in the following form:
https://login.live.com/oauth20_desktop.srf?lc=1033#access_token=EwAwAq1DBAAUlbRW.....
What I want to do is to read this access token from my java program.
I tried to read httpconnection from console:
HttpURLConnection con = (HttpURLConnection) url.openConnection();
BufferedReader reader = new BufferedReader( new InputStreamReader(url.openStream()));
while(reader.readLine()!=null){
System.out.println(reader.readLine());
but it seems that java httpurlconnection does not handle javascript response. It replies:
<html dir="..... Windows Live ID requires JavaScript to sign in. This web browser either does not support JavaScript, or scripts are being blocked......<body onload="evt_LoginHostMobile_onload(event);">
So, Is there any way to retrieve the access token directly from java?
I had the same problem. After hours of brainstorming, I have finally found a solution. I use the JavaFX library to create a WebView. Then you can intercept location change.
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import javafx.application.Application;
import javafx.event.EventHandler;
import javafx.scene.Scene;
import javafx.scene.layout.BorderPane;
import javafx.scene.web.WebEngine;
import javafx.scene.web.WebEvent;
import javafx.scene.web.WebView;
import javafx.stage.Stage;
public class Authenticate extends Application {
static final String APP_ID = "...";
static final String REDIRECT_URL = "https://login.live.com/oauth20_desktop.srf";
static final String RESPONSE_TYPE = "token";
static final String SCOPE = "wl.signin%20wl.offline_access";
private Scene scene;
#Override
public void start(final Stage stage) throws Exception {
final String url = "https://login.live.com/oauth20_authorize.srf?client_id="+APP_ID
+"&scope="+SCOPE+"&response_type="+RESPONSE_TYPE+"&oauth_callback=oob&redirect_uri="+REDIRECT_URL;
BorderPane borderPane = new BorderPane();
WebView browser = new WebView();
WebEngine webEngine = browser.getEngine();
webEngine.load(url);
borderPane.setCenter(browser);
webEngine.setOnStatusChanged(new EventHandler<WebEvent<String>>() {
public void handle(WebEvent<String> event) {
if (event.getSource() instanceof WebEngine) {
WebEngine we = (WebEngine) event.getSource();
String location = we.getLocation();
if (location.startsWith(REDIRECT_URL) && location.contains("access_token")) {
try {
URL url = new URL(location);
String[] params = url.getRef().split("&");
Map<String, String> map = new HashMap<String, String>();
for (String param : params) {
String name = param.split("=")[0];
String value = param.split("=")[1];
map.put(name, value);
}
System.out.println("The access token: "+map.get("access_token"));
stage.hide();
} catch (MalformedURLException e) {
e.printStackTrace();
}
}
}
}
});
// create scene
stage.setTitle("Skydrive");
scene = new Scene(borderPane, 750, 500);
stage.setScene(scene);
stage.show();
}
public static void main(String[] args) {
launch(args);
}
}