I'm a rookie test developer using selenium 2.45 and I'm trying to configure my FirefoxDriver to use my company's proxy settings. I am failing to do so :)
I am following the instruction from here to create a profile on-the-fly:
Using a Proxy for FF
My code looks like this:
public static WebDriver driver;
String usedProxy = "http://myproxy:8080";
org.openqa.selenium.Proxy proxy = new org.openqa.selenium.Proxy();
proxy.setHttpProxy(usedProxy).setFtpProxy(usedProxy).setSslProxy(usedProxy);
DesiredCapabilities cap = new DesiredCapabilities();
cap.setCapability(CapabilityType.PROXY, proxy);
driver = new FirefoxDriver(cap);
driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
driver.get("https://TestWebsite.com");
I am not receiving any kind of errors but the connection is not working for this browser. When checking Options>Advanced>Network>Connection Settings from Firefox menu, proxy is set to manual but text inputs only contain "http://"
PS: I have a feeling this is relevant but am not sure: TestWebsite.com will load via https only (it's a shopping cart)
If your goal is to test your functionality on different IP addresses, you can use the Tor Browser.
public IWebDriver Driver { get; set; }
public Process TorProcess { get; set; }
public WebDriverWait Wait { get; set; }
[TestInitialize]
public void SetupTest()
{
String torBinaryPath = #"C:\Users\aangelov\Desktop\Tor Browser\Browser\firefox.exe";
this.TorProcess = new Process();
this.TorProcess.StartInfo.FileName = torBinaryPath;
this.TorProcess.StartInfo.Arguments = "-n";
this.TorProcess.StartInfo.WindowStyle = ProcessWindowStyle.Maximized;
this.TorProcess.Start();
FirefoxProfile profile = new FirefoxProfile();
profile.SetPreference("network.proxy.type", 1);
profile.SetPreference("network.proxy.socks", "127.0.0.1");
profile.SetPreference("network.proxy.socks_port", 9150);
this.Driver = new FirefoxDriver(profile);
this.Wait = new WebDriverWait(this.Driver, TimeSpan.FromSeconds(60));
}
[TestCleanup]
public void TeardownTest()
{
this.Driver.Quit();
this.TorProcess.Kill();
}
Here is the code to refresh the Tor identity.
public void RefreshTorIdentity()
{
Socket server = null;
try
{
IPEndPoint ip = new IPEndPoint(IPAddress.Parse("127.0.0.1"), 9151);
server = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
server.Connect(ip);
server.Send(Encoding.ASCII.GetBytes("AUTHENTICATE \"johnsmith\"" + Environment.NewLine));
byte[] data = new byte[1024];
int receivedDataLength = server.Receive(data);
string stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength);
server.Send(Encoding.ASCII.GetBytes("SIGNAL NEWNYM" + Environment.NewLine));
data = new byte[1024];
receivedDataLength = server.Receive(data);
stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength);
if (!stringData.Contains("250"))
{
Console.WriteLine("Unable to signal new user to server.");
server.Shutdown(SocketShutdown.Both);
server.Close();
}
}
finally
{
server.Close();
}
}
You can find more detailed information here: http://automatetheplanet.com/using-selenium-webdriver-tor-c-code/
The code examples are in C# but the code should be identical in Java.
Try firefox profile as well. Note this is C# code and converting to Java should be fairly simply
string usedProxy = "http://myproxy:8080";
Proxy proxy = new OpenQA.Selenium.Proxy();
proxy.HttpProxy = usedProxy;
proxy.FtpProxy = usedProxy;
proxy.SslProxy = usedProxy;
FirefoxProfile profile = new FirefoxProfile();
profile.SetProxyPreferences(proxy);
This code works for me:
FirefoxOptions options = new FirefoxOptions();
FirefoxProfile profile = new FirefoxProfile();
profile.setPreference("network.proxy.type", 1);
profile.setPreference("network.proxy.http", <proxy_url>);
profile.setPreference("network.proxy.http_port", 8080);
profile.setPreference("network.proxy.ssl", <proxy_url>);
profile.setPreference("network.proxy.ssl_port", 8080);
options.setProfile(profile);
options.setAcceptInsecureCerts(true)
options.setCapability("disable-restore-session-state", true);
options.setCapability("marionette", true);
WebDriver driver = new FirefoxDriver(options);
I followed answers from this link: Webdriver and proxy server for firefox
Related
I want to automate the web application on all browsers.My code is running fine for chrome,firefox but when i tried it on Edge,it showing the ssl certificate error.
how i can handle this.tried so many workarounds but failed to achieve it.
if((browser.equalsIgnoreCase("Edge"))){
//browserName = "";
browser = "Edge";
//set path to Edge.exe
System.setProperty("webdriver.edge.driver","C:\\edgedriver.exe");
if(enableProxy == true) {
proxy = new ProxyServer();
proxy.setTrustAllServers(true);
proxy.start();
System.out.println( proxy.getPort());
Proxy seleniumProxy = new Proxy();
EdgeOptions options = new EdgeOptions();
String hostIp = Inet4Address.getLocalHost().getHostAddress();
seleniumProxy.setHttpProxy(hostIp + ":" + proxy.getPort());
seleniumProxy.setSslProxy(hostIp + ":" + proxy.getPort());
seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
options.setProxy(seleniumProxy);
options.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true);
// DesiredCapabilities capabilities = new DesiredCapabilities();
// capabilities.setCapability(CapabilityType.PROXY, proxy);
proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
proxy.newHar();
driver = new EdgeDriver(EdgeDriverService.createDefaultService(),options);
//driver = new EdgeDriver(capabilities);
driver.manage().window().maximize();
}else {
//create Edge instance
driver = new EdgeDriver();
driver.manage().window().maximize();
}
}
You can set ACCEPT_SSL_CERTS to true in DesiredCapabilities:
DesiredCapabilities capabilities = new DesiredCapabilities();
capabilities.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true);
System.setProperty("webdriver.edge.driver", "C://EdgeDriver.exe");
WebDriver driver = new EdgeDriver(capabilities);
public class Test {
public static void main(String[] args) throws Exception{
String driverGoogleChromePath = "D:\\Project Doc\\Jar\\selenium\\chromedriver_win32\\chromedriver.exe";
String refererurl = "http://www.iteye.com/";
String hostAndPort = "112.91.159.66:3128";
int port = 3128;
//String host = "112.91.159.66";
System.setProperty("webdriver.chrome.driver",driverGoogleChromePath);
//start the proxy
BrowserMobProxy proxy = new BrowserMobProxyServer();
Map<String, String> headers = new HashMap<String, String>();
headers.put("Referer",refererurl);
proxy.addHeaders(headers);
proxy.start(port);
Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
seleniumProxy.setHttpProxy(hostAndPort)
.setSslProxy(hostAndPort)
.setFtpProxy(hostAndPort);
DesiredCapabilities capabilities = new DesiredCapabilities();
capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
ChromeOptions option = new ChromeOptions().merge(capabilities);
WebDriver driver = new ChromeDriver(option);
Thread.sleep(3000);
driver.get("http://www.java1234.com/");
proxy.stop();
//driver.close();
}
This is my java code, I have set the referer to use the .addHeaders, but when I run it, the browser didn't show the referer, why, how to do show this ?
selenium: selenium-server-standalone-3.141.5.jar
browser: browsermob-dist-2.1.4.jar
enter image description here
I am writing automation test and to capture the network call made in background, I am using browsermob-proxy.
In browsermob-proxy, I want to set cookie before making requests. How can i do it?
Below is my code:-
String strFilePath = "data.har";
// start the proxy
ProxyServer server = new ProxyServer(4444);
server.start();
server.setCaptureHeaders(true);
server.setCaptureContent(true);
// get the Selenium proxy object
Proxy proxy = server.seleniumProxy();
FirefoxProfile profile = new FirefoxProfile();
String userAgent = "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 MobileWeb/8H7 Safari/6533.18.5";
profile.setPreference("general.useragent.override", userAgent);
// configure it as a desired capability
DesiredCapabilities capabilities = DesiredCapabilities.firefox();
capabilities.setCapability(CapabilityType.PROXY, proxy);
capabilities.setCapability(FirefoxDriver.PROFILE, profile);
// start the browser up
WebDriver driver = new FirefoxDriver(capabilities);
final String[] remoteHost = {null};
final String[] analytics = {null};
final String[] fetchAdjs = {null};
server.addRequestInterceptor(new RequestInterceptor()
{
int googleCount = 0;
int adjs = 0;
#Override
public void process(BrowserMobHttpRequest browserMobHttpRequest, Har har)
{
remoteHost[0] = browserMobHttpRequest.getProxyRequest().getRemoteHost();
String request = browserMobHttpRequest.getProxyRequest().getRequestURL().toString();
if (request.matches(".*google.*"))
googleCount = googleCount + 1;
if (request.matches(".*test.*"))
adjs = adjs + 1;
analytics[0] = String.valueOf(googleCount);
fetchAdjs[0] = String.valueOf(adjs);
// System.out.println(browserMobHttpRequest.getMethod().getAllHeaders()[1]); // user agent
System.out.println(browserMobHttpRequest.getProxyRequest());
}
});
// create a new HAR with the label "apple.com"
server.newHar("assertselenium.com");
// open yahoo.com
driver.get("http://test.com");
Thread.sleep(3000);
Thread.sleep(3000);
driver.get("http://test.com/316782/content/fDxL4zzv");
Thread.sleep(3000);
// get the HAR data
Har har = server.getHar();
FileOutputStream fos = new FileOutputStream(strFilePath);
// view har file here --> http://pcapperf.appspot.com/
har.writeTo(fos);
server.stop();
driver.quit();
Using a request filter is the right approach. However, I would highly recommend building the latest version of BrowserMob Proxy with LittleProxy integration. Its filters are much more reliable and easier to use. See the github page for information on building and using the latest version. The LittleProxy interceptors section will be particularly relevant.
Here's a simple example of adding a cookie to every request using the new request filters:
BrowserMobProxy proxy = new BrowserMobProxyServer();
proxy.start();
proxy.addRequestFilter(new RequestFilter() {
#Override
public HttpResponse filterRequest(HttpRequest request, HttpMessageContents contents, HttpRequest originalRequest) {
request.headers().add("Cookie", "added-cookie=added-value");
return null;
}
});
So I am looking to do some simple data collection on a few .onion sites. I am going about this by using selenium webdriver to call Tor as part of the Firefox webdriver. However, I can't seem to figure out how to get firefox to successfully go to .onion sites. Here is the code.
public static void main(String[] args) throws InterruptedException, IOException {
File torProfileDir = new File("C:\\Users\\Chambers\\Desktop\\Tor Browser\\Browser\\TorBrowser\\Data\\Browser\\profile.default");
FirefoxBinary binary = new FirefoxBinary(new File("C:\\Users\\Chambers\\Desktop\\Tor Browser\\Browser\\firefox.exe"));
FirefoxProfile torProfile = new FirefoxProfile(torProfileDir);
torProfile.setPreference("webdriver.load.strategy", "unstable");
try {
binary.startProfile(torProfile, torProfileDir, "");
} catch (IOException e) {
e.printStackTrace();
}
FirefoxProfile profile = new FirefoxProfile();
profile.setPreference("network.proxy.type", 1);
profile.setPreference("network.proxy.socks", "127.0.0.1");
profile.setPreference("network.proxy.socks_port", 9150);
FirefoxDriver driver = new FirefoxDriver(profile);
int firstCheck = "0";
while (firstCheck == 0) {
driver.navigate().to("onion site here");
......
The problem I am having is that I end up with a firefox browser that cannot connect to .onion sites. If I change FirefoxDriver driver = new FirefoxDriver(profile); to FirefoxDriver driver = new FirefoxDriver(binary, profile); then I am left with a blank Tor window that I can't seem to control with the webdriver.
Anyone have any ideas on how to fix this? any help would be appreciated!
Figured it out. Needed to add a lot of permissions to the new profile. Here is the fixed code for those that are interested. It allows you to browse the dark web with Firefox as if you were using Tor. The commands for controlling the webdriver don't change.
File torProfileDir = new File("C:\\Users\\Chambers\\Desktop\\Tor Browser\\Browser\\TorBrowser\\Data\\Browser\\profile.default");
FirefoxBinary binary = new FirefoxBinary(new File("C:\\Users\\Chambers\\Desktop\\Tor Browser\\Browser\\firefox.exe"));
FirefoxProfile torProfile = new FirefoxProfile(torProfileDir);
torProfile.setPreference("webdriver.load.strategy", "unstable");
try {
binary.startProfile(torProfile, torProfileDir, "");
} catch (IOException e) {
e.printStackTrace();
}
FirefoxProfile profile = new FirefoxProfile();
profile.setPreference("network.proxy.type", 1);
profile.setPreference("network.proxy.socks", "127.0.0.1");
profile.setPreference("network.proxy.socks_port", 9150);
profile.setPreference("network.proxy.socks_version", 5);
profile.setPreference("places.history.enabled", false);
profile.setPreference("privacy.clearOnShutdown.offlineApps", true);
profile.setPreference("privacy.clearOnShutdown.passwords", true);
profile.setPreference("privacy.clearOnShutdown.siteSettings", true);
profile.setPreference("privacy.sanitize.sanitizeOnShutdown", true);
profile.setPreference("signon.rememberSignons", false);
profile.setPreference("network.cookie.lifetimePolicy", 2);
profile.setPreference("network.dns.disablePrefetch", true);
profile.setPreference("network.http.sendRefererHeader", 0);
profile.setPreference("network.proxy.socks_remote_dns", true);
FirefoxDriver driver = new FirefoxDriver(profile);
String firstCheck = "";
while (firstCheck == 0) {
driver.get("http://kbhpodhnfxl3clb4.onion/");
.........................
public class Test_One
{
public static void main(String[] args) throws Exception {
ProxyServer server = new ProxyServer(8105);
server.start();
server.setCaptureHeaders(true);
server.setCaptureContent(true);
server.newHar("test");
DesiredCapabilities capabilities = new DesiredCapabilities();
Proxy proxy = server.seleniumProxy();
FirefoxProfile profile = new FirefoxProfile();
profile.setAcceptUntrustedCertificates(true);
profile.setAssumeUntrustedCertificateIssuer(true);
profile.setPreference("network.proxy.http", "localhost");
profile.setPreference("network.proxy.http_port", 8105);
profile.setPreference("network.proxy.ssl", "localhost");
profile.setPreference("network.proxy.ssl_port", 8105);
profile.setPreference("network.proxy.type", 1);
profile.setPreference("network.proxy.no_proxies_on", "");
//profile.setProxyPreferences(proxy);
profile.setPreference(key, value)
capabilities.setCapability(FirefoxDriver.PROFILE,profile);
capabilities.setCapability(CapabilityType.PROXY, proxy);
WebDriver driver = new FirefoxDriver(capabilities);
driver.get("http://www.google.com");
Har har1 = server.getHar();
}
}
I'm new to selenium and broswermob. This is my code. Whe I try to execute this and getting error The method setProxyPreferences(Proxy) is undefined for the type FirefoxProfile. How to solve this?
You (generally) don't need to configure the FirefoxProfile settings manually. The Using With Selenium section of the readme file gives an example of how to use the proxy with Selenium:
// start the proxy
ProxyServer server = new ProxyServer(4444);
server.start();
// get the Selenium proxy object
Proxy proxy = server.seleniumProxy();
// configure it as a desired capability
DesiredCapabilities capabilities = new DesiredCapabilities();
capabilities.setCapability(CapabilityType.PROXY, proxy);
// start the browser up
WebDriver driver = new FirefoxDriver(capabilities);
// create a new HAR with the label "yahoo.com"
server.newHar("yahoo.com");
// open yahoo.com
driver.get("http://yahoo.com");
// get the HAR data
Har har = server.getHar();
If that doesn't work, it may be a problem with your specific version of Firefox and/or Selenium and/or BrowserMob Proxy. What versions are you using? A stack trace with the exact error message would also help.
Try removing profile from capabilities and passing it directly to FirefoxDriver:
driver = new FirefoxDriver(new FirefoxBinary(),profile,capabilities);