We Keep Coding

Hello Analytics API: Java quickstart errors

I'm trying to access data from the Google analytics reporting API using Java.
I was following the "Hello Analytics API: Java quickstart for installed applications" tutorial, and i did everything it tells you, and i get following issues:
com.google.api.client.util.store.FileDataStoreFactory setPermissionsToOwnerOnly
WARNING: unable to change permissions for everybody: C:\Users\<user>\.store\hello_analytics
com.google.api.client.util.store.FileDataStoreFactory setPermissionsToOwnerOnly
WARNING: unable to change permissions for owner: C:\Users\timst\.store\hello_analytics
java.lang.NullPointerException
at java.io.Reader.<init>(Reader.java:78)
at java.io.InputStreamReader.<init>(InputStreamReader.java:72)
at com.example.demo.HelloAnalytics.initializeAnalytics(HelloAnalytics.java:60)
at com.example.demo.HelloAnalytics.main(HelloAnalytics.java:44)
I tried using the full path for the client_secret.json.
tried using different methods i found online, but none seem to work.
After getting frustrated by this error i tried the "Hello Analytics API: Java quickstart for service accounts" tutorial.
But here i have the issue that i can't add users to the account, property or view for the accounts i can access.
I have access to other peoples analytics accounts and I can only remove myself from the accounts.
All code I'm using is from the tutorials, using Intellij and gradle.
tl;dr; All I want to do is access the analytics data for all my
accounts, using the reporting API so i can put all this data in my own
database and use this database for my other projects.
the Tutorials google provides doesn't work for me. (the data is mostly Google Adwords data.)

So the Warning is not the problem, it's a known issue with it not working properly on windows.
The java.lang.NullPointerException is because the profile I call to has no rows of data for the given metric. so the return value of the call doesn't have a .getRows() methode because there isn't a row value.
you should check for the row's first,
GaData results;
if (null != results) {
if(results.get("rows") != null){
if (!results.getRows().isEmpty()){
//do something with the rows exp.
for (List<String> row : results.getRows()) {
for (int i=0; i<results.getColumnHeaders().size();i++) {
List<GaData.ColumnHeaders> headers = results.getColumnHeaders();
System.out.println( headers.get(i).getName()+": " + row.get(i));
}
}
}
}
}
In the example I also use the ColumnHeaders, wich you should also check first.
It was also easier to check every single account i had access to and every webProperty and Profile and not just the first value of each of those.
Also, the query explorer is really useful. you should use it to check out which metrics you can use and which dimensions.
Here is my full HelloAnalytics class i just print everything that might be usefull to the console i also use multiple metrics and a dimension from Google AdWords in the getResults methode:
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver;
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.gson.GsonFactory;
import com.google.api.client.util.store.FileDataStoreFactory;
import com.google.api.services.analytics.Analytics;
import com.google.api.services.analytics.AnalyticsScopes;
import com.google.api.services.analytics.model.*;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* A simple example of how to access the Google Analytics API.
*/
public class HelloAnalytics {
// Path to client_secrets.json file downloaded from the Developer's Console.
// The path is relative to HelloAnalytics.java.
private static final String CLIENT_SECRET_JSON_RESOURCE = "/client_secret.json";
// The directory where the user's credentials will be stored.
private static final File DATA_STORE_DIR = new File("out/DataStore/hello_analytics");
private static final File OUTPUT_FILE = new File("out/DataStore/output.text");
private static final String APPLICATION_NAME = "Online Marketing Buddy";
private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance();
private static NetHttpTransport httpTransport;
private static FileDataStoreFactory dataStoreFactory;
public static void main(String[] args) {
try {
Analytics analytics = initializeAnalytics();
getProfileIds(analytics);
} catch (Exception e) {
e.printStackTrace();
}
}
private static Analytics initializeAnalytics() throws Exception {
httpTransport = GoogleNetHttpTransport.newTrustedTransport();
dataStoreFactory = new FileDataStoreFactory(DATA_STORE_DIR);
// Load client secrets.
InputStream in =
HelloAnalytics.class.getResourceAsStream(CLIENT_SECRET_JSON_RESOURCE);
GoogleClientSecrets clientSecrets =
GoogleClientSecrets.load(JSON_FACTORY, new InputStreamReader(in));
// Set up authorization code flow for all auth scopes.
GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow
.Builder(httpTransport, JSON_FACTORY, clientSecrets,AnalyticsScopes.all())
.setDataStoreFactory(dataStoreFactory)
.build();
// Authorize.
Credential credential = new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver())
.authorize("user");
// Construct the Analytics service object.
Analytics response = new Analytics
.Builder(httpTransport, JSON_FACTORY, credential)
.setApplicationName(APPLICATION_NAME).build();
return response;
}
private static void getProfileIds(Analytics analytics) throws IOException {
// Get the all view (profile) IDs for the authorized user.
List<String> profileIds = new ArrayList<>();
// Query for the list of all accounts associated with the service account.
Accounts accounts = analytics.management().accounts().list().execute();
if (accounts.getItems().isEmpty()) {
System.err.println("No accounts found");
} else {
for (Account account : accounts.getItems()) {
System.out.println("account: " + account.getName());
String accountId = account.getId();
// Query for the list of properties associated with the each account.
Webproperties properties = analytics.management().webproperties()
.list(accountId).execute();
if (properties.getItems().isEmpty()) {
System.err.println("No properties found for accountId: " + accountId);
} else {
for (Webproperty webproperty : properties.getItems()) {
System.out.println("\nwebproperty: " + webproperty.getName());
String webpropertyId = webproperty.getId();
// Query for the list views (profiles) associated with the property.
Profiles profiles = analytics.management().profiles()
.list(accountId, webpropertyId).execute();
if (profiles.getItems().isEmpty()) {
System.err.println("No views (profiles) found for accoundId: " + accountId + "and webpropertyId: " + webpropertyId);
} else {
// Return the first (view) profile associated with the property.
for (Profile profile : profiles.getItems()) {
System.out.println("\nprofileId added for profile: " + profile.getName());
profileIds.add(profile.getId());
printResults(getResults(analytics,profile.getId()), profile.getId());
}
}
System.out.println("---------- ---------- end webproperty: " + webproperty.getName() + "---------- ----------");
}
}
System.out.println("---------- ---------- end account: " + account.getName() + "---------- ----------");
}
}
}
private static GaData getResults(Analytics analytics, String profileId) throws IOException {
// Query the Core Reporting API for the number of sessions
// in the past 30 days.
GaData data = analytics.data().ga()
.get("ga:" + profileId, "30daysAgo", "yesterday", "ga:adClicks, ga:adCost, ga:transactions, ga:transactionRevenue, ga:users, ga:sessions")
.setDimensions("ga:adwordsCampaignID")
.execute();
return data;
}
private static void printResults(GaData results, String profile) {
// Parse the response from the Core Reporting API for
// the profile name and number of sessions.
if (null != results) {
System.out.println("View (Profile: " + profile + ") Name: "
+ results.getProfileInfo().getProfileName() + "\n");
if (results.get("rows") != null && results.get("columnHeaders") != null) {
if (!results.getRows().isEmpty() && !results.getColumnHeaders().isEmpty()) {
for (List<String> row : results.getRows()) {
for (int i=0; i<results.getColumnHeaders().size();i++) {
List<GaData.ColumnHeaders> headers = results.getColumnHeaders();
System.out.println( headers.get(i).getName()+": " + row.get(i) + "\n");
}
System.out.println("---------- ---------- ----------\n");
}
} else {
System.out.println("No rows or columHeaders empty\n");
}
} else {
System.out.println("No rows or columHeaders\n");
}
}
}
}

how to create custom JDBCrealm?

i'm trying to secure my enterprise web-app!
i have to constraint resources.
Since i have all stored in my db (users and roles), i won't create a fileRealm or store any user's credential in (Glassfish) server. Moreover, i'm using jBCrypt to encrypt users' passwords, so i can't use standard jdbcRealm.
How can i secure my resources?
i'm thinking about custom jdbcRealm, it's the right way? How can i create and use it?
Some existing framework can help me?
Thank you in advance.

I would suggest you to use a framework Apache Shiro. The configuration file is below
[main]
sha256Matcher = org.apache.shiro.authc.credential.HashedCredentialsMatcher
sha256Matcher.hashAlgorithmName = SHA-256
sha256Matcher.hashIterations=1
# base64 encoding
sha256Matcher.storedCredentialsHexEncoded = false
#datasource type
ds = org.apache.shiro.jndi.JndiObjectFactory
#datasourcename
ds.resourceName = cfresource
#datasourcetype
ds.requiredType = javax.sql.DataSource
#configuring jdbc realm
jdbcRealm = com.connectifier.authc.realm.CustomJDBCRealm
jdbcRealm.credentialsMatcher = $sha256Matcher
jdbcRealm.dataSource=$ds
jdbcRealm.userRolesQuery=select name from role where email = ? and isactive=1
jdbcRealm.authenticationQuery=select hash, salt from user where email = ?
jdbcRealm.permissionsLookupEnabled=false
securityManager.realms = $jdbcRealm
#login url
authc.loginUrl = /
#page to redirected to after logout
logout.redirectUrl = /
#page to where to land after login
authc.successUrl = /
#username parameter name in the loginform
authc.usernameParam = username
#password parameter name in the loginform
authc.passwordParam = password
#rememberme parameter name in the loginform
authc.rememberMeParam=rememberme
#cacheManager = org.apache.shiro.cache.ehcache.EhCacheManager
#securityManager.cacheManager = $cacheManager
#jdbcRealm.authenticationCachingEnabled = true
[urls]
# The /login.jsp is not restricted to authenticated users (otherwise no one could log in!), but
# the 'authc' filter must still be specified for it so it can process that url's
# login submissions. It is 'smart' enough to allow those requests through as specified by the
# shiro.loginUrl above.
/* = anon
The CustomJDBCRealm overriding JDBCRealm is below
package com.connectifier.authc.realm;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.shiro.authc.AccountException;
import org.apache.shiro.authc.AuthenticationException;
import org.apache.shiro.authc.AuthenticationInfo;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.SimpleAuthenticationInfo;
import org.apache.shiro.authc.UnknownAccountException;
import org.apache.shiro.authc.UsernamePasswordToken;
import org.apache.shiro.codec.Base64;
import org.apache.shiro.realm.jdbc.JdbcRealm;
import org.apache.shiro.util.ByteSource;
import org.apache.shiro.util.JdbcUtils;
import org.apache.shiro.util.SimpleByteSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* #author kiranchowdhary
*
* Application specific JDBC realm. If required override methods of {#link JdbcRealm} to load users, roles and
* permissions from database.
*
* Do not override configuration in code if it can be done via shiro.ini file.
*/
public class CustomJDBCRealm extends JdbcRealm {
private static final Logger log = LoggerFactory.getLogger(JdbcRealm.class);
public CustomJDBCRealm() {
super();
setSaltStyle(SaltStyle.COLUMN);
}
/**
* overriding the method which is in JdbcRealm. If SaltStyle is COLUMN, then gets String salt value from database
* and forms salt byte array of type {#link ByteSource} with decoded string salt value and sets it to salt value of
* AuthenticationInfo.
*/
protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException {
UsernamePasswordToken upToken = (UsernamePasswordToken) token;
String username = upToken.getUsername();
// Null username is invalid
if (username == null) {
throw new AccountException("Null usernames are not allowed by this realm.");
}
Connection conn = null;
SimpleAuthenticationInfo info = null;
try {
conn = dataSource.getConnection();
String password = null;
String salt = null;
switch (saltStyle) {
case NO_SALT:
case CRYPT:
case EXTERNAL:
return super.doGetAuthenticationInfo(token);
case COLUMN:
String[] queryResults = getPasswordForUser(conn, username);
password = queryResults[0];
salt = queryResults[1];
break;
}
if (password == null) {
throw new UnknownAccountException("No account found for user [" + username + "]");
}
info = new SimpleAuthenticationInfo(username, password.toCharArray(), getName());
if (salt != null) {
info.setCredentialsSalt(new SimpleByteSource(Base64.decode(salt)));
}
} catch (SQLException e) {
final String message = "There was a SQL error while authenticating user [" + username + "]";
if (log.isErrorEnabled()) {
log.error(message, e);
}
// Rethrow any SQL errors as an authentication exception
throw new AuthenticationException(message, e);
} finally {
JdbcUtils.closeConnection(conn);
}
return info;
}
private String[] getPasswordForUser(Connection conn, String username) throws SQLException {
String[] result;
boolean returningSeparatedSalt = false;
switch (saltStyle) {
case NO_SALT:
case CRYPT:
case EXTERNAL:
result = new String[1];
break;
default:
result = new String[2];
returningSeparatedSalt = true;
}
PreparedStatement ps = null;
ResultSet rs = null;
try {
ps = conn.prepareStatement(authenticationQuery);
ps.setString(1, username);
// Execute query
rs = ps.executeQuery();
// Loop over results - although we are only expecting one result,
// since usernames should be unique
boolean foundResult = false;
while (rs.next()) {
// Check to ensure only one row is processed
if (foundResult) {
throw new AuthenticationException("More than one user row found for user [" + username
+ "]. Usernames must be unique.");
}
result[0] = rs.getString(1);
if (returningSeparatedSalt) {
result[1] = rs.getString(2);
}
foundResult = true;
}
} finally {
JdbcUtils.closeResultSet(rs);
JdbcUtils.closeStatement(ps);
}
return result;
}
}

how to retrieve the attribute "unicodePwd" in Active Directory through java programming

First of all, I apology for my bad english. I'm brazilian, so if there is any mistakes at the text, please, just disconsidered.
I read a lot of articles here about retrieving the attribute "unicodePwd" in Active Directory, but none of then actually helped me out.
Well, why do I need that information? I'll explain:
I have here some java routines that unify user information from differents systems one to another.
This routines get the information needed in a main Oracle Database and set the information in another Databases (Oracle and MySQL, basically).
For example: We have a private cloud system, that runs in a CentOS Linux OS, that has it own MySQL Database. To unify the users informations, including the users passwords, we get the information from the main Oracle Database and set do this system's MySQL Database, to unify user details and login information.
All the routines that i have here are working and there's no problems, but now we have a new challenge.
We need to do the same unification with ours Active Directory users, getting the information needed in this main Oracle Database and then setting all the information into Active Directory users, including the users passwords.
I already updated the password succesfully in Active Directory users, but I don't want that the password get updated everytime that this java routine runs, but only when the password changes in the main Oracle Database.
Example: When one of the users change the password in the main Oracle Database, the java routine gets this user information to set then in the same user in Active Directory. To do that properly, the routine gets the same information in Active Diretory, then it compares both passwords (Oracle's password and Active Diretory's password) and finally, if the password is different, the routine will update it, but if the password is not different, the routine will do nothing.
That is why i need to retrieve the attribute "unicodePwd" in Active Directory.
Here is some of my code:
import java.util.Hashtable;
import javax.naming.Context;
import javax.naming.NamingEnumeration;
import javax.naming.NamingException;
import javax.naming.directory.*;
import org.apache.commons.mail.EmailException;
import javax.naming.ldap.InitialLdapContext;
import javax.naming.ldap.LdapContext;
public class ldapQuery {
String distinguishedName = "";
String department = "";
String physicalDeliveryOfficeName = "";
String telephoneNumber = "";
String mobile = "";
String title = "";
String sAMAccountName = "";
String unicodePwd = "";
public ldapQuery(String mail) {
try {
final Hashtable<String, String> env = new Hashtable<String, String>();
final String adminName = "CN=MY DOMAIN ADMIN,CN=MY DOMAIN ADMIN FOLDER LOCALIZATION,DC=MY DOMAIN,DC=MY DOMAIN,DC=MY DOMAIN";
final String adminPasswd = "MY DOMAIN ADMIN PASSWORD";
final String ldapUrl = "ldaps://MY ACTIVE DIRECTORY SERVER:636";
final String factory = "com.sun.jndi.ldap.LdapCtxFactory";
final String authType = "simple";
final String protocol = "ssl";
env.put(Context.INITIAL_CONTEXT_FACTORY, factory);
env.put(Context.SECURITY_AUTHENTICATION, authType);
env.put(Context.SECURITY_PRINCIPAL, adminName);
env.put(Context.SECURITY_CREDENTIALS, adminPasswd);
env.put(Context.SECURITY_PROTOCOL, protocol);
env.put(Context.PROVIDER_URL, ldapUrl);
DirContext ctx = new InitialLdapContext (env,null);
SearchControls searchCtls = new SearchControls();
String returnedAtts[] = {"sAMAccountName", "distinguishedName","department", "physicalDeliveryOfficeName", "telephoneNumber", "mobile", "title", "unicodePwd"};
searchCtls.setReturningAttributes(returnedAtts);
searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE);
String searchFilter = "(&(objectClass=user)(mail=" + mail +"))";
String searchBase = "DC=MY DOMAIN,DC=MY DOMAIN,DC=MY DOMAIN";
int totalResults = 0;
NamingEnumeration<SearchResult> answer =ctx.search(searchBase, searchFilter, searchCtls);
while (answer.hasMoreElements()) {
SearchResult sr = (SearchResult)answer.next();
totalResults++;
Attributes attrs = sr.getAttributes();
if (attrs != null) {
distinguishedName = (String) attrs.get("distinguishedName").get();
department = (String) attrs.get("department").get();
physicalDeliveryOfficeName = (String) attrs.get("physicalDeliveryOfficeName").get();
telephoneNumber = (String) attrs.get("telephoneNumber").get();
mobile = (String) attrs.get("mobile").get();
title = (String) attrs.get("title").get();
sAMAccountName = (String) attrs.get("sAMAccountName").get();
Attribute passwd = attrs.get("unicodePwd");
unicodePwd = unicodePwd + passwd;
if (department == null) {
department = "";
}
if (physicalDeliveryOfficeName == null) {
physicalDeliveryOfficeName = "";
}
if (telephoneNumber == null) {
telephoneNumber = "";
}
if (mobile == null) {
mobile = "";
}
if (title == null) {
title = "";
}
}
}
}
catch (NamingException e){
System.err.println("FAIL MESSAGE: " + e);
}
}
public String ldapSearchResultDistinguishedName() {
return distinguishedName;
}
public String ldapSearchResultDepartment() {
return department;
}
public String ldapSearchResultPhysicalDeliveryOfficeName() {
return physicalDeliveryOfficeName;
}
public String ldapSearchResultTelephoneNumber() {
return telephoneNumber;
}
public String ldapSearchResultMobile() {
return mobile;
}
public String ldapSearchResultTitle() {
return title;
}
public String ldapSearchResultUnicodePwd() {
return unicodePwd;
}
public String ldapSearchResultSAMAccountName() {
return sAMAccountName;
}
}
After running the code, all the variables return the correct information but the variable "unicodePwd", that returns "null", even though the user has a password.
I know about the byte UTF-16LE thing and that the "unicodePwd" field in Active Directory is encrypted, but, as I explained earlier, i need that information decrypted in a String variable.
Any ideias?
Thank you!

I know this is an old question but I stumbled across it as I was also looking for an answer to the same question. I found the answer and thought it might help anybody else who lands here.
According to Microsoft Documentation it would appear that the unicodePwd attribute is NEVER returned by an LDAP search.
In my case, I need to validate that the credentials received are correct. So my plan is to use the username/password received and create a custom LdapContextFactory on the fly with those credentials. If I can contact the server successfully by doing an LdapContextFactory.get and get back an LdapContext then I can be certain that the password supplied was correct. If you don't get it back then you know it's wrong and can take it from there.

How to get the configured HTTP and HTTPS port numbers in server.xml from Java code at runtime

In our project, we have implemented SOAP webservices using Apache CXF framework. Clients used to request the server for some command execution. The request consists of host, port and the protocol used for connection. If the client uses a HTTPS configured port number and specify the protocol as HTTP, then we get a connection refused - socket exception as expected. But, I need to throw a proper error message like "Unable to connect to host "XYZ" with port "ABC" using http protocol". For this, I need to get the configured http and https port numbers from tomcat server.xml file at runtime and then compare it with my request parameters.
Anyone, please help me out on how to retrieve that?

You can always parse the tomcat's server.xml file and fetch the port values:
public static Integer getTomcatPortFromConfigXml(File serverXml) {
Integer port;
try {
DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
domFactory.setNamespaceAware(true); // never forget this!
DocumentBuilder builder = domFactory.newDocumentBuilder();
Document doc = builder.parse(serverXml);
XPathFactory factory = XPathFactory.newInstance();
XPath xpath = factory.newXPath();
XPathExpression expr = xpath.compile
("/Server/Service[#name='Catalina']/Connector[count(#scheme)=0]/#port[1]");
String result = (String) expr.evaluate(doc, XPathConstants.STRING);
port = result != null && result.length() > 0 ? Integer.valueOf(result) : null;
} catch (Exception e) {
port = null;
}
return port;
}
Above code should get you the HTTP port from server.xml. For HTTPS port, the XPathExpression has to be modified to
XPathExpression expr = xpath.compile
("/Server/Service[#name='Catalina']/Connector[#scheme='https']/#port[1]");
Please note that the above snippets are based on the assumption that the server.xml is the standard tomcat's server file where the service name is defined as "Catalina". Following is a standard server.xml file:
<Server>
<Service name="Catalina">
<Connector port="8080">
#...
</Connector>
</Service>
</Server>
Reference: Code link

It is possible to access Tomcat core classes on runtime this way (For Tomcat 7, I've not tested with Tomcat 8):
import java.lang.management.ManagementFactory;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import org.apache.catalina.Service;
import org.apache.catalina.connector.Connector;
import org.apache.catalina.core.StandardServer;
import org.apache.log4j.Logger;
public class TomcatConnectors {
public static final String CATALINA_SERVICE_NAME = "Catalina";
public static final String CONNECTOR_HTTP_PROTOCOL_NAME = "HTTP/1.1";
private Logger logger = Logger.getLogger(this.getClass());
private Collection<Connector> connectors;
/**
*
*/
public TomcatConnectors() {
super();
this.connectors = new HashSet<Connector>();
this.loadConnectors();
this.getConnectorPorts();
}
/**
*
* #return
*/
protected StandardServer getServerInstance(){
org.apache.catalina.core.StandardServer server = null;
try{
MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
server = (StandardServer)mbeanServer.getAttribute(
new ObjectName("Catalina:type=Server"),
"managedResource"
);
if(logger.isDebugEnabled()){
logger.debug("Server found. Info: ");
logger.debug(" - address : " + server.getAddress());
logger.debug(" - domain : " + server.getDomain());
logger.debug(" - info : " + server.getInfo());
logger.debug(" - shutdown port : " + server.getPort());
logger.debug(" - shutdown command : " + server.getShutdown());
logger.debug(" - serverInfo : " + server.getServerInfo());
logger.debug(" - status : " + server.getStateName());
}
}catch(Throwable t){
logger.fatal("Fatal Error Recovering StandardServer from MBeanServer : " + t.getClass().getName() + ": " + t.getMessage(), t);
}
return server;
}
/*
*
*/
protected Service getCatalinaService(){
org.apache.catalina.core.StandardServer server = this.getServerInstance();
Service[] services = server.findServices();
for(Service aService : services){
if(logger.isDebugEnabled()){
logger.debug("Service: " + aService.getName() +
", info: " + aService.getInfo() +
", state: " + aService.getStateName());
}
if(aService.getName().equalsIgnoreCase(CATALINA_SERVICE_NAME)){
return aService;
}
}
return null;
}
protected void loadConnectors() {
Service catalinaService = this.getCatalinaService();
if(catalinaService == null){
throw new IllegalStateException("Service Catalina cannot be null");
}
if(catalinaService.findConnectors() != null && catalinaService.findConnectors().length > 0){
logger.debug("List of connectors: ");
for(Connector aConnector : catalinaService.findConnectors()){
if(logger.isDebugEnabled()){
logger.debug("Connector.getProtocol: " + aConnector.getProtocol());
logger.debug("Connector.getPort: " + aConnector.getPort());
logger.debug("Connector.getInfo: " + aConnector.getInfo());
logger.debug("Connector.getStateName: " + aConnector.getStateName());
logger.debug("Connector.property.bindOnInit: " + aConnector.getProperty("bindOnInit"));
logger.debug("Connector.attribute.bindOnInit: " + aConnector.getAttribute("bindOnInit"));
logger.debug("Connector.getState: " + aConnector.getState());
}
this.connectors.add(aConnector);
}
}
}
/**
* #return the connectors
*/
public Collection<Connector> getConnectors() {
if(this.connectors.isEmpty()){
this.loadConnectors();
}
return connectors;
}
public Map<String, Set<Integer>> getConnectorPorts(){
if(this.connectors.isEmpty()){
this.loadConnectors();
}
Map<String, Set<Integer>> connectorPorts = new HashMap<String, Set<Integer>>();
for(Connector c: this.connectors){
Set<Integer> set;
if(!connectorPorts.containsKey(c.getProtocol())){
set = new HashSet<Integer>();
set.add(c.getLocalPort());
}else{
set = connectorPorts.get(c.getProtocol());
set.add(c.getLocalPort());
}
connectorPorts.put(c.getProtocol(), set);
}
logger.debug("connectorPorts : " + connectorPorts);
return connectorPorts;
}
}
This is the config which I've tested with:
<Service name="Catalina">
<Connector port="8787" protocol="HTTP/1.1" />
<Connector port="8009" protocol="AJP/1.3" />
...
And this is the output:
TomcatConnectors:137 - connectorPorts : {HTTP/1.1=[8787], AJP/1.3=[8009]}

Is there a useDirtyFlag option for Tomcat 6 cluster configuration?

In Tomcat 5.0.x you had the ability to set useDirtyFlag="false" to force replication of the session after every request rather than checking for set/removeAttribute calls.
<Cluster className="org.apache.catalina.cluster.tcp.SimpleTcpCluster"
managerClassName="org.apache.catalina.cluster.session.SimpleTcpReplicationManager"
expireSessionsOnShutdown="false"
**useDirtyFlag="false"**
doClusterLog="true"
clusterLogName="clusterLog"> ...
The comments in the server.xml stated this may be used to make the following work:
<%
HashMap map = (HashMap)session.getAttribute("map");
map.put("key","value");
%>
i.e. change the state of an object that has already been put in the session and you can be sure that this object still be replicated to the other nodes in the cluster.
According to the Tomcat 6 documentation you only have two "Manager" options - DeltaManager & BackupManager ... neither of these seem to allow this option or anything like it. In my testing the default setup:
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
where you get the DeltaManager by default, it's definitely behaving as useDirtyFlag="true" (as I'd expect).
So my question is - is there an equivalent in Tomcat 6?
Looking at the source I can see a manager implementation "org.apache.catalina.ha.session.SimpleTcpReplicationManager" which does have the useDirtyFlag but the javadoc comments in this state it's "Tomcat Session Replication for Tomcat 4.0" ... I don't know if this is ok to use - I'm guessing not as it's not mentioned in the main cluster configuration documentation.

I posted essentially the same question on the tomcat-users mailing list and the responses to this along with some information in the tomcat bugzilla ([43866]) led me to the following conclusions:
There is no equivalent to the useDirtyFlag, if you're putting mutable (ie changing) objects in the session you need a custom coded solution.
A Tomcat ClusterValve seems to be an effecting place for this solution - plug into the cluster mechanism, manipulate attributes to make it appear to the DeltaManager that all attributes in the session have changed. This forces replication of the entire session.
Step 1: Write the ForceReplicationValve (extends ValveBase implements ClusterValve)
I won't include the whole class but the key bit of logic (taking out the logging and instanceof checking):
#Override
public void invoke(Request request, Response response)
throws IOException, ServletException {
getNext().invoke(request, response);
Session session = request.getSessionInternal();
HttpSession deltaSession = (HttpSession) session;
for (Enumeration<String> names = deltaSession.getAttributeNames();
names.hasMoreElements(); ) {
String name = names.nextElement();
deltaSession.setAttribute(name, deltaSession.getAttribute(name));
}
}
Step 2: Alter the cluster config (in conf/server.xml)
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"
channelSendOptions="8">
<Valve className="org.apache.catalina.ha.tcp.ForceReplicationValve"/>
<Valve className="org.apache.catalina.ha.tcp.ReplicationValve"
filter=".*\.gif;.*\.jpg;.*\.png;.*\.js;.*\.htm;.*\.html;.*\.txt;.*\.css;"/>
<Valve className="org.apache.catalina.ha.session.JvmRouteBinderValve"/>
<ClusterListener className="org.apache.catalina.ha.session.JvmRouteSessionIDBinderListener"/>
<ClusterListener className="org.apache.catalina.ha.session.ClusterSessionListener"/>
</Cluster>
Replication of the session to all cluster nodes will now happen after every request.
Aside: Note the channelSendOptions setting. This replaces the replicationMode=asynchronous/synchronous/pooled from Tomcat 5.0.x. See the cluster documentation for the possible int values.
Appendix: Full Valve source as requested
package org.apache.catalina.ha.tcp;
import java.io.IOException;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpSession;
import org.apache.catalina.Lifecycle;
import org.apache.catalina.LifecycleException;
import org.apache.catalina.LifecycleListener;
import org.apache.catalina.Session;
import org.apache.catalina.connector.Request;
import org.apache.catalina.connector.Response;
import org.apache.catalina.ha.CatalinaCluster;
import org.apache.catalina.ha.ClusterValve;
import org.apache.catalina.ha.session.ReplicatedSession;
import org.apache.catalina.ha.session.SimpleTcpReplicationManager;
import org.apache.catalina.util.LifecycleSupport;
//import org.apache.catalina.util.StringManager;
import org.apache.catalina.valves.ValveBase;
/**
* <p>With the {#link SimpleTcpReplicationManager} effectively deprecated, this allows
* mutable objects to be replicated in the cluster by forcing the "dirty" status on
* every request.</p>
*
* #author Jon Brisbin (via post on tomcat-users http://markmail.org/thread/rdo3drcir75dzzrq)
* #author Kevin Jansz
*/
public class ForceReplicationValve extends ValveBase implements Lifecycle, ClusterValve {
private static org.apache.juli.logging.Log log =
org.apache.juli.logging.LogFactory.getLog( ForceReplicationValve.class );
#SuppressWarnings("hiding")
protected static final String info = "org.apache.catalina.ha.tcp.ForceReplicationValve/1.0";
// this could be used if ForceReplicationValve messages were setup
// in org/apache/catalina/ha/tcp/LocalStrings.properties
//
// /**
// * The StringManager for this package.
// */
// #SuppressWarnings("hiding")
// protected static StringManager sm =
// StringManager.getManager(Constants.Package);
/**
* Not actually required but this must implement {#link ClusterValve} to
* be allowed to be added to the Cluster.
*/
private CatalinaCluster cluster = null ;
/**
* Also not really required, implementing {#link Lifecycle} to allow
* initialisation and shutdown to be logged.
*/
protected LifecycleSupport lifecycle = new LifecycleSupport(this);
/**
* Default constructor
*/
public ForceReplicationValve() {
super();
if (log.isInfoEnabled()) {
log.info(getInfo() + ": created");
}
}
#Override
public String getInfo() {
return info;
}
#Override
public void invoke(Request request, Response response) throws IOException,
ServletException {
getNext().invoke(request, response);
Session session = null;
try {
session = request.getSessionInternal();
} catch (Throwable e) {
log.error(getInfo() + ": Unable to perform replication request.", e);
}
String context = request.getContext().getName();
String task = request.getPathInfo();
if(task == null) {
task = request.getRequestURI();
}
if (session != null) {
if (log.isDebugEnabled()) {
log.debug(getInfo() + ": [session=" + session.getId() + ", instanceof=" + session.getClass().getName() + ", context=" + context + ", request=" + task + "]");
}
if (session instanceof ReplicatedSession) {
// it's a SimpleTcpReplicationManager - can just set to dirty
((ReplicatedSession) session).setIsDirty(true);
if (log.isDebugEnabled()) {
log.debug(getInfo() + ": [session=" + session.getId() + ", context=" + context + ", request=" + task + "] maked DIRTY");
}
} else {
// for everything else - cycle all attributes
List cycledNames = new LinkedList();
// in a cluster where the app is <distributable/> this should be
// org.apache.catalina.ha.session.DeltaSession - implements HttpSession
HttpSession deltaSession = (HttpSession) session;
for (Enumeration<String> names = deltaSession.getAttributeNames(); names.hasMoreElements(); ) {
String name = names.nextElement();
deltaSession.setAttribute(name, deltaSession.getAttribute(name));
cycledNames.add(name);
}
if (log.isDebugEnabled()) {
log.debug(getInfo() + ": [session=" + session.getId() + ", context=" + context + ", request=" + task + "] cycled atrributes=" + cycledNames + "");
}
}
} else {
String id = request.getRequestedSessionId();
log.warn(getInfo() + ": [session=" + id + ", context=" + context + ", request=" + task + "] Session not available, unable to send session over cluster.");
}
}
/*
* ClusterValve methods - implemented to ensure this valve is not ignored by Cluster
*/
public CatalinaCluster getCluster() {
return cluster;
}
public void setCluster(CatalinaCluster cluster) {
this.cluster = cluster;
}
/*
* Lifecycle methods - currently implemented just for logging startup
*/
/**
* Add a lifecycle event listener to this component.
*
* #param listener The listener to add
*/
public void addLifecycleListener(LifecycleListener listener) {
lifecycle.addLifecycleListener(listener);
}
/**
* Get the lifecycle listeners associated with this lifecycle. If this
* Lifecycle has no listeners registered, a zero-length array is returned.
*/
public LifecycleListener[] findLifecycleListeners() {
return lifecycle.findLifecycleListeners();
}
/**
* Remove a lifecycle event listener from this component.
*
* #param listener The listener to remove
*/
public void removeLifecycleListener(LifecycleListener listener) {
lifecycle.removeLifecycleListener(listener);
}
public void start() throws LifecycleException {
lifecycle.fireLifecycleEvent(START_EVENT, null);
if (log.isInfoEnabled()) {
log.info(getInfo() + ": started");
}
}
public void stop() throws LifecycleException {
lifecycle.fireLifecycleEvent(STOP_EVENT, null);
if (log.isInfoEnabled()) {
log.info(getInfo() + ": stopped");
}
}
}

Many thanks to kevinjansz for providing the source for ForceReplicationValve.
I adjusted it for Tomcat7, here it is if anyone needs it:
package org.apache.catalina.ha.tcp;
import java.io.IOException;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpSession;
import org.apache.catalina.Lifecycle;
import org.apache.catalina.LifecycleException;
import org.apache.catalina.LifecycleListener;
import org.apache.catalina.Session;
import org.apache.catalina.connector.Request;
import org.apache.catalina.connector.Response;
import org.apache.catalina.ha.CatalinaCluster;
import org.apache.catalina.ha.ClusterValve;
import org.apache.catalina.util.LifecycleSupport;
import org.apache.catalina.valves.ValveBase;
import org.apache.catalina.LifecycleState;
// import org.apache.tomcat.util.res.StringManager;
/**
* <p>With the {#link SimpleTcpReplicationManager} effectively deprecated, this allows
* mutable objects to be replicated in the cluster by forcing the "dirty" status on
* every request.</p>
*
* #author Jon Brisbin (via post on tomcat-users http://markmail.org/thread/rdo3drcir75dzzrq)
* #author Kevin Jansz
*/
public class ForceReplicationValve extends ValveBase implements Lifecycle, ClusterValve {
private static org.apache.juli.logging.Log log =
org.apache.juli.logging.LogFactory.getLog( ForceReplicationValve.class );
#SuppressWarnings("hiding")
protected static final String info = "org.apache.catalina.ha.tcp.ForceReplicationValve/1.0";
// this could be used if ForceReplicationValve messages were setup
// in org/apache/catalina/ha/tcp/LocalStrings.properties
//
// /**
// * The StringManager for this package.
// */
// #SuppressWarnings("hiding")
// protected static StringManager sm =
// StringManager.getManager(Constants.Package);
/**
* Not actually required but this must implement {#link ClusterValve} to
* be allowed to be added to the Cluster.
*/
private CatalinaCluster cluster = null;
/**
* Also not really required, implementing {#link Lifecycle} to allow
* initialisation and shutdown to be logged.
*/
protected LifecycleSupport lifecycle = new LifecycleSupport(this);
/**
* Default constructor
*/
public ForceReplicationValve() {
super();
if (log.isInfoEnabled()) {
log.info(getInfo() + ": created");
}
}
#Override
public String getInfo() {
return info;
}
#Override
public void invoke(Request request, Response response) throws IOException,
ServletException {
getNext().invoke(request, response);
Session session = null;
try {
session = request.getSessionInternal();
} catch (Throwable e) {
log.error(getInfo() + ": Unable to perform replication request.", e);
}
String context = request.getContext().getName();
String task = request.getPathInfo();
if(task == null) {
task = request.getRequestURI();
}
if (session != null) {
if (log.isDebugEnabled()) {
log.debug(getInfo() + ": [session=" + session.getId() + ", instanceof=" + session.getClass().getName() + ", context=" + context + ", request=" + task + "]");
}
//cycle all attributes
List<String> cycledNames = new LinkedList<String>();
// in a cluster where the app is <distributable/> this should be
// org.apache.catalina.ha.session.DeltaSession - implements HttpSession
HttpSession deltaSession = (HttpSession) session;
for (Enumeration<String> names = deltaSession.getAttributeNames(); names.hasMoreElements(); ) {
String name = names.nextElement();
deltaSession.setAttribute(name, deltaSession.getAttribute(name));
cycledNames.add(name);
}
if (log.isDebugEnabled()) {
log.debug(getInfo() + ": [session=" + session.getId() + ", context=" + context + ", request=" + task + "] cycled atrributes=" + cycledNames + "");
}
} else {
String id = request.getRequestedSessionId();
log.warn(getInfo() + ": [session=" + id + ", context=" + context + ", request=" + task + "] Session not available, unable to send session over cluster.");
}
}
/*
* ClusterValve methods - implemented to ensure this valve is not ignored by Cluster
*/
public CatalinaCluster getCluster() {
return cluster;
}
public void setCluster(CatalinaCluster cluster) {
this.cluster = cluster;
}
/*
* Lifecycle methods - currently implemented just for logging startup
*/
/**
* Add a lifecycle event listener to this component.
*
* #param listener The listener to add
*/
public void addLifecycleListener(LifecycleListener listener) {
lifecycle.addLifecycleListener(listener);
}
/**
* Get the lifecycle listeners associated with this lifecycle. If this
* Lifecycle has no listeners registered, a zero-length array is returned.
*/
public LifecycleListener[] findLifecycleListeners() {
return lifecycle.findLifecycleListeners();
}
/**
* Remove a lifecycle event listener from this component.
*
* #param listener The listener to remove
*/
public void removeLifecycleListener(LifecycleListener listener) {
lifecycle.removeLifecycleListener(listener);
}
protected synchronized void startInternal() throws LifecycleException {
setState(LifecycleState.STARTING);
if (log.isInfoEnabled()) {
log.info(getInfo() + ": started");
}
}
protected synchronized void stopInternal() throws LifecycleException {
setState(LifecycleState.STOPPING);
if (log.isInfoEnabled()) {
log.info(getInfo() + ": stopped");
}
}
}