We Keep Coding

The message attribute in the JSON log has backslash

I am using spring boot in a project and currently exploring the logging behaviour. For that I'm using the zipkin service.
My logback.xml is as follows:
<appender name="STASH"
class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>192.168.0.83:5000</destination>
<encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder"
<providers>
<mdc /> <!-- MDC variables on the Thread will be written as JSON fields -->
<context /> <!--Outputs entries from logback's context -->
<version /> <!-- Logstash json format version, the #version field in the output -->
<logLevel />
<loggerName />
<pattern>
<pattern>
{
"serviceName": "zipkin-demo"
}
</pattern>
</pattern>
<threadName />
<message />
<logstashMarkers />
<arguments />
<stackTrace />
</providers>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STASH" />
</root>
Now for what I have understood, to log in json, you need a custom logger implementation which I have done as follows:
public class CustomLogger {
public static void info(Logger logger, Message message, String msg) {
log(logger.info(), message, msg).log();
}
private static JsonLogger log(JsonLogger logger, Message message, String msg) {
try {
if (message == null) {
return logger.field("Message", "Empty message").field("LogTime", new Date() );
}
logger
.field("message", message.getMessage())
.field("id", message.getId());
StackTraceElement ste = (new Exception()).getStackTrace()[2];
logger.field(
"Caller",
ste.getClassName() + "." + ste.getMethodName() + ":" + ste.getLineNumber());
return logger;
} catch (Exception e) {
return logger.exception("Error while logging", e).stack();
}
}
My message class is:
public class Message {
private int id;
private String message;
..constructor & setters and getters
}
Now in my controller class I'm using my custom logger as:
static com.savoirtech.logging.slf4j.json.logger.Logger log = com.savoirtech.logging.slf4j.json.LoggerFactory.getLogger(Controller.class.getClass());
Message msg = new Message(1, "hello");
CustomLogger customLogger = new CustomLogger();
customLogger.info(log, msg, "message");
My logs end up in kibana as:
"message": "{\"message\":\"hello\",\"id\":1,\"Caller\":\"<class_name>:65\",\"level\":\"INFO\",\"thread_name\":\"http-nio-3333-exec-2\",\"class\":\"<custom_logger_class>\",\"logger_name\":\"java.lang.Class\",\"#timestamp\":\"2018-07-20 19:02:14.090+0530\",\"mdc\":{\"X-B3-TraceId\":\"554c43b0275c3430\",\"X-Span-Export\":\"true\",\"X-B3-SpanId\":\"368702fffa40d2cd\"}}",
Now instead of this I would want my message part of the logs as a json entry. Where am I going wrong?
I have tried searching a way extensively but to no avail. Is it even possible?

I finally got it working. I just changed the logstash config file and added:
input
{
tcp
{
port => 5000
type => "java"
codec => "json"
}
}
filter
{
if [type]== "java"
{
json
{
source => "message"
}
}
}
The filter part was missing earlier.

How to create multiple log file programatically in log4j2?

I am developing a java application which communicates with lots of devices. For each device I need to create a different log file to log it's communication with device. This is the wrapper class I developed. It creates two log files but the data is written to only the first one. The second file is created but nothing is written to it. The output that should go to second file goes to console. If I uncomment createRootLogger() in constructor nothing is written to both the files, everything goes to console. I have gone through log4j2 documentation but it is poorly written with very few code samples. Here is my wrapper class, where is the error? I am using log4j-api-2.9.0.jar and log4j-core-2.9.0.jar.
package xyz;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.ConsoleAppender;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.Configurator;
import org.apache.logging.log4j.core.config.builder.api.*;
import java.util.Hashtable;
public class LogManager
{
static protected LogManager m_clsInstance = null;
protected Hashtable<String, Logger> m_clsLoggers = new Hashtable<String, Logger>();
private LogManager()
{
//createRootLogger();
}
/**
* getInstance is used to get reference to the singalton class obj ......
*/
static synchronized public LogManager getInstance()
{
try
{
if (m_clsInstance == null)
{
m_clsInstance = new LogManager();
//Configurator.setRootLevel(Level.TRACE);
}
}
catch (Exception xcpE)
{
System.err.println(xcpE);
}
return m_clsInstance;
}
static public Logger getLogger(String sLogger)
{
try
{
return getInstance().m_clsLoggers.get(sLogger);
}
catch (Exception xcpE)
{
System.err.println(xcpE);
}
return null;
}
public Logger createLogger(String strName, String sPath, int nBackupSize, long lngMaxSize, String strPattern, String strLevel)
{
try
{
ConfigurationBuilder builder = ConfigurationBuilderFactory.newConfigurationBuilder();
builder.setStatusLevel(Level.getLevel(strLevel));
builder.setConfigurationName("RollingBuilder"+strName);
// create a console appender
AppenderComponentBuilder appenderBuilder = builder.newAppender("Stdout", "CONSOLE").addAttribute("target",
ConsoleAppender.Target.SYSTEM_OUT);
appenderBuilder.add(builder.newLayout("PatternLayout")
.addAttribute("pattern", strPattern));
builder.add( appenderBuilder );
// create a rolling file appender
LayoutComponentBuilder layoutBuilder = builder.newLayout("PatternLayout")
.addAttribute("pattern", strPattern);
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
// .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", lngMaxSize));
appenderBuilder = builder.newAppender("rolling"+strName, "RollingFile")
.addAttribute("fileName", sPath)
.addAttribute("filePattern", "d:\\trash\\archive\\rolling-%d{MM-dd-yy}.log.gz")
.add(layoutBuilder)
.addComponent(triggeringPolicy);
builder.add(appenderBuilder);
// create the new logger
builder.add( builder.newLogger( strName, Level.getLevel(strLevel) )
.add( builder.newAppenderRef( "rolling"+strName ) )
.addAttribute( "additivity", false ) );
Configuration clsCnfg = (Configuration) builder.build();
LoggerContext ctx = Configurator.initialize(clsCnfg);
Logger clsLogger = ctx.getLogger(strName);
m_clsLoggers.put(strName, clsLogger);
return clsLogger;
}
catch (Exception xcpE)
{
System.err.println(xcpE);
}
return null;
}
protected void createRootLogger()
{
try
{
ConfigurationBuilder builder = ConfigurationBuilderFactory.newConfigurationBuilder();
builder.setStatusLevel(Level.getLevel("TRACE"));
builder.setConfigurationName("rootConfig");
// create a console appender
AppenderComponentBuilder appenderBuilder = builder.newAppender("Stdout", "CONSOLE").addAttribute("target",
ConsoleAppender.Target.SYSTEM_OUT);
appenderBuilder.add(builder.newLayout("PatternLayout")
.addAttribute("pattern", "[%d{yyyy-MMM-dd HH:mm:ss:SSS}][%-5p %l][%t] %m%n"));
builder.add( appenderBuilder );
builder.add( builder.newRootLogger( Level.getLevel("TRACE"))
.add( builder.newAppenderRef( "Stdout") ) );
Configuration clsCnfg = (Configuration) builder.build();
LoggerContext ctx = Configurator.initialize(clsCnfg);
Logger clsLogger = ctx.getRootLogger();
m_clsLoggers.put("root", clsLogger);
}
catch (Exception xcpE)
{
System.err.println(xcpE);
}
}
static public void main(String args[])
{
//Logger clsLogger = setLogger();
Logger clsLogger = Emflex.LogManager.getInstance().createLogger(
"AnsiAmrController_" + 5555,
"d:\\trash\\LogManagerTest5555.log",
10,
100000000,
"[%d{yyyy-MMM-dd HH:mm:ss:SSS}][%-5p %l][%t] %m%n",
"TRACE"
);
Logger clsLogger2 = Emflex.LogManager.getInstance().createLogger(
"AnsiAmrController_" + 6666,
"d:\\trash\\LogManagerTest6666.log",
10,
100000000,
"[%d{yyyy-MMM-dd HH:mm:ss:SSS}][%-5p %l][%t] %m%n",
"TRACE"
);
for (int i=0;i<100;i++)
{
clsLogger.error("Testing - ["+i+"]");
clsLogger2.error("Testing - ["+(i*i)+"]");
}
}
}

You said your objective is:
For each device I need to create a different log file to log it's communication with device.
There are many different ways to accomplish this without programmatic configuration. Programmatic configuration is bad because it forces you to depend on the logging implementation rather than the public interface.
For example you could use a context map key in conjunction with a Routing Appender to separate your logs, similar to the example I gave in another answer. Note that in the other answer I used the variable as the folder where the log is stored but you can use it for the log name if you wish.
Another way to do what you want would be to use a MapMessage as shown in the log4j2 manual.
Yet another way would be to use markers in combination with a RoutingAppender. Here is some example code for this approach:
package example;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Marker;
import org.apache.logging.log4j.MarkerManager;
public class LogLvlByMarkerMain {
private static final Logger log = LogManager.getLogger();
private static final Marker DEVICE1 = MarkerManager.getMarker("DEVICE1");
private static final Marker DEVICE2 = MarkerManager.getMarker("DEVICE2");
public static void main(String[] args) {
log.info(DEVICE1, "The first device got some input");
log.info(DEVICE2, "The second device now has input");
}
}
Configuration:
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Appenders>
<Routing name="MyRoutingAppender">
<Routes pattern="$${marker:}">
<Route>
<File
fileName="logs/${marker:}.txt"
name="appender-${marker:}">
<PatternLayout>
<Pattern>[%date{ISO8601}][%-5level][%t] %m%n</Pattern>
</PatternLayout>
</File>
</Route>
</Routes>
</Routing>
<Console name="STDOUT" target="SYSTEM_OUT">
<PatternLayout pattern="[%date{ISO8601}][%-5level][%t] %m%n" />
</Console>
</Appenders>
<Loggers>
<Logger name="example" level="TRACE" additivity="false">
<AppenderRef ref="STDOUT" />
<AppenderRef ref="MyRoutingAppender" />
</Logger>
<Root level="WARN">
<AppenderRef ref="STDOUT" />
</Root>
</Loggers>
</Configuration>
Output:
This will generate 2 log files - DEVICE1.txt and DEVICE2.txt as shown in the image below.
The first log will contain only messages that were marked as DEVICE1 and the second will contain only DEVICE2 logs.
I.e. the first log contains:
[2017-09-21T09:52:04,171][INFO ][main] The first device got some input
and the second contains:
[2017-09-21T09:52:04,176][INFO ][main] The second device now has input

The approach log4j2 is initialize programmatically and later configuration is modified is different. And you you trying to add dynamic appender and logger using initialization approach.
So, first you should initialize your RootLogger using initialization approach that seems correct in your code.
After that, add dynamic appender and logger using approach mentioned here

adding on D.B answer:
I had trouble making this write to file. (and yes I tried using log4j2 version 2.8.1 but still didn't work)
To make it work I edited this part
<Root level="WARN">
<AppenderRef ref="STDOUT" />
</Root>
to this:
<Root level="WARN">
<AppenderRef ref="STDOUT" />
<AppenderRef ref="MyRoutingAppender" />
</Root>
And since the Debug level is set to WARN
<Configuration status="WARN">
and we trying to log info
log.info(DEVICE$, "The $ device now has input");
the info log wont be written (WARN will only print: warn, error, fatal check this link log4j logging level)
you can simply change
log.info() --> log.warn()
just as a proof of concept.

Programmatically log into multiple files using log4j

I want to write logs in separate files Programmatically,
public class ProgLoggerMultipaleFiles {
HashMap<LogCategory, Logger> myLogHashMap = new HashMap<LogCategory, Logger>();
public ProgLoggerMultipaleFiles() {
JLogger jlog = new JLogger();
jlog.startFileLog("mylog");
jlog.startFileLog("HC");
jlog.startFileLog("MC");
jlog.startFileLog("DC");
myLogHashMap.put(LogCategory.mylog, Logger.getLogger("mylog"));
myLogHashMap.put(LogCategory.HC, Logger.getLogger("HC"));
myLogHashMap.put(LogCategory.MC, Logger.getLogger("MC"));
myLogHashMap.put(LogCategory.DC, Logger.getLogger("DC"));
String parameter = "Hello";
log(LogCategory.mylog,Priority.DEBUG,"This is debug : " + parameter);
log(LogCategory.mylog,Priority.INFO,"This is info : " + parameter);
log(LogCategory.mylog,Priority.WARN,"This is warn : " + parameter);
log(LogCategory.mylog,Priority.ERROR,"This is error : " + parameter);
log(LogCategory.mylog,Priority.FATAL,"This is fatal : " + parameter);
log(LogCategory.HC,Priority.FATAL,"HC");
log(LogCategory.MC,Priority.FATAL,"MC");
log(LogCategory.DC,Priority.FATAL,"DC");
}
public void log(LogCategory category,Priority priority,String msg){
myLogHashMap.get(category).log(priority, msg);
}
public enum LogCategory{
mylog,HC,MC,DC
}
public static void main(String[] args) {
ProgLoggerMultipaleFiles plog = new ProgLoggerMultipaleFiles();
}
}
And I initialize loggers and Appenders in this class,
public class JLogger {
public JLogger() {
startConsolLog();
}
public void startConsolLog(){
ConsoleAppender console = new ConsoleAppender(); //create appender
//configure the appender
String PATTERN = "%d [%p|%c|%C{1}] %m%n";
console.setLayout(new PatternLayout(PATTERN));
console.setThreshold(Level.FATAL);
console.activateOptions();
//add appender to any Logger (here is root)
Logger.getRootLogger().addAppender(console);
}
public void startFileLog(String fileName){
FileAppender fa = new FileAppender();
fa.setName(fileName);
fa.setFile(fileName+".log");
fa.setLayout(new PatternLayout("%d %-5p [%c{1}] %m%n"));
fa.setThreshold(Level.DEBUG);
fa.setAppend(true);
fa.activateOptions();
//add appender to any Logger (here is root)
Logger.getRootLogger().setAdditivity(false);
Logger.getRootLogger().addAppender(fa);
//repeat with all other desired appenders
}
}
When I run this code creates 4 diff files but all messages are logged into all files.
Thanks in advance.

Log4j Loggers work like a tree. When you get Loggers like LogManager.getLogger(MyClass.class) and MyClass is in org.my.company namespace, log4j will traverse the configuration and look for "org.my.company.MyClass" then "org.my.company" and so on until it finds one. If not it uses the rootlogger.
So you can "tap" that tree with appenders. For example you want all classes in "org.my.company.api" to log into a special file: Configure a logger named "org.my.company.api" , add that fileappender to it and get the Loggers with *.class.
In your case it's a bit different. You get Loggers with a specific name that is probably not in the namespace tree. So if there is no logger with that special name, the root logger is used. Thus all messages go everywhere.
So what you have to do is configure not only appenders, but also Loggers with those specific names - "mylog" for example and add the respective appender to only that logger.
Your tree:
root ← Appender Console, mylog, HC, MC, ...
but you need actually:
root ← Appender Console
|- mylog ← Appender for mylog.log
|- MC ← Appender for MC.log
|- ...
Try:
public void startFileLog(String fileName){
FileAppender fa = new FileAppender();
fa.setName(fileName);
fa.setFile(fileName+".log");
fa.setLayout(new PatternLayout("%d %-5p [%c{1}] %m%n"));
fa.setThreshold(Level.DEBUG);
fa.setAppend(true);
fa.activateOptions();
//add appender to any Logger (here is NOT root)
Logger.getLogger(fileName).setAdditivity(false); // messages will not go to root logger anymore!
Logger.getLogger(fileName).addAppender(fa);
//repeat with all other desired appenders
}

How to add a PatternLayout to the root logger at runtime?

I am using logback as the backend for Slf4j. Currently, I configure the logger using a logback.xml file. My issue is that sensitive information is being logged (outside of my control) and I want to mask this sensitive information. To mask the information, I have wrote a custom PatternLayout class that essentially does:
#Override
public String doLayout(ILoggingEvent event) {
String message = super.doLayout(event);
Matcher matcher = sesnsitiveInfoPattern.matcher(message);
if (matcher.find()) {
message = matcher.replaceAll("XXX");
}
return message;
}
My issue is that I need to tell logback to use this custom pattern layout. I don't want to add this to the XML configuration however. My current configuration looks like this:
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<layout class="com.my.MaskingPatternLayout"> <!-- here -->
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</layout>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
</root>
</configuration>
In XML, my desired configuration would look like this (but I don't want to use XML):

Hello Max I hope you are using Log4j 2.x because this solution uses the plugins approache introduced in log4j 2.x . first you should create a package where you are going to put your plugins classes and you put there these two classes :
my.log4j.pluggins.CustomConfigurationFactory :
#Plugin(name = "CustomConfigurationFactory", category = ConfigurationFactory.CATEGORY)
#Order(value = 0)
public class CustomConfigurationFactory extends ConfigurationFactory {
private Configuration createConfiguration(final String name,
ConfigurationBuilder<BuiltConfiguration> builder) {
System.out.println("init logger");
builder.setConfigurationName(name);
builder.setStatusLevel(Level.INFO);
builder.setPackages("my.log4j.pluggins");
AppenderComponentBuilder appenderBuilder = builder.newAppender(
"Stdout", "CONSOLE").addAttribute("target",
ConsoleAppender.Target.SYSTEM_OUT);
appenderBuilder
.add(builder
.newLayout("PatternLayout")
.addAttribute("pattern", "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %myMsg%n"));
builder.add(appenderBuilder);
builder.add(builder.newRootLogger(Level.TRACE).add(
builder.newAppenderRef("Stdout")));
return builder.build();
}
#Override
protected String[] getSupportedTypes() {
String[] supportedExt = { "*" };
return supportedExt;
}
#Override
public Configuration getConfiguration(ConfigurationSource source) {
ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder();
return createConfiguration(source.toString(), builder);
}
#Override
public Configuration getConfiguration(String name, URI configLocation) {
ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder();
return createConfiguration(name, builder);
}
}
my.log4j.pluggins.SampleLayout :
#Plugin(name = "CustomConverter", category = "Converter")
#ConverterKeys({"myMsg"})
public class SampleLayout extends LogEventPatternConverter {
protected SampleLayout(String name, String style) {
super(name, style);
}
public static SampleLayout newInstance(){
return new SampleLayout("custConv", "custConv");
}
#Override
public void format(LogEvent event, StringBuilder stringBuilder) {
//replace the %myMsg by XXXXX if sensitive
if (sensitive()){
stringBuilder.append("XXXX");}
else {
stringBuilder.append(event.getMessage().getFormattedMessage());}
}
}
the CustomConfiguration class is responsable for creating the configuration of log4j and the line 9 where 'builder.setPackages("my.log4j.pluggins")' is important in order to scan that package and pick up the converter pluggin wich is SampleLayout.
the second class will be responsible for formatting the new key '%myMsg' in the pattern that contains my sensitive message, this Converter class checks if that message is sensitive and actes accordingly.
Before you start logging you should configure your log4j like this
ConfigurationFactory.setConfigurationFactory(new CustomConfigurationFactory());

how to create a log4j custom appender and control the filename

My company uses a software package that reads in log files from our servers, parses them, and spits performance data into a database. We dont have access / permission to modify the source code for the app that reads the files but we do have access to the code that writes the files. I need to change the way the log files are being written and I would like to use log4j (so I can use an AsyncAppender). The program expects a few things:
1). There should be 10 log files that roll and each log file will be one day of logs. The files need to be named 0 through 9 and I need to be able to programatically set the file name and when they roll based on the server time.
2). Essentially when generating the 11th log file it should delete the oldest one and start writing to that one.
3). When a new log file is generated I need to be able to insert a timestamp as the first line of the file (System.currentTimeMillis()).
Is it possible to meet the above requirements with a custom log4j file appender? Ive looked at DailyRollingFileAppender but cant seem to figure out how to control the file names exactly like I need to. Also I cant seem to figure out how to write the first line in the log when it is generated (for example is there some callback function I can register when a new log file gets rolled in)?

I think you can achieve first 2 with
using RollingFileAppender and specifying FixedWindowRollingPolicy for RollingPolicy
as for the #3 you can always write your own handler

For the sake of posterity. I used the below class as my custom rolling policy
import org.apache.log4j.rolling.RollingPolicyBase;
import org.apache.log4j.rolling.RolloverDescription;
import org.apache.log4j.rolling.RolloverDescriptionImpl;
import org.apache.log4j.rolling.TriggeringPolicy;
import org.apache.log4j.Appender;
import org.apache.log4j.spi.LoggingEvent;
public final class CustomRollingPolicy extends RollingPolicyBase
implements TriggeringPolicy
{
private short curFileId = -1;
private String lastFileName = null;
static private final long FILETIMEINTERVAL = 86400000l;
static private final int NUM_FILES = 10;//86400000l;
public String folderName = "";
public String getFolderName() {
return folderName;
}
public void setFolderName(String folderName) {
this.folderName = folderName;
}
private short calculateID(long startTime) {
return (short) ((startTime / FILETIMEINTERVAL) % NUM_FILES);
}
public String getCurrentFileName()
{
StringBuffer buf = new StringBuffer();
buf.append(folderName);
buf.append(calculateID(System.currentTimeMillis()));
return buf.toString();
}
public void activateOptions()
{
super.activateOptions();
this.lastFileName = getCurrentFileName();
}
public RolloverDescription initialize(String currentActiveFile, boolean append)
{
curFileId = this.calculateID(System.currentTimeMillis());
lastFileName = getCurrentFileName();
String fileToUse = activeFileName != null? activeFileName: currentActiveFile != null?currentActiveFile:lastFileName;
return new RolloverDescriptionImpl(fileToUse, append, null, null);
}
public RolloverDescription rollover(String currentActiveFile)
{
curFileId = this.calculateID(System.currentTimeMillis());
String newFileName = getCurrentFileName();
if (newFileName.equals(this.lastFileName))
{
return null;
}
String lastBaseName = this.lastFileName;
String nextActiveFile = newFileName;
if (!currentActiveFile.equals(lastBaseName))
{
nextActiveFile = currentActiveFile;
}
this.lastFileName = newFileName;
return new RolloverDescriptionImpl(nextActiveFile, false, null, null);
}
public boolean isTriggeringEvent(Appender appender, LoggingEvent event, String filename, long fileLength)
{
short fileIdForCurrentServerTime = this.calculateID(System.currentTimeMillis());
return curFileId != fileIdForCurrentServerTime;
}
}
And here is the appender config in my log4j xml file:
<!-- ROLLING FILE APPENDER FOR RUM LOGS -->
<appender name="rumRollingFileAppender" class="org.apache.log4j.rolling.RollingFileAppender">
<rollingPolicy class="com.ntrs.wpa.util.CustomRollingPolicy">
<param name="folderName" value="C:/bea-portal-10.3.2/logs/"/>
<param name="FileNamePattern" value="C:/bea-portal-10.3.2/logs/foo.%d{yyyy-MM}.gz"/>
</rollingPolicy>
<layout class="com.ntrs.els.log4j.AppServerPatternLayout">
<param name="ConversionPattern" value="%m%n" />
</layout>
</appender>