We Keep Coding

antlr4/java: pretty print parse tree to stdout

Beginners question: how do I print a readable version of the parse tree to stdout?
CharStream input = CharStreams.fromFileName("testdata/test.txt");
MyLexer lexer = new MyLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
MyParser parser = new MyParser(tokens);
parser.setBuildParseTree(true);
RuleContext tree = parser.record();
System.out.println(tree.toStringTree(parser));
this prints the whole tree on a single line delimited by brackets '()'.
(record (husband <4601> (name KOHAI Nikolaus) \n (birth * um.1872 (place Ploschitz)) \n\n) (wife (marriage oo) \n (name SCHLOTTHAUER Maria) \n (birth * um.1877
...
I would like to have something like this
record
husband
<id>
name
<name>
...
wife

Extracted from SnippetsTest as a standalone utility class:
import java.util.List;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.runtime.tree.Tree;
import org.antlr.v4.runtime.tree.Trees;
public class TreeUtils {
/** Platform dependent end-of-line marker */
public static final String Eol = System.lineSeparator();
/** The literal indent char(s) used for pretty-printing */
public static final String Indents = " ";
private static int level;
private TreeUtils() {}
/**
* Pretty print out a whole tree. {#link #getNodeText} is used on the node payloads to get the text
* for the nodes. (Derived from Trees.toStringTree(....))
*/
public static String toPrettyTree(final Tree t, final List<String> ruleNames) {
level = 0;
return process(t, ruleNames).replaceAll("(?m)^\\s+$", "").replaceAll("\\r?\\n\\r?\\n", Eol);
}
private static String process(final Tree t, final List<String> ruleNames) {
if (t.getChildCount() == 0) return Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);
StringBuilder sb = new StringBuilder();
sb.append(lead(level));
level++;
String s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);
sb.append(s + ' ');
for (int i = 0; i < t.getChildCount(); i++) {
sb.append(process(t.getChild(i), ruleNames));
}
level--;
sb.append(lead(level));
return sb.toString();
}
private static String lead(int level) {
StringBuilder sb = new StringBuilder();
if (level > 0) {
sb.append(Eol);
for (int cnt = 0; cnt < level; cnt++) {
sb.append(Indents);
}
}
return sb.toString();
}
}
Call the method as follows:
List<String> ruleNamesList = Arrays.asList(parser.getRuleNames());
String prettyTree = TreeUtils.toPrettyTree(tree, ruleNamesList);

Besides a graphical parse tree my ANTLR4 extension for Visual Studio Code also produces a formatted text parse tree:

If you like to use regex only for what it's really for, you can always print a tree by yourself:
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.Trees;
public static String printSyntaxTree(Parser parser, ParseTree root) {
StringBuilder buf = new StringBuilder();
recursive(root, buf, 0, Arrays.asList(parser.getRuleNames()));
return buf.toString();
}
private static void recursive(ParseTree aRoot, StringBuilder buf, int offset, List<String> ruleNames) {
for (int i = 0; i < offset; i++) {
buf.append(" ");
}
buf.append(Trees.getNodeText(aRoot, ruleNames)).append("\n");
if (aRoot instanceof ParserRuleContext) {
ParserRuleContext prc = (ParserRuleContext) aRoot;
if (prc.children != null) {
for (ParseTree child : prc.children) {
recursive(child, buf, offset + 1, ruleNames);
}
}
}
}
Usage:
ParseTree root = parser.yourOwnRule();
System.out.println(printSyntaxTree(parser, root));

I wanted to put in my own spin on this, taking advantage of the fact that I already use StringTemplate in my project. This means I don't have to manually deal with levels like the other answers. It also makes the output format easier to customize.
On top of that, the main reason I'm posting this is because I decided to skip printing rules that I'm only 'passing through', i.e. when using chain rules
a : b | something_else ;
b : c | another ;
c : d | yet_more ;
d : rule that matters ;
since they cluttered my output when checking trees from small inputs without adding any usefull information. This is also easy to change, at the //pass-through rules comment location.
I also copied in the definition of Trees.getNodeText and modified it to use a plain array to get rid of the unnecessary wrapping, and even let me customize it if I feel like it.
Finally, I made it take the parser and tree and just straight dump to System.out, since that's the only situation I need it in.
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.antlr.v4.runtime.tree.Tree;
import org.stringtemplate.v4.ST;
//for pretty-dumping trees in short form
public class TreeUtils {
private static final ST template() {
return new ST("<rule_text>\n\t<child; separator=\"\n\">");
}
private static final ST literal(String text) {
return new ST("<text>").add("text", text);
}
public static void dump(Parser parser, Tree tree) {
System.out.println(process(parser.getRuleNames(),tree).render());
}
private static String getNodeText(Tree t, String[] ruleNames) {
if ( t instanceof RuleContext ) {
int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex();
String ruleName = ruleNames[ruleIndex];
return ruleName;
}
else if ( t instanceof ErrorNode) {
return t.toString();
}
else if ( t instanceof TerminalNode) {
Token symbol = ((TerminalNode)t).getSymbol();
if (symbol != null) {
String s = symbol.getText();
return s;
}
}
Object payload = t.getPayload();
if ( payload instanceof Token ) {
return ((Token)payload).getText();
}
return t.getPayload().toString();
}
private static ST process(String[] ruleNames, Tree t) {
if(t.getChildCount()==0) {
return literal(getNodeText(t, ruleNames));
} else if(t.getChildCount()==1) {
//pass-through rules
return process(ruleNames,t.getChild(0));
} else {
ST out=template();
out.add("rule_text", getNodeText(t, ruleNames));
for(int i=0;i<t.getChildCount();i++) {
out.add("child", process(ruleNames,t.getChild(i)));
}
return out;
}
}
}

For Kotlin, you can use this extension function
fun Tree.format(parser: Parser, indent: Int = 0): String = buildString {
val tree = this#format
val prefix = " ".repeat(indent)
append(prefix)
append(Trees.getNodeText(tree, parser))
if (tree.childCount != 0) {
append(" (\n")
for (i in 0 until tree.childCount) {
append(tree.getChild(i).format(parser, indent + 1))
append("\n")
}
append(prefix).append(")")
}
}

parse JSON Tree in java

I could not find out how to traverse a JSON-tree (nested structure) and decide on the keys of the elements what to expect next and traverse from node to node. Like this (pseudocode):
int traverse(node) {
if (node.key == ADD) {
int res = 0;
for (child:node.children)
res = res + traverse(child);
}
if (node.key == "ADD") {
int res = 0;
for (child:node.children)
res = res + traverse(children);
}
if (node.key == "MULT") {
int res = 0;
for (child:node.children)
res = res * traverse(children);
}
if (node.key == "INT")
return node.value;
}
The json-string to parse could be like this:
{"ADD":[{"INT":"1"},{"INT":"3"},{"INT":"4"}]}
or this:
{"ADD":[{"INT":"1"},{"INT":"3"},{"ADD":[{"INT":"5"},{"INT":"6"}]},
{"INT":"4"}]}
How could I use JSON-Object or JSON-Arrays and
inside the objects access the key and value variables to traverse through this tree recursively?
EDITED:
After all the comments I try to put this as first running example
(still looks a little uneasy to me, but it works):
public static int evaluate(javax.json.JsonObject node) {
Set<?> keySet = node.keySet();
Iterator<?> i = keySet.iterator();
if (i.hasNext()) {
String key = i.next().toString();
System.out.println("key: " + key);
if (key.equals("ADD")) {
JsonArray ja = node.getJsonArray("ADD");
int res = 0;
for (JsonValue jvx: ja) {
if (jvx.getValueType().toString().equals("OBJECT")) {
res = res + evaluate((JsonObject)jvx);
} else{
System.err.println("jvx should not be a " + jvx.getValueType().toString() + " here");
}
}
return res;
}
if (key.equals("MULT")) {
JsonArray ja = node.getJsonArray("MULT");
int res = 1;
for (JsonValue jvx: ja) {
if (jvx.getValueType().toString().equals("OBJECT")) {
res = res * evaluate((JsonObject)jvx);
} else{
System.err.println("jvx should not be a " + jvx.getValueType().toString() + " here");
}
}
return res;
}
if (key.equals("INT")) {
String intStr = node.getString("INT");
System.out.println ("found int = " + intStr);
return Integer.parseInt(intStr);
}
}
return 0;
}
public static void readJSON() {
String jsonText = "{\"ADD\":[{\"INT\":\"1\"},{\"INT\":\"3\"},{\"ADD\":[{\"INT\":\"5\"},{\"INT\":\"6\"}]},{\"INT\":\"4\"}]}";
JsonReader reader = Json.createReader(new StringReader(jsonText));
JsonObject obj = reader.readObject();
reader.close();
int res = evaluate(obj);
System.out.println("res: " + res);
}

Your evaluating pseudocode is OK (just pay attention to the initial value when multiplying!). To adapt it to the javax.json hierarchy, you should code your evaluating method like this:
int evaluate(javax.json.JsonObject node):
Get each on of the admitted keys (ADD, MULT, INT, etc) through node.getJsonObject(key): In case it returns null, check the next admitted key, and stop at the first you find.
On each operation, a proper logic must be coded:
In case the key is a constant value (INT), return its value immediately.
In case the key is an operation, check the value's type (through node.getValueType()): If it is a single value, return it as is. If it is an array, iterate through its elements and call evaluate for each one of them, and perform the proper operation with the returned value (adding, multiplying, etc). Last, return the computation's result.
After your first edit
Your first real approach looks OK; I'd just suggest you some improvements to make the code more readable:
Use an enhanced for.
Replace the if-else chanin by a switch.
Replace each case by a call to a private method.
int result;
Set<String> keySet = node.keySet();
for (String key : keySet)
{
switch (key)
{
case "ADD":
result=evaluateAdd(node.getJsonArray("ADD"));
break;
case "MULT":
result=evaluateMult(node.getJsonArray("ADD"));
break;
case "INT":
result=node.getInt("INT");
break;
...
}
}

JSON [{
"name": "com",
"children": [
{
"name": "project",
"children": [
{
"name": "server"
},
{
"name": "client",
"children": [
{
"name": "util"
}
]
}
]
}
]
}]
Try this:
public static JSONArray getNames(JSONArray inputArray, JSONArray outputArray) {
for (int i = 0; i < inputArray.length(); i++) {
JSONObject inputObject = inputArray.getJSONObject(i);
JSONObject outputObject = new JSONObject();
outputObject.put("name", inputObject.getString("name"));
outputArray.put(outputObject);
if (inputObject.has("children")) {
JSONArray children = inputObject.getJSONArray("children");
getNames(children, outputArray);
}
}
return outputArray;
}

Get JSON String by iterating over array inside an object

Im working on an app where Im parsing JSON file and get the strings from it, but there is one String I have no idea why cant I get it into my activity.
the String is Object > Array > String
I have 2 activities and 1 model.
MainActivity: where Im parsing the JSON.
DetailActivity: where I need the String.
PostModel: a model where I have all setter and getter.
JSON:
{
"status":"ok",
"count":10,
"count_total":184,
"pages":19,
"posts":[
{ },
{
"id":2413,
,
"categories":[
{
"id":100,
"slug":"logging",
"title":"logging",
"description":"",
"parent":0,
"post_count":1
}
],
"comments":[
{
"id":3564,
"content":"<p>\u47 <\/p>\n",
"parent":0
}
],
"comment_count":1,
"thumbnail":"http:\/\/www.5.com\/wtent\g",
"custom_fields":{
"dsq_thread_id":[
"2365140"
],
"videoID":[
"--ffwf92jvDFy"
]
},
"thumbnail_images":{
"full":{
"url":"http:\/\/www.5.com\/jpg",
"width":727,
"height":454
},
"thumbnail":{
"url":"http:\/\/www.5.com\/wp-con50.jpg",
"width":150,
"height":150
}
}
}
]
}
PostModel:
private List<VidCast> videoIDList;
private String videoID;
public String getVideoID() {
return videoID;
}
public void setVideoID(String videoID) {
this.videoID = videoID;
}
public List<VidCast> getvideoIDList() { return videoIDList; }
public void setvideoIDList(List<VidCast> videoIDList) {
this.videoIDList = videoIDList;
}
public static class VidCast {
private String name;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
}
MainActivity:
List<PostModel.VidCast> vidCasts = JsonPath.parse(URL_TO_HIT).read("$.posts.[*].custom_fields.[*].videoID[*]");
vidCasts = new ArrayList<>();
for (int s = 0 ; s < finalObject.getJSONArray("custom_fields").length() ; s++){
PostModel.VidCast vidCast = new PostModel.VidCast();
vidCast.setName(videoID);
vidCasts.add(vidCast);
}
postModel.setvideoIDList(vidCasts);
// adding the final object in the list
postModelList.add(postModel);
}
return postModelList;
}
}
DetailActivity:
StringBuffer stringBuffer = new StringBuffer();
for(PostModel.CategoryCast categoryCast : postModel.getCategoryCastList()){
stringBuffer.append(categoryCast.getName() + ", ");
}
StringBuffer videoStringBuffer = new StringBuffer();
for(PostModel.VidCast videoIDList : postModel.getvideoIDList()) {
videoStringBuffer.append(videoStringBuffer.toString());
}
At the last file is where I need to get the <> String. I spent a lot of time I just cant figure it out how I can iterate over array inside an object.
Thanks in advance!
__________update________
I managed to parse it that way :
JSONObject customFields = finalObject.getJSONObject("custom_fields");
JSONArray vidCastsJson = customFields.getJSONArray("videoID");
List<PostModel.VidCast> videoIds = new ArrayList<>();
for (int s = 0 ; s < vidCastsJson.length() ; s++){
PostModel.VidCast vidCast = new PostModel.VidCast();
vidCast.setName(vidCastsJson.optString(s));
videoIds.add(vidCast);
String videoID = String.valueOf(vidCastsJson);
vidCast.setName(videoID);
and I use Stringbuffer at DetailActivityStringBuffer
videoStringBuffer = new StringBuffer();
for(PostModel.VidCast vidCast : postModel.getvideoIDList()) {
videoStringBuffer.append(vidCast.getName());
String videoID = vidCast.toString();
}
But now I'm getting the videoID with the array brackets like that ["F3lyzrt"] I want it as a string to be only F3lyzrt, so I can pass it to my youtube player. Any advice will be appropriated.
Thanks,

It would look something like this:
JSONObject root = // however you get your root JSON object
JSONArray posts = root.optJSONArray("posts");
for(int i=0; i < posts.length(); i++){
JSONObject post = posts.optJSONObject(i);
int id = post.optInt("id");
JSONArray categories = post.optJSONArray("categories");
// etc.
}
Though you might want to consider using GSON or Jackson. With those libraries you can define a model to represent the data (jsonschema2pojo.org can help with that) and then it does all the parsing for you.
EDIT
You're not even trying to get the video id. Here's your code:
for (int s = 0; s < finalObject.getJSONArray("videoID").length(); s++){
{
postModel.setVideoID(videoID);
postModelList.add(postModel);
}
You see how you're not retrieving the contents of the json array?
JSONArray videoIds = finalObject.getJSONArray("videoID");
for (int s = 0; s < videoIds.length(); s++){
String videoID = videoIds.optString(s);
postModel.setVideoID(videoID);
postModelList.add(postModel);
}

Parse the JSON String efficiently and in a clean way

In my below code, colData stores JSON String. Sample example for colData-
{"lv":[{"v":{"price":70.0,"userId":419},"cn":3},
{"v":{"price":149.99,"userId":419},"cn":3},
{"v":{"price":54.95,"userId":419},"cn":3}],
"lmd":20130206212543}
Now I am trying to match id value with userId value in the above JSON String. I am getting id value from a different source.
Meaning if id value is 419 then in the above JSON String userId value should also be 419. And in the JSON String, it might be possible there are lot of userId values so all the userId values should be matching with id. If any of them doesn't matches then log the exception.
So I was trying something like this-
final int id = generateRandomId(random);
for (String str : colData) {
if (!isJSONValid(str, id)) {
// log the exception here
LOG.error("Invalid JSON String " +str+ "with id" +id);
}
}
public boolean isJSONValid(final String str, final int id) {
boolean valid = false;
try {
final JSONObject obj = new JSONObject(str);
final JSONArray geodata = obj.getJSONArray("lv");
final int n = geodata.length();
for (int i = 0; i < n; ++i) {
final JSONObject person = geodata.getJSONObject(i);
JSONObject menu = person.getJSONObject("v");
if(menu.getInt("userId") == id) {
valid = true;
}
}
} catch (JSONException ex) {
valid = false;
}
return valid;
}
As per my understanding it looks like I can make isJSONValid method more cleaner. In my above isJSONValid method as I am repeating some stuff which I shouldn't be doing. Can anyone help me out how to make this more cleaner if I have missed anything. I will be able to learn some more stuff. Thanks for the help

You can initialize valid = true and set it to false when you find a non-valid userId and immediately fail:
public boolean isJSONValid(final String str, final int id) {
boolean valid = true;
try {
final JSONObject obj = new JSONObject(str);
final JSONArray geodata = obj.getJSONArray("lv");
final int n = geodata.length();
for (int i = 0; i < n; ++i) {
final JSONObject person = geodata.getJSONObject(i);
JSONObject menu = person.getJSONObject("v");
if(menu.getInt("userId") != id) {
valid = false;
break;
}
}
} catch (JSONException ex) {
valid = false;
}
return valid;
}
This way you iterate through all array's elements only if all are valid, which is the only case you actually have to.

How can I flag duplicates in an array?

I would like to know a method for flagging values in an array, removing the duplicates and combining some of the data in Java.
I am keeping a record of geo locations using lat, long and description this is encoded in a JSON array as follows:
[{"lon": 0.001, "lat": 0.001, "desc": test}, {"lon": 0.001, "lat": 0.001, "desc": test2}]
I would like to be able to remove the duplicate geo locations while keeping the "desc" part of the array, e.g.
[{"lon": 0.001, "lat": 0.001, "desc": test, test2}]
Edit:
This is what I am currently doing:
//Store locPoints from server in JSONArray
JSONArray jPointsArray = new JSONArray(serverData);
List<JSONObject> jObjects = new ArrayList<JSONObject>();
List<JSONObject> seenObjects = new ArrayList<JSONObject>();
for(int i = 0; i < jPointsArray.length(); ++i)
{
jObjects.add(jPointsArray.getJSONObject(i));
}
for (JSONObject obj : jObjects)
{
//This always returns true
if (!seenObjects.contains(obj))// && !seenObjects.contains(obj.get("lon")))
{
Log.i("Sucess", "Huzzah!");
seenObjects.add(obj);
}
else
{
//merge the 'desc' field in 'obj' with the 'desc' field in
JSONObject original = (JSONObject)seenObjects.get(seenObjects.indexOf(obj));
JSONObject update = obj;
original.put("desc", original.get("desc") + ", " + update.get("desc"));
seenObjects.get(seenObjects.indexOf(obj)).get("desc"));
}
}

You could do something like:
//assuming that the array you are filtering is called 'myArray'
List<Object> seenObjects = new ArrayList<Object>();
for (Object obj : myArray) {
if (! seenObjects.contains(obj)) {
seenObjects.add(obj);
}
else {
//merge the 'desc' field in 'obj' with the 'desc' field in
//'seenObjects.get(seenObjects.indexOf(obj))'
}
}
Note that this will only work if the objects you are comparing have implementations of equals() and hashCode() that do what you want (in your case, they should only take into consideration the 'lat' and 'lon' fields).
Update:
Here is some complete example code:
import java.util.ArrayList;
import java.util.List;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
public class JsonMergeTest {
#SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] args) {
List<Object> myArray = new ArrayList<Object>();
myArray.add(MyJsonObject.parse("{\"lon\": 0.001, \"lat\": 0.001, \"desc\": \"test\"}"));
myArray.add(MyJsonObject.parse("{\"lon\": 0.001, \"lat\": 0.001, \"desc\": \"test2\"}"));
List seenObjects = new ArrayList<Object>();
for (Object obj : myArray) {
if (! seenObjects.contains(obj)) {
seenObjects.add(obj);
}
else {
//merge the 'desc' field in 'obj' with the 'desc' field in the list
MyJsonObject original = (MyJsonObject)seenObjects.get(seenObjects.indexOf(obj));
MyJsonObject update = (MyJsonObject)obj;
original.put("desc", original.get("desc") + ", " + update.get("desc"));
}
}
for (MyJsonObject obj : (List<MyJsonObject>)seenObjects) {
System.out.println(obj.toJSONString());
}
}
private static class MyJsonObject extends JSONObject {
#Override
public boolean equals(Object obj) {
if (obj == null || ! (obj instanceof MyJsonObject) || ! this.containsKey("lat") || ! this.containsKey("lon")) {
return super.equals(obj);
}
MyJsonObject jsonObj = (MyJsonObject)obj;
return this.get("lat").equals(jsonObj.get("lat")) && this.get("lon").equals(jsonObj.get("lon"));
}
#Override
public int hashCode() {
if (! this.containsKey("lat") || ! this.containsKey("lon")) {
return super.hashCode();
}
return this.get("lat").hashCode() ^ this.get("lon").hashCode();
}
#SuppressWarnings("unchecked")
public static Object parse(String json) {
Object parsedJson = JSONValue.parse(json);
if (! (parsedJson instanceof JSONObject)) {
return parsedJson;
}
MyJsonObject result = new MyJsonObject();
result.putAll((JSONObject)parsedJson);
return result;
}
}
}

You can use GSon. And follow the steps:
1. Define an equivalent POJO in Java, to map the JSON String
public class Location implements Comparable<Location> {
public String lon;
public String lat;
public String desc;
#Override
public String toString() {
return "<lon: " + lon +", lat: "+ lat +", desc: " + desc +">";
}
#Override
public boolean equals(Object obj) {
return ((Location)obj).lon.equals(lon) && ((Location)obj).lat.equals(lat);
}
public int compareTo(Location obj) {
return ((Location)obj).lon.compareTo(lon) + ((Location)obj).lat.compareTo(lat);
}
}
2. Write the code that merges similar location. OK, it's Sunday, lets do it :)
public static void main(String[] args){
//Some test data
String s = "[" +
" {\"lon\": 0.001, \"lat\": 0.001, \"desc\": \"test\"}," +
" {\"lon\": 0.002, \"lat\": 0.001, \"desc\": \"test3\"}," +
" {\"lon\": 0.002, \"lat\": 0.005, \"desc\": \"test4\"}," +
" {\"lon\": 0.002, \"lat\": 0.001, \"desc\": \"test5\"}," +
" {\"lon\": 0.001, \"lat\": 0.001, \"desc\": \"test2\"}]";
Gson gson = new Gson();
Location[] al = gson.fromJson(s, Location[].class);
List<Location> tl = Arrays.asList(al);
//lets sort so that similar locations are grouped
Collections.sort(tl);
List<Location> fl = new ArrayList<Location>();
Location current = null;
//merge!
for(Iterator<Location> it = tl.iterator(); it.hasNext();){
current = current==null?it.next():current;
Location ltmp = null;
while(it.hasNext() && (ltmp = it.next()).equals(current))
current.desc = current.desc + "," + ltmp.desc;
fl.add(current);
current = ltmp;
}
//convert back to JSON?
System.out.println(gson.toJson(fl));
}
3. output
[{"lon":"0.002","lat":"0.005","desc":"test4"},
{"lon":"0.002","lat":"0.001","desc":"test3,test5"},
{"lon":"0.001","lat":"0.001","desc":"test,test2"}]