We Keep Coding

WebFlux read MultipartFormData is not formFiled when one part subContent is application/json

i try to migration a servlet project to webflux，but a MultipartFormData parse is different from them.
send request
#Test
public void testMutiplepart1() {
String url = "http://localhost:8081/multipartPart?category_id=115348&app_id=1000912&timestamp=1673339039&sig=041b5b6e4e6eae430208f9fbc45dc3a8";
MultiValueMap<String, Object> map = new LinkedMultiValueMap<>(5);
map.add("category_id", "115348");
map.add("app_id", 1000912);
map.add("app_food_code", "1253");
map.add("timestamp", 1673339039);
map.add("sig", "041b5b6e4e6eae430208f9fbc45dc3a8");
String result = restTemplate.postForObject(url, map, String.class);
System.out.println(result);
}
parse multipValue
app_id and timestamp is not formField
private Map<String, String> getFormData(String path,
MultiValueMap<String, Part> partMultiValueMap) {
if (partMultiValueMap != null) {
Map<String, String> formData = new HashMap<>();
Map<String, Part> multiPartMap = partMultiValueMap.toSingleValueMap();
for (Map.Entry<String, Part> partEntry : multiPartMap.entrySet()) {
Part part = partEntry.getValue();
if (part instanceof FormFieldPart) {
formData.put(partEntry.getKey(), ((FormFieldPart) part).value());
} else {
log.warn("not support file part , uri:{} key:{} name:{}", path, partEntry.getKey(), part.name());
}
}
return formData;
}
return null;
}
webFlux judge a formField code
private static boolean isFormField(HttpHeaders headers) {
MediaType contentType = headers.getContentType();
//MediType is Text/Plain subContent
return (contentType == null || MediaType.TEXT_PLAIN.equalsTypeAndSubtype(contentType))
&& headers.getContentDisposition().getFilename() == null;
}
servlet judge formFiled code
String fieldName = getFieldName(headers);
if (fieldName != null) {
String subContentType = headers.getHeader(CONTENT_TYPE);
if (subContentType != null
&& subContentType.toLowerCase(Locale.ENGLISH)
.startsWith(MULTIPART_MIXED)) {
currentFieldName = fieldName;
// Multiple files associated with this field name
byte[] subBoundary = getBoundary(subContentType);
multi.setBoundary(subBoundary);
skipPreamble = true;
continue;
}
String fileName = getFileName(headers);
currentItem = new FileItemStreamImpl(fileName,
fieldName, headers.getHeader(CONTENT_TYPE),
//if fileName is null , the field is formField
fileName == null, getContentLength(headers));
currentItem.setHeaders(headers);
notifier.noteItem();
itemValid = true;
return true;
}
how can i parse the value ?

Hive UDF : Generic UDF cannot access struct from nested map

here is my hive table
create table if not exists dumdum (val map<string,map<string,struct<student_id:string,age:int>>>);
insert into dumdum select map('A',map('1',named_struct('student_id','123a', 'age',11)));
insert into dumdum select map('B',map('2',named_struct('student_id','987z', 'age',11)));
select * from dumdum;
and i see
{"A":{"1":{"student_id":"123a","age":11}}}
{"B":{"2":{"student_id":"987z","age":11}}}
I want to extract all the student_id from the inner map i.e. 123a and 987z. So here is what i want to do
select some_udf(val) from dumdum;
and the result should be
["123a","987z"]
Here is the Java UDF i wrote
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.*;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class CustomUDF extends GenericUDF {
private MapObjectInspector inputMapOI = null;
private Converter inputMapKeyConverter = null;
private MapObjectInspector inputMapValueMapOI = null;
private Converter inputMapValueConverter;
#Override
public String getDisplayString(String[] arguments) {
return "my udf";
}
#Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
if ((null == arguments) || (arguments.length != 1)) {
throw new UDFArgumentLengthException("1 arguments are expected.");
}
if (!(arguments[0] instanceof MapObjectInspector)) {
throw new UDFArgumentException("The first parameter should be a map object ");
}
inputMapOI = (MapObjectInspector) arguments[0];
ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
inputMapKeyConverter = ObjectInspectorConverters.getConverter(this.inputMapOI.getMapKeyObjectInspector(), mapKeyOI);
if (!(inputMapOI.getMapValueObjectInspector() instanceof MapObjectInspector)) {
throw new UDFArgumentException("The map value type must be a map ");
}
inputMapValueMapOI = (MapObjectInspector) this.inputMapOI.getMapValueObjectInspector();
List<String> structFieldNames = new ArrayList<String>();
structFieldNames.add("student_id");
structFieldNames.add("age");
List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
ObjectInspector inputMapElementOI = inputMapValueMapOI.getMapValueObjectInspector();
ObjectInspector outputMapElementOI = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
inputMapValueConverter = ObjectInspectorConverters.getConverter(inputMapElementOI, outputMapElementOI);
return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
}
#Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
if ((null == arguments) || (arguments.length != 1)) {
throw new UDFArgumentLengthException("1 argument is expected.");
}
Map<?, ?> map = inputMapOI.getMap(arguments[0].get());
List<String> dataList = new ArrayList<String>();
for (Object key : map.keySet()) {
Map<?, ?> valueMap = this.inputMapValueMapOI.getMap(map.get(key));
if ((valueMap == null) || (valueMap.size() == 0)) {
continue;
}
for (Object value : valueMap.keySet()) {
try{
String innerkey = (String) this.inputMapKeyConverter.convert(value);
System.out.println("Got "+innerKey);
Map<?, ?> innerMap = (Map<?, ?>) this.inputMapValueMapOI.getMap(valueMap.get(key));
if ((innerMap == null) || (innerMap.size() == 0)) {
System.out.println("Got null");
continue;
}
for (Object struct : innerMap.keySet()) {
String strValue = (String) this.inputMapValueConverter.convert(struct);
StructField str = (StructField) inputMapValueConverter.convert(innerMap.get(strValue));
/*
Not sure what to do here. Maybe
str.getFieldID();
dataList.add(str.toString());
*/
}
}
catch (ClassCastException c){
System.out.println("Got ClassCastException");
}
}
}
return dataList;
}
}
when i invoke it as
add jar /path/to/my/jar;
CREATE TEMPORARY FUNCTION modudf AS 'some.package.CustomUDF';
select modudf(val) from dumdum;
i never get past
Map<?, ?> innerMap = (Map<?, ?>) this.inputMapValueMapOI.getMap(valueMap.get(inner));
if ((innerMap == null) || (innerMap.size() == 0)) {
System.out.println("Got null");
continue;
}
I can see the output of
System.out.println("Got "+innerKey);
on the console.
Why cant my converter access the inner map ?
Also, how will i dereference the StructField once i am able to access the inner map ?
Update
Thanks serge_k for the suggestion. I'm afraid i still need one converter else i wont be able to get the key. here is what i tried
First i defined the second map inspector and the struct inspector as
inputMapValueMapOI = (MapObjectInspector) this.inputMapOI.getMapValueObjectInspector();
List<String> structFieldNames = new ArrayList<String>();
structFieldNames.add("student_id");
structFieldNames.add("age");
List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
structOI = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
then here is what i tried
String innerkey = (String) inputMapKeyConverter.convert(value);
System.out.println(innerKey);
Map<?, ?> innerMap = (Map<?, ?>) this.inputMapValueMapOI.getMap(valueMap.get(innerkey));
if ((innerMap == null) || (innerMap.size() == 0)) {
System.out.println("null inner map");
continue;
}
for (Object struct : innerMap.keySet()) {
String ikey = (String) inputMapKeyConverter.convert(struct);
Object obj = structOI.getStructFieldData(innerMap.get(ikey), structOI.getStructFieldRef("student_id"));
dataList.add(obj.toString());
}
but i still see
null inner map
have i not defined the inner map inspector properly ?

I would recommend you not to use converters, just define second MapObjectInspector for the inner map, get outer map value and call getMap like for the first map. To get the struct values you need to define a variable of StructObjectInspector type in initialize, e.g.
StructObjectInspector soi = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors)
then
Object obj = soi.getStructFieldData(innerMapValue, soi.getStructFieldRef("student_id"))
Update:
Try to convert map keys to standard keys as follows
private Map stdKeys(Map inspectMap) {
Map objMap = new HashMap();
for (Object inspKey : inspectMap.keySet()) {
Object objKey = ((PrimitiveObjectInspector) mapInspector.getMapKeyObjectInspector()).getPrimitiveJavaObject(inspKey);
objMap.put(objKey, inspKey);
}
return objMap;
}
See https://github.com/klout/brickhouse/blob/master/src/main/java/brickhouse/udf/collect/MapRemoveKeysUDF.java for more details

How to write common custom GSON deserializer for all fields of a class?

I have a class MySimpleObject that have various member fields. Given a json, it will populate the field accordingly. However if the json is stated as "nil", I plan to set it to null instead of string "nil".
The below example should result is an MySimpleObject with null for all it's fields, and a 0 length list of subItemList. myObj1 should be equal to myObj2.
#Test
public void myTestFunction() {
String myJson1 = "{\"item1\":\"nil\",\"item2\":\"nil\",\"subItemList\":[{\"subItem1\":\"nil\",\"subItem2\":\"nil\"}]}";
String myJson2 = "{\"subItemList\":[]}";
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.registerTypeAdapter(new TypeToken<List<MySubItems>>(){ }.getType(), new MyOwnListDeserializer());
gsonBuilder.registerTypeAdapter(String.class, new MyOwnStringDeserializer());
Gson gson = gsonBuilder.create();
MySimpleObject myObj1 = gson.fromJson(myJson1, MySimpleObject.class);
MySimpleObject myObj2 = gson.fromJson(myJson2, MySimpleObject.class);
assertThat(myObj1.equals((myObj2))).isTrue();
}
class MySimpleObject implements Serializable {
String item1 = null;
String item2 = null;
List<MySubItems> subItemList;
#Override
public int hashCode() {
int hash = 17;
hash = 31*hash + ((item1 == null)? 0 :item1.hashCode());
hash = 31*hash + ((item2 == null)? 0 :item2.hashCode());
return hash;
}
#Override
public boolean equals(Object obj) {
if (obj instanceof MySimpleObject) {
return this.hashCode() == obj.hashCode();
}
return super.equals(obj);
}
}
class MySubItems implements Serializable {
String subItem1 = null;
String subItem2 = null;
#Override
public int hashCode() {
int hash = 17;
hash = 31*hash + ((subItem1 == null)? 0 :subItem1.hashCode());
hash = 31*hash + ((subItem2 == null)? 0 :subItem2.hashCode());
return hash;
}
#Override
public boolean equals(Object obj) {
if (obj instanceof MySubItems) {
return this.hashCode() == obj.hashCode();
}
return super.equals(obj);
}
}
How to write the custom serializer without having to loop through each jsonObject and check for "nil" to set to null?

I looked at the Gson library and also the gson-fire project, but none of them seem to allow for a real generic (and performant) solution.
One way to go is to systematically replace "nil" by "null" in the json string before passing it to the gson object. It is not very clean, but it is quite performant and could work.
Here is a basic method (must be refined):
public static String convertNil( String json ){
return json.replaceAll( ":\\s*\"nil\"", ": null" );
}
Then use it like:
MySimpleObject myObj1 = gson.fromJson( convertNil( myJson1 ), MySimpleObject.class );
MySimpleObject myObj2 = gson.fromJson( convertNil( myJson2 ), MySimpleObject.class );

I manage to come up with some custom deserialization that do the job as below.
#Test
public void myTestFunction() {
String myJson1 = "{\"item1\":\"nil\",\"item2\":\"nil\",\"subItemList\":[{\"subItem1\":\"nil\",\"subItem2\":\"nil\"}]}";
String myJson2 = "{\"subItemList\":[]}";
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.registerTypeAdapter(new TypeToken<List<MySubItems>>(){ }.getType(), new MyOwnListDeserializer());
gsonBuilder.registerTypeAdapter(String.class, new MyOwnStringDeserializer());
Gson gson = gsonBuilder.create();
MySimpleObject myObj1 = gson.fromJson(myJson1, MySimpleObject.class);
MySimpleObject myObj2 = gson.fromJson(myJson2, MySimpleObject.class);
assertThat(myObj1.equals((myObj2))).isTrue();
}
class MySimpleObject implements Serializable {
String item1 = null;
String item2 = null;
List<MySubItems> subItemList;
#Override
public int hashCode() {
int hash = 17;
hash = 31*hash + ((item1 == null)? 0 :item1.hashCode());
hash = 31*hash + ((item2 == null)? 0 :item2.hashCode());
return hash;
}
#Override
public boolean equals(Object obj) {
if (obj instanceof MySimpleObject) {
return this.hashCode() == obj.hashCode();
}
return super.equals(obj);
}
}
class MySubItems implements Serializable {
String subItem1 = null;
String subItem2 = null;
#Override
public int hashCode() {
int hash = 17;
hash = 31*hash + ((subItem1 == null)? 0 :subItem1.hashCode());
hash = 31*hash + ((subItem2 == null)? 0 :subItem2.hashCode());
return hash;
}
#Override
public boolean equals(Object obj) {
if (obj instanceof MySubItems) {
return this.hashCode() == obj.hashCode();
}
return super.equals(obj);
}
}
class MyOwnStringDeserializer implements JsonDeserializer<String> {
#Override
public String deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
return (json.getAsString().equals("nil"))? null : json.getAsString();
}
}
class MyOwnListDeserializer implements JsonDeserializer<List<MySubItems>> {
#Override
public List<MySubItems> deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
List<MySubItems> list = new ArrayList<>();
for (JsonElement element : json.getAsJsonArray()) {
JsonObject subObj = element.getAsJsonObject();
MySubItems subItems = new MySubItems();
if (!subObj.get("subItem1").getAsString().equals("nil")) {
subItems.subItem1 = subObj.get("subItem1").getAsString();
}
if (!subObj.get("subItem2").getAsString().equals("nil")) {
subItems.subItem2 = subObj.get("subItem1").getAsString();
}
if (subItems.subItem1 != null || subItems.subItem2 != null) {
list.add(subItems);
}
}
return (list.size() == 0)? null : list;
}
}
I'm still not very happy that MyOwnListDeserializer, has to manually handle subItem1 and subItem2, despite they should have the same rule as defined by MyOwnStringDeserializer. But I just don't know how to apply MyOwnStringDeserializer to MyOwnListDeserializer.
Would be still open for other better answers than mine.
Update A more optimized answer could be found in
https://stackoverflow.com/a/39671580/3286489

Find value for a key from a dynamic json using java

I need to get the value of the key from a dynamic json.
Input-> json Object, String key
Output-> json element(corresponds to the value of the key)
Example
JsonObject Jmsg =
{
"id": "1753_CORE1",
"name": "Gtx cuda Service:1753",
"shortName": "gt-service-1753",
"createdDate": "Mar 31, 2015 4:47:10 PM",
"config": {
"oauthSecret": [
{
"id": 45,
"config123": {
"oauthSecret": "P8n2x5Hsst0nFRRB0A",
"status": "CREATED"
},
"SERVER132": "1000"
},
{
"id": 46,
"config123": {
"oauthSecret": "P8n2x5Htss0nFRRB0A"
},
"SERVER132": "1000"
}
],
"oauthKey": "154284-service-1753",
"SERVER": "1000"
},
"features": [
9004,
9005
]
}
and String key = "status";
then
JsonElement Jvalue = jsonGetValueformKey(Jmsg,key);
should return 'CREATED' in JsonElement or string type.
if String key = "features";
then
JsonElement Jvalue = jsonGetValueformKey(Jmsg,key);
should return [9004,9005] in JsonElement or jsonArray type.
if key not found then return null
JsonObject Jmsg can be anything

please try this
package json;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
public class MyApp {
static List<String> list = new ArrayList<String>();
public static void main(String[] args) {
String key = "oauthSecret";
String json2 = "{\"config\": {\"oauthSecret\": [{\"id\": 45,\"config123\": {\"oauthSecret\": \"P8n2x5Ht0nFRRB0A\",\"status\": \"CREATED\"},\"SERVER132\": \"1000\"},{\"id\": 46,\"config123\": {\"oauthSecret\": \"wP8n2x5Ht0nFRRB0A\",\"status\": \"CREATED\"},\"SERVER132\": \"1000\"}],\"oauthKey\": \"newtest\",\"SERVER\": \"1000\"},\"features\": [ 9004, 9005] ,\"d\":\"dd\"}";
System.out.println("JSON: " + json2);
JsonParser p = new JsonParser();
check(key, p.parse(json2));
System.out.println("list size: " + list.size());
System.out.println(list);
}
private static void check(String key, JsonElement jsonElement) {
if (jsonElement.isJsonArray()) {
for (JsonElement jsonElement1 : jsonElement.getAsJsonArray()) {
check(key, jsonElement1);
}
} else {
if (jsonElement.isJsonObject()) {
Set<Map.Entry<String, JsonElement>> entrySet = jsonElement
.getAsJsonObject().entrySet();
for (Map.Entry<String, JsonElement> entry : entrySet) {
String key1 = entry.getKey();
if (key1.equals(key)) {
list.add(entry.getValue().toString());
}
check(key, entry.getValue());
}
} else {
if (jsonElement.toString().equals(key)) {
list.add(jsonElement.toString());
}
}
}
}
}

This is a draft of a recursive approach for that.
Object find(JSONObject jObj, String k) throws JSONException {
Iterator<?> keys = jObj.keys();
while( keys.hasNext() ) {
String key = (String)keys.next();
if (key.equals(k)) {
return jObj.get(key);
}
if ( jObj.get(key) instanceof JSONObject ) {
return find((JSONObject)jObj.get(key), k);
}
if ( jObj.get(key) instanceof JSONArray ) {
JSONArray jar = (JSONArray)jObj.get(key);
for (int i = 0; i < jar.length(); i++) {
JSONObject j = jar.getJSONObject(i);
find(j, k);
}
}
}

You could use something like this :-
public Object checkKey(JSONObject object, String searchedKey) {
boolean exists = object.containsKey(searchedKey);
Object obj = null;
if(exists){
obj = object.get(searchedKey);
}
if(!exists) {
Set<String> keys = object.keySet();
for(String key : keys){
if ( object.get(key) instanceof JSONObject ) {
obj = checkKey((JSONObject)object.get(key), searchedKey);
}
}
}
return obj;
}
This will give you the Object type for a key OR null if key does not exists.
You can modify it & caste the return Object type to any JSONObject, String or JSONArray depending upon your condition by checking its class using getClass().
Note :- This is just a reference but you can edit it according to your needs.

private Object findJsonParam(JSONObject payload, String param) {
Iterator<?> keys = payload.keys();
System.out.println("payload " + payload);
while (keys.hasNext()) {
String key = (String) keys.next();
if (key.equals(param)) {
return payload.get(key);
} else if (payload.get(key) instanceof JSONObject) {
Object val = findJsonParam(payload.getJSONObject(key), param);
if (val != null)
return val;
} else if (payload.get(key) instanceof JSONArray) {
JSONArray jar = payload.getJSONArray(key);
for (Object jsonObj : jar) {
Object val = findJsonParam((JSONObject) jsonObj, param);
if (val != null)
return val;
}
}
}
return null;
}

I had to make some changes in DrB awnser, it worked for me this way. Because on the first nested json it returned without checking the value, so if the key wasn't in the first one, then it couldn't find it.
public Object find(JSONObject jObj, String k) throws JSONException {
Iterator<?> keys = jObj.keys();
while (keys.hasNext()) {
String key = (String) keys.next();
if (key.equals(k)) {
return jObj.get(key);
}
if (jObj.get(key) instanceof JSONObject) {
Object probableKeyValue = find((JSONObject) jObj.get(key), k);
if(probableKeyValue != null){
return probableKeyValue;
}
}
if (jObj.get(key) instanceof JSONArray) {
JSONArray jar = (JSONArray) jObj.get(key);
for (int i = 0; i < jar.length(); i++) {
JSONObject j = jar.getJSONObject(i);
find(j, k);
}
}
}
return null;
}

How to show JSON fields hierarchically in java

I have a JSON object and want to show the fields hierarchically.
checkedTreeSelectionDialog.setInput(jsonObject);
Inside the getchildren() method of the JsonContentProvider class, I am parsing the json object
private void getChildrenForJson(final Object object) {
final JSONObject jsonObject = (JSONObject) object;
for (final Object objct : jsonObject.keySet()) {
try {
if (jsonObject.get(objct) instanceof JSONArray) {
getArray(jsonObject.get(objct));
} else {
if (jsonObject.get(objct) instanceof JSONObject) {
getChildrenForJson(jsonObject.get(objct));
}
}
However, the returned fields are not coming hierarchically.
How can I get those fields hierarchically?

function sortObject(o) {
var sorted = {},
key, a = [];
for (key in o) {
if (o.hasOwnProperty(key)) {
a.push(key);
}
}
a.sort();
for (key = 0; key < a.length; key++) {
sorted[a[key]] = o[a[key]];
}
return sorted;
}
private void getChildrenForJson(final Object object) {
final JSONObject jsonObject = (JSONObject) object;
for (final Object objct : jsonObject.keySet()) {
try {
if (jsonObject.get(objct) instanceof JSONArray) {
var q = getArray(jsonObject.get(objct));
var p = sortObject(q);
console.log(p);
} else {
if (jsonObject.get(objct) instanceof JSONObject) {
var q =getChildrenForJson(jsonObject.get(objct));
var p = sortObject(q);
console.log(p);
}
}
This is what you were expecting , I hope !