AWS4 Signature with Java for Browser Based Post - java

I'm following the example above to create a "Base64-encoded policy string" and "signature-value" with a Java backend: http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-post-example.html
But my "signature-value" doesn't match the example.
Am I missing something?
I believe the getSignatureKey and HmacSHA256 are OK, because I found in a trusted source. Maybe the getStringToSign is not correct.
Here is my code:
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import static com.google.common.io.BaseEncoding.base16;
public class AwsSignatureGenerator {
public static void main(String[] args) throws Exception {
String signature = getSignature(getSignatureKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
"20151229",
"us-east-1",
"s3"));
System.out.println(signature);
}
public static String getStringToSign(){
String s3Policy = "{ \"expiration\": \"2015-12-30T12:00:00.000Z\", \"conditions\": [ {\"bucket\": \"sigv4examplebucket\"}, [\"starts-with\", \"$key\", \"user/user1/\"], {\"acl\": \"public-read\"}, {\"success_action_redirect\": \"http://sigv4examplebucket.s3.amazonaws.com/successful_upload.html\"}, [\"starts-with\", \"$Content-Type\", \"image/\"], {\"x-amz-meta-uuid\": \"14365123651274\"}, {\"x-amz-server-side-encryption\": \"AES256\"}, [\"starts-with\", \"$x-amz-meta-tag\", \"\"], {\"x-amz-credential\": \"AKIAIOSFODNN7EXAMPLE/20151229/us-east-1/s3/aws4_request\"}, {\"x-amz-algorithm\": \"AWS4-HMAC-SHA256\"}, {\"x-amz-date\": \"20151229T000000Z\" } ] }";
return new Base64().encodeAsString(s3Policy.getBytes());
}
public static byte[] HmacSHA256(String data, byte[] key) throws Exception {
String algorithm="HmacSHA256";
Mac mac = Mac.getInstance(algorithm);
mac.init(new SecretKeySpec(key, algorithm));
return mac.doFinal(data.getBytes("UTF8"));
}
public static byte[] getSignatureKey(String key, String dateStamp, String regionName, String serviceName) throws Exception {
byte[] kSecret = ("AWS4" + key).getBytes("UTF8");
byte[] kDate = HmacSHA256(dateStamp, kSecret);
byte[] kRegion = HmacSHA256(regionName, kDate);
byte[] kService = HmacSHA256(serviceName, kRegion);
byte[] kSigning = HmacSHA256("aws4_request", kService);
return kSigning;
}
public static String getSignature(byte[] key) throws Exception{
return base16().lowerCase().encode(HmacSHA256(getStringToSign(), key));
}
}

I checked your policy
In base64 it looks like this:
eyAiZXhwaXJhdGlvbiI6ICIyMDE1LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAic2lndjRleGFtcGxlYnVja2V0In0sIFsic3RhcnRzLXdpdGgiLCAiJGtleSIsICJ1c2VyL3VzZXIxLyJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly9zaWd2NGV4YW1wbGVidWNrZXQuczMuYW1hem9uYXdzLmNvbS9zdWNjZXNzZnVsX3VwbG9hZC5odG1sIn0sIFsic3RhcnRzLXdpdGgiLCAiJENvbnRlbnQtVHlwZSIsICJpbWFnZS8iXSwgeyJ4LWFtei1tZXRhLXV1aWQiOiAiMTQzNjUxMjM2NTEyNzQifSwgeyJ4LWFtei1zZXJ2ZXItc2lkZS1lbmNyeXB0aW9uIjogIkFFUzI1NiJ9LCBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1tZXRhLXRhZyIsICIiXSwgeyJ4LWFtei1jcmVkZW50aWFsIjogIkFLSUFJT1NGT0ROTjdFWEFNUExFLzIwMTUxMjI5L3VzLWVhc3QtMS9zMy9hd3M0X3JlcXVlc3QifSwgeyJ4LWFtei1hbGdvcml0aG0iOiAiQVdTNC1ITUFDLVNIQTI1NiJ9LCB7IngtYW16LWRhdGUiOiAiMjAxNTEyMjlUMDAwMDAwWiIgfSBdIH0=
In JSON, it looks like this:
{
"expiration": "2015-12-30T12:00:00.000Z",
"conditions": [
{
"bucket": "sigv4examplebucket"
},
[
"starts-with",
"$key",
"user/user1/"
],
{
"acl": "public-read"
},
{
"success_action_redirect": "http://sigv4examplebucket.s3.amazonaws.com/successful_upload.html"
},
[
"starts-with",
"$Content-Type",
"image/"
],
{
"x-amz-meta-uuid": "14365123651274"
},
{
"x-amz-server-side-encryption": "AES256"
},
[
"starts-with",
"$x-amz-meta-tag",
""
],
{
"x-amz-credential": "AKIAIOSFODNN7EXAMPLE/20151229/us-east-1/s3/aws4_request"
},
{
"x-amz-algorithm": "AWS4-HMAC-SHA256"
},
{
"x-amz-date": "20151229T000000Z"
}
]
}
Based on your JSON and keys, the correct signature should be:
eb9df8c47e9c4bd6090809ed799f0146c71a10f7707aa1acc2c13e736d41ed54
As per this application that I created -> http://ttwd80.github.io/s3postcalculatorverify/
I would however skip
x-amz-date and x-amz-meta-uuid
Also note that the "file" field must be the last input in the form (aside from submit button).

Related

Token validation failed Visa X-pay

I need to tweak an API https://sandbox.api.visa.com/cybersource/payments/flex/v1/keys?apikey={apikey}
I am imitating the official document X-Pay Token,but it fail with "Token validation failed" error.
{
"responseStatus": {
"status": 401,
"code": "9159",
"severity": "ERROR",
"message": "Token validation failed",
"info": ""
}
}
Below is my x-pay-token generation code.
import java.math.BigInteger;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.SignatureException;
public class T {
private static String resoucePath = "payments/flex/v1/keys";
private static String queryString = "apikey=6DC0NMXO53QQFE6NFOLE213HXA-pvG6xE-1NtuCd5oOQr-O-s";
private static String requestBody = "{encryptionType:RsaOaep256}";
private static String sharedSecret = "gAynzAGf89+V}3{Q4Jx5cp-/R#Y#PEv#1XvxnjQC";
public static void main(String[] args) throws SignatureException {
System.out.println(T.generateXpaytoken(resoucePath, queryString, requestBody, sharedSecret));
}
public static String generateXpaytoken(String resourcePath, String queryString, String requestBody, String sharedSecret) throws SignatureException {
String timestamp = timeStamp();
String beforeHash = timestamp + resourcePath + queryString + requestBody;
String hash = hmacSha256Digest(beforeHash, sharedSecret);
String token = "xv2:" + timestamp + ":" + hash;
return token;
}
private static String timeStamp() {
return String.valueOf(System.currentTimeMillis() / 1000L);
}
private static String hmacSha256Digest(String data, String sharedSecret) throws SignatureException {
return getDigest("HmacSHA256", sharedSecret, data, true);
}
private static String getDigest(String algorithm, String sharedSecret, String data, boolean toLower) throws SignatureException {
try {
Mac sha256HMAC = Mac.getInstance(algorithm);
SecretKeySpec secretKey = new SecretKeySpec(sharedSecret.getBytes(StandardCharsets.UTF_8), algorithm);
sha256HMAC.init(secretKey);
byte[] hashByte = sha256HMAC.doFinal(data.getBytes(StandardCharsets.UTF_8));
String hashString = toHex(hashByte);
return toLower ? hashString.toLowerCase() : hashString;
} catch (Exception e) {
throw new SignatureException(e);
}
}
private static String toHex(byte[] bytes) {
BigInteger bi = new BigInteger(1, bytes);
return String.format("%0" + (bytes.length << 1) + "X", bi);
}
}
somebody can help me please?
The URL you are using:
https://sandbox.api.visa.com/cybersource/payments/flex/v1/keys?apikey={apikey}
Should be:
https://sandbox.api.visa.com/cybersource/payments/flex/v1/keys?apikey=6DC0NMXO53QQFE6NFOLE213HXA-pvG6xE-1NtuCd5oOQr-O-s
As on Feb 2020, If some one still has issue below points can help resolve issue.
Please make sure you have generated API Key and Secret for your sandbox project.
You can get these details in Dashboard -> Project -> Credentials -> Inbound and Authentication Keys -> API Key / Secret.
Please check the "Status" of the Key which should be Active.
If your "Credentials" tab does not have details for "Inbound and Authentication Keys" Please make sure to add the respective API then this section automatically appears.
Visa has "Visa Developer Center PlayGround" [similar to SoapUI/Postman] tool where you can easily test your API's. Unfortunately this is only supported with Windows as on Feb 2020, In future they may release the same for Mac/Linux too.
You can find this tool in Dashboard -> Project -> Assets -> Bottom of the page.

How to convert list of filepaths to a JSON object in Java

I've a requirement to convert a set of file structure obtained from DB into a JSON.
For example:
From DB, I get the following path and file attributes:
Record 1:
"path": "/sd/card/camera/pics/selfie.jpg"
"fileName": "selfie.jpg",
"mimeType": "image/jpeg"
Record 2:
"path": "/sd/card/camera/pics/personal/selfie1.jpg"
"fileName": "selfie1.jpg",
"mimeType": "image/jpeg"
and so on..
I need to convert this to a JSON like:
[{
"sd": [{
"card": [{
"camera": [{
"pics": [{
"fileName": "selfie.jpg",
"path": "/sd/card/camera/pics/selfie.jpg",
"mimeType": "image/jpeg"
},
{
"personal": [{
"fileName": "selfie1.jpg",
"path": "/sd/card/camera/pics/personal/selfie1.jpg",
"mimeType": "image/jpeg"
}]
}
]
}]
}]
}]
}]
I'm going to give you a 'jackson' solution.
First, build an object (or more, I let you deal with the Java inheritance, or use any kind of structure you want to use).
Like this one by example :
#JsonSerialize(using = CustomSerializer.class)
public class Something {
private String currentFolder; // Name of the folder if this instance of something is a folder
private Something[] childs;
private Map<String,String> currentPicture; // Picture properties if this instance of something is a picture
public Something() {currentPicture = new HashMap<String,String>();}
public Something[] getChilds() {
return childs;
}
public void setContent(Something[] _childs) {this.childs = _childs;}
public String getCurrentFolder() {return currentFolder;}
public void setCurrentFolder(String _currentFolder) {this.currentFolder = _currentFolder;}
public Map<String,String> getCurrentPicture() {return currentPicture;}
public void setCurrentPicture(Map<String,String> currentPicture) {this.currentPicture = currentPicture;}
}
Then, create the CustomSerializer, that will help you do whatever you want to do :
public class CustomSerializer extends JsonSerializer<Something>{
#Override
public void serialize(Something value, JsonGenerator jgen, SerializerProvider provider)
throws IOException, JsonProcessingException {
jgen.writeStartObject();
// Adding the folder into the json, only if it exists
if(value.getCurrentFolder()!=null){
jgen.writeObjectField(value.getCurrentFolder(), value.getChilds());
}
// Adding properties of the picture, only if they exist
if(value.getCurrentPicture()!= null){
for(String k : value.getCurrentPicture().keySet()){
jgen.writeObjectField(k,value.getCurrentPicture().get(k));
}
}
jgen.writeEndObject();
}
}
Finally (I've not done this one, but you'll do it I'm sure !) create a mapper from what you read to the "Something" class.
I build the object manually here (quickly, so it's not clean):
public static void main(String[] args) throws JsonGenerationException, JsonMappingException, IOException {
Something s = new Something();
s.setCurrentFolder("toto");
Something s2 = new Something();
s2.setCurrentFolder("tata");
Something s2bis = new Something();
s2bis.setCurrentFolder("tataBis");
Something[] s2Group = {s2bis};
s2.setContent(s2Group);
Something s2bispic = new Something();
s2bispic.getCurrentPicture().put("fileName", "ThatPictureOfMysSelfILikeSoMuch.jpg");
s2bispic.getCurrentPicture().put("path", "toto/tata/tataBis/ThatPictureOfMysSelfILikeSoMuch.jpg");
s2bispic.getCurrentPicture().put("mimeType", "image/jpeg");
Something s2bispic2 = new Something();
s2bispic2.getCurrentPicture().put("fileName", "ThatPictureOfMysSelfIDontLike.jpg");
s2bispic2.getCurrentPicture().put("path", "toto/tata/tataBis/ThatPictureOfMysSelfIDontLike.jpg");
s2bispic2.getCurrentPicture().put("mimeType", "image/jpeg");
Something[] s2BisGroup = {s2bispic,s2bispic2};
s2bis.setContent(s2BisGroup);
Something s3 = new Something();
s3.getCurrentPicture().put("fileName", "selfie.jpg");
s3.getCurrentPicture().put("path", "toto/selfie.jpg");
s3.getCurrentPicture().put("mimeType", "image/jpeg");
Something[] sGroup = {s2,s3};
s.setContent(sGroup);
ObjectMapper mapper = new ObjectMapper();
String temp = mapper.writeValueAsString(s);
System.out.println(temp);
}
And this is what I get :
{
"toto":[
{
"tata":[
{
"tataBis":[
{
"path":"toto/tata/tataBis/ThatPictureOfMysSelfILikeSoMuch.jpg",
"fileName":"ThatPictureOfMysSelfILikeSoMuch.jpg",
"mimeType":"image/jpeg"
},
{
"path":"toto/tata/tataBis/ThatPictureOfMysSelfIDontLike.jpg",
"fileName":"ThatPictureOfMysSelfIDontLike.jpg",
"mimeType":"image/jpeg"
}
]
}
]
},
{
"path":"toto/selfie.jpg",
"fileName":"selfie.jpg",
"mimeType":"image/jpeg"
}
]
}
Regards,

Pulling out value of keys in JSON using Java, when key is under hierarchy

I need to pull out value of RecordOne from following JSON.
{
"errors": [],
"data": {
"paging": {
"RecordOne": 8,
"RecordTwo": 9,
"recordThree": 2,
"totalNumberOfRecords": 86052
},
"products": [
{
"testabstract": "test data",
"authors": "Frank Jr.",
"invertedauthors": "Frank VJr.",
"formatCode": "KND"
}
]
}
}
I'm using Java as language and JSON object to achieve the same, following is what I'm using:
protected String getTokenValueUnderHeirarchy(String responseString){
JSONObject json = new JSONObject(responseString);
String val= json.getJSONObject("data").getJSONObject("paging").getString("RecordOne");
System.out.println("val::"+val);
return val;
}
I'm getting value of val = 1, it should be 8
If I try to seek value for key totalNumberOfRecords with same code it returns correct value which is 86052
I know it's something silly but I can't catch it.
When I ran your code with the JSON example, I ended up with a "JSONException: JsonObject["RecordOne"] is not a string"..... which it isn't. Wrapping the 8 with double quotes: "8" returned the value that you expected. You can access this value with other get methods: getInt if you would like.
This test case parses both a String and an int. I pulled this from your example. Does it run for you?
package org.nadnavillus.test;
import org.json.JSONObject;
import org.junit.Test;
public class TestCase {
protected String getTokenValueUnderHeirarchy(String responseString) throws Exception {
JSONObject json = new JSONObject(responseString);
String val= json.getJSONObject("data").getJSONObject("paging").getString("RecordOne");
System.out.println("val::"+val);
return val;
}
protected String getTokenValueUnderHeirarchyInt(String responseString) throws Exception {
JSONObject json = new JSONObject(responseString);
int val= json.getJSONObject("data").getJSONObject("paging").getInt("RecordTwo");
System.out.println("val::"+val);
return String.valueOf(val);
}
#Test
public void testJson() throws Exception {
String input = "{\"errors\": [], \"data\": {\"paging\": {\"RecordOne\": \"8\", \"RecordTwo\": 9, \"recordThree\": 2, \"totalNumberOfRecords\": 86052}}}";
String test = this.getTokenValueUnderHeirarchy(input);
System.out.println(test);
test = this.getTokenValueUnderHeirarchyInt(input);
System.out.println(test);
}
}

Error Unexpected Char javax.json

I am trying to read this JSON code
{
"metadata": {
"clientTransactionId": "",
"serverTransactionId": "20160621101521362-domainrobot-demo.routing-18997-0"
},
"responses": [
{
"domainName": "test.la",
"domainNameUnicode": "test.la",
"domainSuffix": "la",
"earlyAccessStart": null,
"extension": "la",
"generalAvailabilityStart": "2000-01-01T00:00:00Z",
"landrushStart": null,
"launchPhase": "generalAvailability",
"registrarTag": null,
"status": "registered",
"sunriseStart": null,
"transferMethod": "authInfo"
}
],
"status": "success",
"warnings": []
}
With my Java Program:
import javax.json.*;
import java.nio.charset.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.io.*;
public class Main {
public static String readFile(String path, Charset encoding) throws IOException
{
byte[] encoded = Files.readAllBytes(Paths.get(path));
return new String(encoded, encoding);
}
public static void main(String[] args) throws IOException {
String jsonData = readFile("/home/maltepraktikant/workspace/DomainCreator/bin/JsonData.txt", StandardCharsets.UTF_8);
JsonReader jsonreader = Json.createReader(new StringReader(jsonData));
JsonObject object = jsonreader.readObject();
System.out.println(object);
jsonreader.close();
}
}
I have tried different things, but I haven't found a solution yet. It just gives me the error:
Exception in thread "main" javax.json.stream.JsonParsingException: Unexpected char 65.279 at (line no=1, column no=1, offset=0)
at org.glassfish.json.JsonTokenizer.unexpectedChar(JsonTokenizer.java:532)
at org.glassfish.json.JsonTokenizer.nextToken(JsonTokenizer.java:415)
at org.glassfish.json.JsonParserImpl$NoneContext.getNextEvent(JsonParserImpl.java:222)
at org.glassfish.json.JsonParserImpl$StateIterator.next(JsonParserImpl.java:172)
at org.glassfish.json.JsonParserImpl.next(JsonParserImpl.java:149)
at org.glassfish.json.JsonReaderImpl.readObject(JsonReaderImpl.java:101)
at Main.main(Main.java:19)
Has anyone some ideas?
Get the json response and replace all new lines first before parsing it to object.
response.replaceAll("\r?\n", "");
Sample code using GSON API
String json = "{\"msg\" : \"Hello \n World\"}";
System.out.println(json);
json = json.replaceAll("\r?\n", "");
Map<String, String> map = new Gson().fromJson(json, new TypeToken<Map<String, String>>(){}.getType());
System.out.println("Actual message:" + map.get("msg"));
Output:
{"msg" : " Hello
World"}
Actual message: Hello World
In my case error was due to missing quotes for values or key. In the json string null values are not in quotes (ex: "earlyAccessStart": null). Keeping them in quotes would solve the issue.
Note: Validate the json string once through a valitor like

Adding mapping to a type from Java - how do I do it?

I am trying to recreate this example using the Java API more or less.
I think all I need is to add a mapping to the index, but the Java API docs are not exactly clear on how to do this.
Please tell me how I create a mapping in Java that is the equivalent of this from the example in the documentation :
curl -X PUT localhost:9200/test/tweet/_mapping -d '{
"tweet" : {
"_ttl" : { "enabled" : true, "default" : "1d" }
}
}'
Here's my code:
package foo;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import java.io.IOException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
public class MyTestClass {
private static Client getClient() {
ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder();
TransportClient transportClient = new TransportClient(settings);
transportClient = transportClient.addTransportAddress(new InetSocketTransportAddress("localhost", 9300));
return (Client) transportClient;
}
public static void main(String[] args) throws IOException, InterruptedException {
final Client client = getClient();
// Create Index and set settings and mappings
final String indexName = "test";
final String documentType = "tweet";
final String documentId = "1";
final String fieldName = "foo";
final String value = "bar";
IndicesExistsResponse res = client.admin().indices().prepareExists(indexName).execute().actionGet();
if (res.isExists()) {
DeleteIndexRequestBuilder delIdx = client.admin().indices().prepareDelete(indexName);
delIdx.execute().actionGet();
}
CreateIndexRequestBuilder createIndexRequestBuilder = client.admin().indices().prepareCreate(indexName);
// MAPPING GOES HERE
// createIndexRequestBuilder.addMapping(documentType, WHATEVER THE MAPPING IS);
// MAPPING DONE
createIndexRequestBuilder.execute().actionGet();
// Add documents
IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, documentType, documentId);
// build json object
XContentBuilder contentBuilder = jsonBuilder().startObject().prettyPrint();
contentBuilder.field(fieldName, value);
indexRequestBuilder.setSource(contentBuilder);
indexRequestBuilder.execute().actionGet();
// Get document
System.out.println(getValue(client, indexName, documentType, documentId, fieldName));
Thread.sleep(10000L);
// Try again
System.out.println(getValue(client, indexName, documentType, documentId, fieldName));
}
protected static String getValue(final Client client, final String indexName, final String documentType,
final String documentId, final String fieldName) {
GetRequestBuilder getRequestBuilder = client.prepareGet(indexName, documentType, documentId);
getRequestBuilder.setFields(new String[] { fieldName });
GetResponse response2 = getRequestBuilder.execute().actionGet();
String name = response2.getField(fieldName).getValue().toString();
return name;
}
}
Finally a day of googling paid off. Frankly the Java API docs for elasticsearch could use some end-to-end examples, not to mention JavaDoc...
Here's a running example. You must have a node running on localhost for this to work!
package foo;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import java.io.IOException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
public class MyTestClass {
private static final String ID_NOT_FOUND = "<ID NOT FOUND>";
private static Client getClient() {
final ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder();
TransportClient transportClient = new TransportClient(settings);
transportClient = transportClient.addTransportAddress(new InetSocketTransportAddress("localhost", 9300));
return transportClient;
}
public static void main(final String[] args) throws IOException, InterruptedException {
final Client client = getClient();
// Create Index and set settings and mappings
final String indexName = "test";
final String documentType = "tweet";
final String documentId = "1";
final String fieldName = "foo";
final String value = "bar";
final IndicesExistsResponse res = client.admin().indices().prepareExists(indexName).execute().actionGet();
if (res.isExists()) {
final DeleteIndexRequestBuilder delIdx = client.admin().indices().prepareDelete(indexName);
delIdx.execute().actionGet();
}
final CreateIndexRequestBuilder createIndexRequestBuilder = client.admin().indices().prepareCreate(indexName);
// MAPPING GOES HERE
final XContentBuilder mappingBuilder = jsonBuilder().startObject().startObject(documentType)
.startObject("_ttl").field("enabled", "true").field("default", "1s").endObject().endObject()
.endObject();
System.out.println(mappingBuilder.string());
createIndexRequestBuilder.addMapping(documentType, mappingBuilder);
// MAPPING DONE
createIndexRequestBuilder.execute().actionGet();
// Add documents
final IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, documentType, documentId);
// build json object
final XContentBuilder contentBuilder = jsonBuilder().startObject().prettyPrint();
contentBuilder.field(fieldName, value);
indexRequestBuilder.setSource(contentBuilder);
indexRequestBuilder.execute().actionGet();
// Get document
System.out.println(getValue(client, indexName, documentType, documentId, fieldName));
int idx = 0;
while (true) {
Thread.sleep(10000L);
idx++;
System.out.println(idx * 10 + " seconds passed");
final String name = getValue(client, indexName, documentType, documentId, fieldName);
if (ID_NOT_FOUND.equals(name)) {
break;
} else {
// Try again
System.out.println(name);
}
}
System.out.println("Document was garbage collected");
}
protected static String getValue(final Client client, final String indexName, final String documentType,
final String documentId, final String fieldName) {
final GetRequestBuilder getRequestBuilder = client.prepareGet(indexName, documentType, documentId);
getRequestBuilder.setFields(new String[] { fieldName });
final GetResponse response2 = getRequestBuilder.execute().actionGet();
if (response2.isExists()) {
final String name = response2.getField(fieldName).getValue().toString();
return name;
} else {
return ID_NOT_FOUND;
}
}
}
I am actually going to add another answer here because frankly speaking, the above answers gave a start to my implementation but didn't answer the actual question 100% (updating not just the root level properties, but the ACTUAL field/properties). Took me almost 2 days to figure this out (Documentation is a bit light for ES Java APIs). My "Mapping" class is not 100% yet, but more fields could be added to it ("format" etc) later.
I hope this helps everyone else who is trying to use update mappings!
GET/RETRIEVE MAPPINGS
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings = response.getMappings();
ImmutableOpenMap<String, MappingMetaData> typeMappings = indexMappings.get(indexName);
MappingMetaData mapping = typeMappings.get(type);
Map<String, Mapping> mappingAsMap = new HashMap<>();
try {
Object properties = mapping.sourceAsMap().get("properties");
mappingAsMap = (Map<String, Mapping>) gson.fromJson(gson.toJson(properties), _elasticsearch_type_mapping_map_type);
return mappingAsMap;
}
UPDATE MAPPINGS
PutMappingRequest mappingRequest = new PutMappingRequest(indexName);
Map<String, Object> properties = new HashMap<>();
Map<String, Object> mappingsMap = (Map<String, Object>) gson.fromJson(gson.toJson(mapping), Json._obj_map_type);
properties.put("properties", mappingsMap);
mappingRequest = mappingRequest.ignoreConflicts(true).type(type).source(properties).actionGet();
My GSON mapping types
public static final Type _obj_map_type = new TypeToken<LinkedHashMap<String, Object>>(){}.getType();
public static final Type _elasticsearch_type_mapping_map_type = new TypeToken<LinkedHashMap<String, Mapping>>(){}.getType();
My Mapping Class
public class Mapping {
private String type;
private String index;
private String analyzer;
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getIndex() {
return index;
}
public void setIndex(String index) {
this.index = index;
}
public String getAnalyzer() {
return analyzer;
}
public void setAnalyzer(String analyzer) {
this.analyzer = analyzer;
}
}
An alternative solution would be to use a feature called dynamic templates. The idea is very well described within this article http://joelabrahamsson.com/dynamic-mappings-and-dates-in-elasticsearch/
So this case uses a regular expression declaring any field that starts with tikaprop_ to be of type String.
curl -XPUT "http://localhost:9200/myindex" -d'
{
"mappings": {
"_default_": {
"date_detection": true,
"dynamic_templates": [
{
"tikaprops": {
"match": "tikaprop_.*",
"match_pattern": "regex",
"mapping": {
"type": "string"
}
}
}
]
}
}
}'
or if you'd rather prefer to do it via the Elasticsearch Java API
CreateIndexRequestBuilder cirb = this.client.admin().indices().prepareCreate(INDEX_NAME).addMapping("_default_", getIndexFieldMapping());
CreateIndexResponse createIndexResponse = cirb.execute().actionGet();
private String getIndexFieldMapping() {
return IOUtils.toString(getClass().getClassLoader().getResourceAsStream("elasticsearch_dynamic_templates_config.json"));
}
with elasticsearch_dynamic_templates_config.json beein:
{
"date_detection": true,
"dynamic_templates": [
{
"tikaprops": {
"match": "tikaprop_.*",
"match_pattern": "regex",
"mapping": {
"type": "string"
}
}
}
]
}
I started with the excellent answer given by Anders Johansen and converted it to Groovy (so that the JSON is easier to read). I share with you my answer.
package com.example
import groovy.json.JsonSlurper
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse
import org.elasticsearch.action.get.GetRequestBuilder
import org.elasticsearch.action.get.GetResponse
import org.elasticsearch.action.index.IndexRequestBuilder
import org.elasticsearch.client.Client
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.transport.InetSocketTransportAddress
class StackOverflow {
Client client;
final byte[] ipAddr = [192, 168,33, 10]; // Your ElasticSearch node ip goes here
final String indexName = "classifieds";
final String documentType = "job";
public StackOverflow() {
client = TransportClient.builder().build()
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByAddress(ipAddr), 9300));
}
public void index(){
final IndicesExistsResponse res = client.admin().indices().prepareExists(indexName).execute().actionGet();
if (res.isExists()) {
final DeleteIndexRequestBuilder delIdx = client.admin().indices().prepareDelete(indexName);
delIdx.execute().actionGet();
}
final CreateIndexRequestBuilder createIndexRequestBuilder = client.admin().indices().prepareCreate(indexName);
// Create Mapping
def jsonSlurper = new JsonSlurper()
def mapping = jsonSlurper.parseText '''
{
"job": {
"properties": {
"company": {
"type": "string",
"analyzer": "english"
},
"desc": {
"type": "string",
"analyzer": "english"
},
"loc": {
"type": "string",
"analyzer": "english"
},
"req": {
"type": "string",
"analyzer": "english"
},
"title": {
"type": "string",
"analyzer": "english"
},
"url": {
"type": "string",
"analyzer": "english"
}
}
}
}'''
System.out.println(mapping.toString());
createIndexRequestBuilder.addMapping(documentType, mapping);
// MAPPING DONE
createIndexRequestBuilder.execute().actionGet();
// Add documents
final IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, documentType, "1");
// build json object
def jobcontent = jsonSlurper.parseText '''
{
"company": "ACME",
"title": "Groovy Developer",
"loc": "Puerto Rico",
"desc": "Codes in Groovy",
"req": "ElasticSearch, Groovy ",
"url": "http://stackoverflow.com/questions/22071198/adding-mapping-to-a-type-from-java-how-do-i-do-it#"
}
'''
indexRequestBuilder.setSource(jobcontent);
indexRequestBuilder.execute().actionGet();
}
private String getValue2(final String indexName, final String documentType,
final String documentId, final String fieldName) {
GetRequestBuilder getRequestBuilder = client.prepareGet(indexName, documentType, documentId);
getRequestBuilder.setFields([ fieldName ] as String[]);
GetResponse response2 = getRequestBuilder.execute().actionGet();
String name = response2.getField(fieldName).getValue().toString();
return name;
}
public String getValue(final String documentId, final String fieldName){
getValue2(indexName, documentType, documentId, fieldName )
}
public void close() {
client.close()
}
public static void main (String[] Args){
StackOverflow so = new StackOverflow();
so.index();
Thread.sleep(5000L);
System.out.println(so.getValue("1", "title"));
so.close();
}
}

Categories