We Keep Coding

Getting partial json response for s3select with aws java sdk v2

I am trying to implement s3select in a spring boot app to query parquet file in s3 bucket, I am only getting partial result from the s3select output, Please help to identify the issue, i have used aws java sdk v2.
Upon checking the json output(printed in the console), overall characters in the output is 65k.
I am using eclipse and tried unchecking "Limit console output" in the console preference, which did not help.
Code is here:-
import java.util.List;
import java.util.concurrent.CompletableFuture;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.async.SdkPublisher;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.CompressionType;
import software.amazon.awssdk.services.s3.model.EndEvent;
import software.amazon.awssdk.services.s3.model.ExpressionType;
import software.amazon.awssdk.services.s3.model.InputSerialization;
import software.amazon.awssdk.services.s3.model.JSONOutput;
import software.amazon.awssdk.services.s3.model.OutputSerialization;
import software.amazon.awssdk.services.s3.model.ParquetInput;
import software.amazon.awssdk.services.s3.model.RecordsEvent;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream.EventType;
import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponse;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
public class ParquetSelect {
private static final String BUCKET_NAME = "<bucket-name>";
private static final String KEY = "<object-key>";
private static final String QUERY = "select * from S3Object s";
public static S3AsyncClient s3;
public static void selectObjectContent() {
Handler handler = new Handler();
SelectQueryWithHandler(handler).join();
RecordsEvent recordsEvent = (RecordsEvent) handler.receivedEvents.stream()
.filter(e -> e.sdkEventType() == EventType.RECORDS)
.findFirst()
.orElse(null);
System.out.println(recordsEvent.payload().asUtf8String());
}
private static CompletableFuture<Void> SelectQueryWithHandler(SelectObjectContentResponseHandler handler) {
InputSerialization inputSerialization = InputSerialization.builder()
.parquet(ParquetInput.builder().build())
.compressionType(CompressionType.NONE)
.build();
OutputSerialization outputSerialization = OutputSerialization.builder()
.json(JSONOutput.builder().build())
.build();
SelectObjectContentRequest select = SelectObjectContentRequest.builder()
.bucket(BUCKET_NAME)
.key(KEY)
.expression(QUERY)
.expressionType(ExpressionType.SQL)
.inputSerialization(inputSerialization)
.outputSerialization(outputSerialization)
.build();
return s3.selectObjectContent(select, handler);
}
private static class Handler implements SelectObjectContentResponseHandler {
private SelectObjectContentResponse response;
private List<SelectObjectContentEventStream> receivedEvents = new ArrayList<>();
private Throwable exception;
#Override
public void responseReceived(SelectObjectContentResponse response) {
this.response = response;
}
#Override
public void onEventStream(SdkPublisher<SelectObjectContentEventStream> publisher) {
publisher.subscribe(receivedEvents::add);
}
#Override
public void exceptionOccurred(Throwable throwable) {
exception = throwable;
}
#Override
public void complete() {
}
}
}

I see you are using selectObjectContent(). Have you tried calling the s3AsyncClient.getObject() method. Does that work for you?
For example, here is a code example that gets a PDF file from an Amazon S3 bucket and write the PDF file to a local file.
package com.example.s3.async;
// snippet-start:[s3.java2.async_stream_ops.complete]
// snippet-start:[s3.java2.async_stream_ops.import]
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import java.nio.file.Paths;
import java.util.concurrent.CompletableFuture;
// snippet-end:[s3.java2.async_stream_ops.import]
// snippet-start:[s3.java2.async_stream_ops.main]
/**
* Before running this Java V2 code example, set up your development environment, including your credentials.
*
* For more information, see the following documentation topic:
*
* https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html
*/
public class S3AsyncStreamOps {
public static void main(String[] args) {
final String usage = "\n" +
"Usage:\n" +
" <bucketName> <objectKey> <path>\n\n" +
"Where:\n" +
" bucketName - The name of the Amazon S3 bucket (for example, bucket1). \n\n" +
" objectKey - The name of the object (for example, book.pdf). \n" +
" path - The local path to the file (for example, C:/AWS/book.pdf). \n" ;
if (args.length != 3) {
System.out.println(usage);
System.exit(1);
}
String bucketName = args[0];
String objectKey = args[1];
String path = args[2];
ProfileCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create();
Region region = Region.US_EAST_1;
S3AsyncClient s3AsyncClient = S3AsyncClient.builder()
.region(region)
.credentialsProvider(credentialsProvider)
.build();
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(objectKey)
.build();
CompletableFuture<GetObjectResponse> futureGet = s3AsyncClient.getObject(objectRequest,
AsyncResponseTransformer.toFile(Paths.get(path)));
futureGet.whenComplete((resp, err) -> {
try {
if (resp != null) {
System.out.println("Object downloaded. Details: "+resp);
} else {
err.printStackTrace();
}
} finally {
// Only close the client when you are completely done with it.
s3AsyncClient.close();
}
});
futureGet.join();
}
}

Braintree webhooks: Error: payload contains illegal characters

I am trying to consume Braintree webhooks in a java microservice (Micronaut fwiw).
The issue that I'm having is that when I try to parse the webhook body, I get an error: " Error: payload contains illegal characters", which it does. So I'm wondering if maybe I'm casting the request body to something thats inserting the characters...? (the body is x-www-form-urlencoded)
package com.autonomy;
import com.braintreegateway.*;
import event_broker.BrokerServiceGrpc;
import event_broker.EventBroker;
import events.Events;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.micronaut.context.annotation.Value;
import io.micronaut.http.HttpStatus;
import io.micronaut.http.MediaType;
import io.micronaut.http.annotation.Body;
import io.micronaut.http.annotation.Controller;
import io.micronaut.http.annotation.Post;
import io.micronaut.http.HttpResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.Normalizer;
import java.time.Clock;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
#Controller("/webhooks")
public class WebhooksController {
private final Logger logger = LoggerFactory.getLogger("BraintreeServiceJava");
#Value("${env}") String env;
#Value("${braintree.merchant.id}") String braintreeMerchantId;
#Value("${braintree.public.key}") String braintreePublicKey;
#Value("${braintree.private.key}") String braintreePrivateKey;
#Post(consumes = MediaType.APPLICATION_FORM_URLENCODED)
public HttpResponse<?> consumeWebhook(#Body String body) {
BraintreeGateway gateway =
new BraintreeGateway(determineEnv(env),
braintreeMerchantId,
braintreePublicKey,
braintreePrivateKey
);
logger.info(body);
try {
String decodedBody = body; // was doing a decode here that didn't do anything
logger.info(decodedBody);
Map<String, String> map = new HashMap<>();
Arrays.stream(decodedBody.split("&")).toList().forEach(pair -> {
String[] param = pair.split("=");
map.put(param[0], param[1]);
});
WebhookNotification webhookNotification = gateway.webhookNotification().parse(
map.get("bt_signature"),
map.get("bt_payload")
);
..... Do stuff
} catch (Exception e) {
logger.error(String.format("Braintree webhook failed for %s. Error: %s", kind, e.getMessage()), e);
return HttpResponse.status(HttpStatus.BAD_REQUEST);
}
return HttpResponse.status(HttpStatus.OK);
}
private Environment determineEnv(String env) {
if (env.equals("beta") || env.equals("prod")) {
return Environment.PRODUCTION;
} else {
return Environment.SANDBOX;
}
}
}

Try:
#Post(consumes = MediaType.APPLICATION_FORM_URLENCODED)
public HttpResponse<?> consumeWebhook(String bt_signature, String bt_payload)
BTW: What 'illegal characters' were in the body that was logged?

Java soap web service header authentication call from windows application c#

I was create a soap web service at java. When called from java working fine, but when called from c# its not working, would you please help me? How to call getHelloWorldAsString method from c# with header authentication. Thanks in advance
Here is the java code:
package com.mkyong.ws;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;
import javax.jws.WebService;
import javax.xml.ws.WebServiceContext;
import javax.xml.ws.handler.MessageContext;
//Service Implementation Bean
#WebService(endpointInterface = "com.mkyong.ws.HelloWorld")
public class HelloWorldImpl implements HelloWorld{
#Resource
WebServiceContext wsctx;
#Override
public String getHelloWorldAsString() {
MessageContext mctx = wsctx.getMessageContext();
//get detail from request headers
Map<?,?> http_headers = (Map<?,?>) mctx.get(MessageContext.HTTP_REQUEST_HEADERS);
List<?> userList = (List<?>) http_headers.get("Username");
List<?> passList = (List<?>) http_headers.get("Password");
String username = "";
String password = "";
if(userList!=null){
//get username
username = userList.get(0).toString();
}
if(passList!=null){
//get password
password = passList.get(0).toString();
}
//Should validate username and password
if (username.equals("abcd") && password.equals("abcd123")){
return "Hello World JAX-WS - Valid User!";
}else{
return "Unknown User!";
}
}
#Override
public String getSum() {
return "10";
}
}
Here is the java calling (this is working):
package com.mkyong.client;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.namespace.QName;
import javax.xml.ws.BindingProvider;
import javax.xml.ws.Service;
import javax.xml.ws.handler.MessageContext;
import com.mkyong.ws.HelloWorld;
public class HelloWorldClient{
private static final String WS_URL ="http://localhost:8080/ws/HelloWorld?wsdl";
public static void main(String[] args) throws Exception {
try {
URL url = new URL(WS_URL);
QName qname = new QName("http://ws.mkyong.com/", "HelloWorldImplService");
Service service = Service.create(url, qname);
HelloWorld hello = service.getPort(HelloWorld.class);
/*******************UserName & Password ******************************/
Map<String, Object> req_ctx = ((BindingProvider)hello).getRequestContext();
req_ctx.put(BindingProvider.ENDPOINT_ADDRESS_PROPERTY, WS_URL);
Map<String, List<String>> headers = new HashMap<String, List<String>>();
headers.put("Username", Collections.singletonList("abcd"));
headers.put("Password", Collections.singletonList("abcd123"));
req_ctx.put(MessageContext.HTTP_REQUEST_HEADERS, headers);
/**********************************************************************/
System.out.println(hello.getHelloWorldAsString());
System.out.println(hello.getSum());
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
C# Client Call:
After adding wsdl to the windows application web service,written following C# code,
webService.HelloWorldImplService ws = new webService.HelloWorldImplService(); Console.WriteLine(ws.getHelloWorldAsString());

JAVA XmlRpcClient

I am using this code to connect server, but get error:
pms.java:25: error: cannot find symbol
server.setconfig(config);
enter code here ^
symbol: method setconfig(XmlRpcClientConfigImpl)
location: variable server of type XmlRpcClient
import java.util.*;
import org.apache.xmlrpc.client.XmlRpcClient;
import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
//import org.apache.xmlrpc.client.XmlRpcClientConfig;
import org.apache.xmlrpc.common.*;
import org.apache.xmlrpc.*;
import java.net.*;
public class pms {
public static void main (String [] args) {
try {
String UserName = "123";
String Password = "123";
String pKey = "123";
Vector params = new Vector();
params.addElement(UserName);
params.addElement(Password);
params.addElement(pKey);
XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl();
config.setServerURL(new URL(" https://wubook.net/xrws/"));
XmlRpcClient server = new XmlRpcClient();
server.setconfig(config);
Object result = server.execute("acquire_token",params);
System.out.println(result);
System.out.println("Hello World");
} catch (Exception e) {
}
}
}

Your code server.setcongfig should be setConfig (capitlization)

i think it should be server.setConfig(config)notserver.setconfig(config)
you can import
import java.net.URL;
import java.util.Vector;
import org.apache.xmlrpc.client.XmlRpcClient;
import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
only
and here is the dependancy
<dependency>
<groupId>org.apache.xmlrpc</groupId>
<artifactId>xmlrpc-client</artifactId>
<version>3.1.3</version>
it work fine and the output

Connecting two unix server's and push data from server to server using java program

I have create a java program that will connect to unix server1 (with user name pwd),now i need to connect another server (with username password) and execute command to push data from server 1 to server 2 using java program.
Server1 connection is working fine and i can execute some basic commands in it.
Code below
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Properties;
import net.neoremind.sshxcute.core.ConnBean;
import net.neoremind.sshxcute.core.Result;
import net.neoremind.sshxcute.core.SSHExec;
import net.neoremind.sshxcute.exception.TaskExecFailException;
import net.neoremind.sshxcute.task.CustomTask;
import net.neoremind.sshxcute.task.impl.ExecCommand;
public class UnixConnect {
// String hostName, String logFile, String userName, String password
static String[] host_names = null;
static String[] user_names = null;
static String[] pwd_text = null;
public void execCommand() throws TaskExecFailException {
ConnBean cb1 = new ConnBean("0.000.00.000", "***", "****");
SSHExec ssh1 = SSHExec.getInstance(cb1);
ssh1.connect();
String[] cmd = {"cd /apps/a/b/c/logs","tail -1 aLoadJob.log"};
CustomTask tasks = new ExecCommand(cmd);
Result res1 = ssh1.exec(tasks);
if (res1.isSuccess) {
/*System.out.println(res1.sysout)*/;
} else {
System.out.println("Return code: " + res1.sysout);
System.out.println("error message: " + res1.error_msg);
}
ssh1.disconnect();
}
/**
* #param args
*/
public static void main(String[] args) throws TaskExecFailException {
UnixConnect ob = new UnixConnect();
ob.execCommand();
}
}
In both Server i need to give username and password

This is a bit strange use case but I would do the following:
Implement bash script on server1 that would do all the job of transfering data to server2. Of course server1 has to be able to connecto server2
Use you program to invoke the bash script on server1.