We Keep Coding

Getting partial json response for s3select with aws java sdk v2

I am trying to implement s3select in a spring boot app to query parquet file in s3 bucket, I am only getting partial result from the s3select output, Please help to identify the issue, i have used aws java sdk v2.
Upon checking the json output(printed in the console), overall characters in the output is 65k.
I am using eclipse and tried unchecking "Limit console output" in the console preference, which did not help.
Code is here:-
import java.util.List;
import java.util.concurrent.CompletableFuture;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.async.SdkPublisher;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.CompressionType;
import software.amazon.awssdk.services.s3.model.EndEvent;
import software.amazon.awssdk.services.s3.model.ExpressionType;
import software.amazon.awssdk.services.s3.model.InputSerialization;
import software.amazon.awssdk.services.s3.model.JSONOutput;
import software.amazon.awssdk.services.s3.model.OutputSerialization;
import software.amazon.awssdk.services.s3.model.ParquetInput;
import software.amazon.awssdk.services.s3.model.RecordsEvent;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream.EventType;
import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponse;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
public class ParquetSelect {
private static final String BUCKET_NAME = "<bucket-name>";
private static final String KEY = "<object-key>";
private static final String QUERY = "select * from S3Object s";
public static S3AsyncClient s3;
public static void selectObjectContent() {
Handler handler = new Handler();
SelectQueryWithHandler(handler).join();
RecordsEvent recordsEvent = (RecordsEvent) handler.receivedEvents.stream()
.filter(e -> e.sdkEventType() == EventType.RECORDS)
.findFirst()
.orElse(null);
System.out.println(recordsEvent.payload().asUtf8String());
}
private static CompletableFuture<Void> SelectQueryWithHandler(SelectObjectContentResponseHandler handler) {
InputSerialization inputSerialization = InputSerialization.builder()
.parquet(ParquetInput.builder().build())
.compressionType(CompressionType.NONE)
.build();
OutputSerialization outputSerialization = OutputSerialization.builder()
.json(JSONOutput.builder().build())
.build();
SelectObjectContentRequest select = SelectObjectContentRequest.builder()
.bucket(BUCKET_NAME)
.key(KEY)
.expression(QUERY)
.expressionType(ExpressionType.SQL)
.inputSerialization(inputSerialization)
.outputSerialization(outputSerialization)
.build();
return s3.selectObjectContent(select, handler);
}
private static class Handler implements SelectObjectContentResponseHandler {
private SelectObjectContentResponse response;
private List<SelectObjectContentEventStream> receivedEvents = new ArrayList<>();
private Throwable exception;
#Override
public void responseReceived(SelectObjectContentResponse response) {
this.response = response;
}
#Override
public void onEventStream(SdkPublisher<SelectObjectContentEventStream> publisher) {
publisher.subscribe(receivedEvents::add);
}
#Override
public void exceptionOccurred(Throwable throwable) {
exception = throwable;
}
#Override
public void complete() {
}
}
}

I see you are using selectObjectContent(). Have you tried calling the s3AsyncClient.getObject() method. Does that work for you?
For example, here is a code example that gets a PDF file from an Amazon S3 bucket and write the PDF file to a local file.
package com.example.s3.async;
// snippet-start:[s3.java2.async_stream_ops.complete]
// snippet-start:[s3.java2.async_stream_ops.import]
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import java.nio.file.Paths;
import java.util.concurrent.CompletableFuture;
// snippet-end:[s3.java2.async_stream_ops.import]
// snippet-start:[s3.java2.async_stream_ops.main]
/**
* Before running this Java V2 code example, set up your development environment, including your credentials.
*
* For more information, see the following documentation topic:
*
* https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html
*/
public class S3AsyncStreamOps {
public static void main(String[] args) {
final String usage = "\n" +
"Usage:\n" +
" <bucketName> <objectKey> <path>\n\n" +
"Where:\n" +
" bucketName - The name of the Amazon S3 bucket (for example, bucket1). \n\n" +
" objectKey - The name of the object (for example, book.pdf). \n" +
" path - The local path to the file (for example, C:/AWS/book.pdf). \n" ;
if (args.length != 3) {
System.out.println(usage);
System.exit(1);
}
String bucketName = args[0];
String objectKey = args[1];
String path = args[2];
ProfileCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create();
Region region = Region.US_EAST_1;
S3AsyncClient s3AsyncClient = S3AsyncClient.builder()
.region(region)
.credentialsProvider(credentialsProvider)
.build();
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(objectKey)
.build();
CompletableFuture<GetObjectResponse> futureGet = s3AsyncClient.getObject(objectRequest,
AsyncResponseTransformer.toFile(Paths.get(path)));
futureGet.whenComplete((resp, err) -> {
try {
if (resp != null) {
System.out.println("Object downloaded. Details: "+resp);
} else {
err.printStackTrace();
}
} finally {
// Only close the client when you are completely done with it.
s3AsyncClient.close();
}
});
futureGet.join();
}
}

Websocket Client Java finnhub always closed

I am currently trying to get the finnhub websocket to run via Java. The connection is also established, but I immediately get a closed message from the connection before I even get a message. Here my code:
import java.io.IOException;
import java.net.URI;
import java.util.concurrent.CountDownLatch;
import jakarta.websocket.ClientEndpoint;
import jakarta.websocket.ContainerProvider;
import jakarta.websocket.OnClose;
import jakarta.websocket.OnMessage;
import jakarta.websocket.OnOpen;
import jakarta.websocket.Session;
import jakarta.websocket.WebSocketContainer;
#ClientEndpoint
public class FinnhubQuoteWebsocket {
private static CountDownLatch latch;
private static final String API_KEY = "YOUR_KEY";
public static void main(String[] args) throws Exception {
String endpoint = "wss://finnhub.io/ws?token="
+ API_KEY;
latch = new CountDownLatch(1);
URI uri = new URI(endpoint);
FinnhubQuoteWebsocket client = new FinnhubQuoteWebsocket();
WebSocketContainer container = ContainerProvider.getWebSocketContainer();
container.connectToServer(client,
uri);
latch.await();
}
#OnClose
public void onClose(Session session) {
System.out.println("###Close###");
}
#OnMessage
public void onMessage(Session session,
String message) {
System.out.println("Received quote message: "
+ message);
latch.countDown();
}
#OnOpen
public void onOpen(Session session) throws IOException {
String subscribeSymbolAAPL = "{\"type\":\"subscribe\",\"symbol\":\"AAPL\"}";
System.out.println("Connected to websocket: "
+ session.getBasicRemote());
System.out.println("Json: "
+ subscribeSymbolAAPL);
session.getBasicRemote().sendText(subscribeSymbolAAPL);
}
}
Consolen Output:
Connected to websocket: Wrapped: Basic
Json: {"type":"subscribe","symbol":"AAPL"}
###Close###
I do not know what I'm doing wrong. Presumably the connection has to be open longer for a message to come?
I tried various examples from Google and ChatGpt. But always the same problem that the connection is immediately closed again.
I hope someone can tell me what I'm doing wrong.

Java call with MS Graph to get Sharepoint sites returns 500 error

I have some code that will do a query to find all sites using the java ms graph libraries.
IDriveCollectionPage drives = graphClient.drives().buildRequest().get();
ISiteCollectionPage sitesPage = graphClient.sites().buildRequest(). get();
The first line still works, however the second line has suddenly started to fail with the error :
500 : Internal Server Error
Cache-Control : no-cache
client-request-id : 6bc81403-33ca-4aae-84d3-7b711ef12b6b
Content-Type : application/json
Date : Mon, 26 Apr 2021 08:10:34 GMT
request-id : e3004346-4297-4864-9802-cc17b81e875e
Strict-Transport-Security : max-age=31536000
Transfer-Encoding : chunked
Vary : Accept-Encoding
x-ms-ags-diagnostic : {"ServerInfo":{"DataCenter":"UK South","Slice":"E","Ring":"3","ScaleUnit":"000","RoleInstance":"LN2PEPF000039ED"}}
{
"error": {
"code": "generalException",
"message": "An assertion failed while processing this request",
"innerError": {
"code": "assertionFailed",
"date": "2021-04-26T08:10:34",
"request-id": "e3004346-4297-4864-9802-cc17b81e875e",
"client-request-id": "6bc81403-33ca-4aae-84d3-7b711ef12b6b"
}
}
}
Checked that the application registration in Azure has NOT changed.
There are a number of other ms graph calls that sill work, as illustrated by the call to one drive.
************** FULL CODE *************************************************
package com.example.testsharepoint;
import com.microsoft.aad.msal4j.ClientCredentialFactory;
import com.microsoft.aad.msal4j.ClientCredentialParameters;
import com.microsoft.aad.msal4j.ConfidentialClientApplication;
import com.microsoft.aad.msal4j.IAuthenticationResult;
import com.microsoft.graph.models.extensions.IGraphServiceClient;
import com.microsoft.graph.requests.extensions.GraphServiceClient;
import com.microsoft.graph.requests.extensions.IDriveCollectionPage;
import com.microsoft.graph.requests.extensions.ISiteCollectionPage;
import org.apache.commons.collections4.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiConsumer;
#Component
public class SharepointAdapter {
private static final transient Logger log = LoggerFactory.getLogger(SharepointAdapter.class);
private String accessToken = null;
private String clientId = "XXX";
private String clientSecret = "XXX";
private String tenantId = "XXX";
protected final static String authority = "https://login.microsoftonline.com/";
public static final String DefaultScope = "https://graph.microsoft.com/.default";
protected Set<String> scopeSet = new HashSet<String>();
private ConfidentialClientApplication app;
protected IAuthenticationResult result;
private IGraphServiceClient graphClient = null;
private SimpleAuthProvider authProvider = null;
public SharepointAdapter()
{
}
#PostConstruct
public void init() throws Exception {
getUserAccessToken();
}
public void getUserAccessToken() throws Exception {
app = ConfidentialClientApplication.builder(
clientId,
ClientCredentialFactory.createFromSecret(clientSecret))
.authority(authority + tenantId + "/")
.build();
String[] appScopes = DefaultScope.split(",");
CollectionUtils.addAll(scopeSet, appScopes);
ClientCredentialParameters.ClientCredentialParametersBuilder builder = ClientCredentialParameters.builder(scopeSet);
ClientCredentialParameters clientCredentialParam = builder.build();
if ( log.isDebugEnabled())
{
log.debug("{} Getting token...", getLogPrefix());
}
CompletableFuture<IAuthenticationResult> future = app.acquireToken(clientCredentialParam);
BiConsumer<IAuthenticationResult, Throwable> processAuthResult = (res, ex) -> {
if (ex != null) {
Throwable error = ex.getCause();
if (error != null) {
log.error("{}Error connecting to Microsoft - {}", getLogPrefix(), error.getMessage());
} else {
log.error("{}Error connecting to Microsoft - {}", getLogPrefix(), ex.getMessage());
}
}
};
future.whenCompleteAsync(processAuthResult);
future.join();
result = future.get();
if (result == null) {
throw new Exception("Unable to connect to Microsoft, did not get an authentication token.");
}
if ( log.isTraceEnabled())
{
log.trace("{}: TOKEN: {}", getLogPrefix(), result.accessToken() );
}
String token = result.accessToken();
authProvider = new SimpleAuthProvider(token);
// Build a Graph client
graphClient = GraphServiceClient.builder()
.authenticationProvider(authProvider)
.logger(MSLogger.getLogger())
.buildClient();
IDriveCollectionPage drives = graphClient.drives().buildRequest().get();
ISiteCollectionPage sitesPage = graphClient.sites().buildRequest(). get();
}
protected String getLogPrefix()
{
return "[ Client ID: "+ clientId + "] ";
}
}
Relevant version set:
<java.version>14</java.version>
<spring-cloud.version>Hoxton.SR10</spring-cloud.version>
<springframework.boot.version>2.3.9.RELEASE</springframework.boot.version>
<microsoft-msal4j-version>1.9.1</microsoft-msal4j-version>
<microsoft-graph-version>2.10.0</microsoft-graph-version>
<azure.version>3.2.0</azure.version>
Any help gratefully received.
Thanks in advance

For anyone else who has this issue, found the following change made the call successful:
LinkedList<Option> requestOptions = new LinkedList<Option>();
requestOptions.add(new QueryOption("search", "*"));
ISiteCollectionPage sitesPage = graphClient.sites().buildRequest(requestOptions).get();
Not sure why the old code suddenly stopped working, but this has fixed it.

Is it possible to use MailDev with Java?

I have a MailDev instance installed on a remote Server.
I'm trying to understand if is it possibile to send email with Java (using the standard JavaMailSender) using this fake SMTP server.
The config needs only the URL and the port but, in my case, it doesn't work.
It returns always:
Mail server connection failed; nested exception is com.sun.mail.util.MailConnectException: Couldn't connect to host ...
The WebUI is running correctly and I can see the empty inbox on server.
Thanks.

It seems like you are experience connections issues. Although I cannot tell you what the cause there is, perhaps I can offer an alternative solution to test your emails?
Using Wiser, you can run an embedded SMTP server and query that inside your junit test. I've used this a lot in my open source project Simple Java Mail and created a Rule for this:
package testutil.testrules;
import org.jetbrains.annotations.NotNull;
import org.junit.rules.ExternalResource;
import org.subethamail.smtp.server.SMTPServer;
import org.subethamail.wiser.Wiser;
import org.subethamail.wiser.WiserMessage;
import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.Iterator;
import java.util.List;
/**
* SmtpServerRule - a TestRule wrapping a Wiser instance (a SMTP server in Java), started and stopped right before and after each test.
* <br>
* SmtpServerRule exposes the same methods as the {#link Wiser} instance by delegating the implementation to the instance. These methods, however, can not be
* used outside a JUnit statement (otherwise an {#link IllegalStateException} is raised).
* <br>
* The {#link Wiser} instance can be directly retrieved but also only from inside a JUnit statement.
*/
public class SmtpServerRule extends ExternalResource {
private final Wiser wiser = new Wiser();
private final int port;
public SmtpServerRule(#NotNull Integer port) {
this.port = port;
}
#Override
protected void before() {
this.wiser.setPort(port);
this.wiser.start();
}
#Override
protected void after() {
this.wiser.stop();
}
#NotNull
public Wiser getWiser() {
checkState("getWiser()");
return this.wiser;
}
#NotNull
public List<WiserMessage> getMessages() {
checkState("getMessages()");
return wiser.getMessages();
}
#NotNull
public MimeMessage getOnlyMessage(String envelopeReceiver)
throws MessagingException {
checkState("getMessages()");
List<WiserMessage> messages = getMessages();
assertThat(messages).hasSize(1);
Iterator<WiserMessage> iterator = messages.iterator();
WiserMessage wiserMessage = iterator.next();
assertThat(wiserMessage.getEnvelopeReceiver()).isEqualTo(envelopeReceiver);
MimeMessage mimeMessage = wiserMessage.getMimeMessage();
iterator.remove();
return mimeMessage;
}
#NotNull
public MimeMessageAndEnvelope getOnlyMessage()
throws MessagingException {
checkState("getMessages()");
List<WiserMessage> messages = getMessages();
assertThat(messages).hasSize(1);
Iterator<WiserMessage> iterator = messages.iterator();
WiserMessage wiserMessage = iterator.next();
iterator.remove();
return new MimeMessageAndEnvelope(wiserMessage.getMimeMessage(), wiserMessage.getEnvelopeSender());
}
#NotNull
public MimeMessage getMessage(String envelopeReceiver)
throws MessagingException {
checkState("getMessages()");
List<WiserMessage> messages = getMessages();
Iterator<WiserMessage> iterator = messages.iterator();
while (iterator.hasNext()) {
WiserMessage wiserMessage = iterator.next();
if (wiserMessage.getEnvelopeReceiver().equals(envelopeReceiver)) {
MimeMessage mimeMessage = wiserMessage.getMimeMessage();
iterator.remove();
return mimeMessage;
}
}
throw new AssertionError("message not found for recipient " + envelopeReceiver);
}
#NotNull
public SMTPServer getServer() {
checkState("getServer()");
return wiser.getServer();
}
public boolean accept(String from, String recipient) {
checkState("accept(String, String)");
return wiser.accept(from, recipient);
}
public void deliver(String from, String recipient, InputStream data)
throws IOException {
checkState("deliver(String, String, InputStream)");
wiser.deliver(from, recipient, data);
}
public void dumpMessages(PrintStream out)
throws MessagingException {
checkState("dumpMessages(PrintStream)");
wiser.dumpMessages(out);
}
private void checkState(String method) {
if (this.wiser == null) {
throw new IllegalStateException(format("%s must not be called outside of a JUnit statement", method));
}
}
}
Then I use it like this:
public class MailerLiveTest {
private static final Integer SERVER_PORT = 251;
#Rule
public final SmtpServerRule smtpServerRule = new SmtpServerRule(SERVER_PORT);
#Before
public void setup() {
mailer = MailerBuilder.withSMTPServer("localhost", SERVER_PORT).buildMailer();
}
#Test
public void createMailSession_EmptySubjectAndBody() {
// send mail using mailer, which goes to localhost:251
MimeMessageAndEnvelope receivedMimeMessage = smtpServerRule.getOnlyMessage();
Email receivedEmail = EmailConverter.mimeMessageToEmail(receivedMimeMessage.getMimeMessage());
// perform assertions on Email object...
}
}

Check if mail.smtp.auth and mail.smtp.starttls.enable are set to false. In my case it worked (I also set localhost as host and 1025 as port).

E*Trade API OAuth Issue

I am having an issue with the OAuthClientImpl.getInstance() method that is on line 26. It is spitting out this error message:
Exception in thread "main" java.lang.ExceptionInInitializerError
at OAuth.main(OAuth.java:26)
Caused by: java.lang.ClassCastException: org.apache.logging.slf4j.SLF4JLoggerContext cannot be cast to org.apache.logging.log4j.core.LoggerContext
at org.apache.log4j.Logger.getLogger(Logger.java:41)
at com.etrade.etws.oauth.sdk.client.OAuthClientImpl.<init>(OAuthClientImpl.java:22)
at com.etrade.etws.oauth.sdk.client.OAuthClientImpl.<clinit>(OAuthClientImpl.java:24)
... 1 more
It seems to me that the issue is a communication issue between log4j and the E*Trade OAuth jar, although with my limited experience with Java/coding in general I have no idea how to go about fixing this issue, since I couldn't find much helpful information on Google. I decompiled the E*Trade jar and the log4j and found this code:
E*Trade:
public class OAuthClientImpl
implements IOAuthClient
{
private Logger logger = Logger.getLogger(getClass());
private static OAuthClientImpl instance = new OAuthClientImpl();
public static OAuthClientImpl getInstance()
{
return instance;
}
Log4j:
public static Logger getLogger(final Class<?> clazz) {
return (Logger) Category.getInstance((LoggerContext) PrivateManager.getContext(), clazz);
}
Below is my code, it is the E*Trade API sample code with a few minor tweaks.
import java.awt.Desktop;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import com.etrade.etws.account.Account;
import com.etrade.etws.account.AccountListResponse;
import com.etrade.etws.account.ETWSException;
import com.etrade.etws.oauth.sdk.client.IOAuthClient;
import com.etrade.etws.oauth.sdk.client.OAuthClientImpl;
import com.etrade.etws.oauth.sdk.common.Token;
import com.etrade.etws.sdk.client.ClientRequest;
import com.etrade.etws.sdk.client.Environment;
public class OAuth
{
public static void main(String[] args) throws IOException, URISyntaxException, com.etrade.etws.sdk.common.ETWSException
{
//Variables
IOAuthClient client = null;
ClientRequest request = null;
Token token = null;
String oauth_consumer_key = null; // Your consumer key
String oauth_consumer_secret = null; // Your consumer secret
String oauth_request_token = null; // Request token
String oauth_request_token_secret = null; // Request token secret
client = OAuthClientImpl.getInstance(); // Instantiate IOAUthClient
request = new ClientRequest(); // Instantiate ClientRequest
request.setEnv(Environment.SANDBOX); // Use sandbox environment
request.setConsumerKey(oauth_consumer_key); //Set consumer key
request.setConsumerSecret(oauth_consumer_secret);
token = client.getRequestToken(request); // Get request-token object
oauth_request_token = token.getToken(); // Get token string
oauth_request_token_secret = token.getSecret(); // Get token secret
}
public void OAuthVerify(IOAuthClient client, ClientRequest request) throws IOException, URISyntaxException, com.etrade.etws.sdk.common.ETWSException
{
String authorizeURL = null;
authorizeURL = client.getAuthorizeUrl(request); // E*TRADE authorization URL
URI uri = new java.net.URI(authorizeURL);
Desktop desktop = Desktop.getDesktop();
desktop.browse(uri);
}
}