We Keep Coding

Getting partial json response for s3select with aws java sdk v2

I am trying to implement s3select in a spring boot app to query parquet file in s3 bucket, I am only getting partial result from the s3select output, Please help to identify the issue, i have used aws java sdk v2.
Upon checking the json output(printed in the console), overall characters in the output is 65k.
I am using eclipse and tried unchecking "Limit console output" in the console preference, which did not help.
Code is here:-
import java.util.List;
import java.util.concurrent.CompletableFuture;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.async.SdkPublisher;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.CompressionType;
import software.amazon.awssdk.services.s3.model.EndEvent;
import software.amazon.awssdk.services.s3.model.ExpressionType;
import software.amazon.awssdk.services.s3.model.InputSerialization;
import software.amazon.awssdk.services.s3.model.JSONOutput;
import software.amazon.awssdk.services.s3.model.OutputSerialization;
import software.amazon.awssdk.services.s3.model.ParquetInput;
import software.amazon.awssdk.services.s3.model.RecordsEvent;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream.EventType;
import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponse;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
public class ParquetSelect {
private static final String BUCKET_NAME = "<bucket-name>";
private static final String KEY = "<object-key>";
private static final String QUERY = "select * from S3Object s";
public static S3AsyncClient s3;
public static void selectObjectContent() {
Handler handler = new Handler();
SelectQueryWithHandler(handler).join();
RecordsEvent recordsEvent = (RecordsEvent) handler.receivedEvents.stream()
.filter(e -> e.sdkEventType() == EventType.RECORDS)
.findFirst()
.orElse(null);
System.out.println(recordsEvent.payload().asUtf8String());
}
private static CompletableFuture<Void> SelectQueryWithHandler(SelectObjectContentResponseHandler handler) {
InputSerialization inputSerialization = InputSerialization.builder()
.parquet(ParquetInput.builder().build())
.compressionType(CompressionType.NONE)
.build();
OutputSerialization outputSerialization = OutputSerialization.builder()
.json(JSONOutput.builder().build())
.build();
SelectObjectContentRequest select = SelectObjectContentRequest.builder()
.bucket(BUCKET_NAME)
.key(KEY)
.expression(QUERY)
.expressionType(ExpressionType.SQL)
.inputSerialization(inputSerialization)
.outputSerialization(outputSerialization)
.build();
return s3.selectObjectContent(select, handler);
}
private static class Handler implements SelectObjectContentResponseHandler {
private SelectObjectContentResponse response;
private List<SelectObjectContentEventStream> receivedEvents = new ArrayList<>();
private Throwable exception;
#Override
public void responseReceived(SelectObjectContentResponse response) {
this.response = response;
}
#Override
public void onEventStream(SdkPublisher<SelectObjectContentEventStream> publisher) {
publisher.subscribe(receivedEvents::add);
}
#Override
public void exceptionOccurred(Throwable throwable) {
exception = throwable;
}
#Override
public void complete() {
}
}
}

I see you are using selectObjectContent(). Have you tried calling the s3AsyncClient.getObject() method. Does that work for you?
For example, here is a code example that gets a PDF file from an Amazon S3 bucket and write the PDF file to a local file.
package com.example.s3.async;
// snippet-start:[s3.java2.async_stream_ops.complete]
// snippet-start:[s3.java2.async_stream_ops.import]
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import java.nio.file.Paths;
import java.util.concurrent.CompletableFuture;
// snippet-end:[s3.java2.async_stream_ops.import]
// snippet-start:[s3.java2.async_stream_ops.main]
/**
* Before running this Java V2 code example, set up your development environment, including your credentials.
*
* For more information, see the following documentation topic:
*
* https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html
*/
public class S3AsyncStreamOps {
public static void main(String[] args) {
final String usage = "\n" +
"Usage:\n" +
" <bucketName> <objectKey> <path>\n\n" +
"Where:\n" +
" bucketName - The name of the Amazon S3 bucket (for example, bucket1). \n\n" +
" objectKey - The name of the object (for example, book.pdf). \n" +
" path - The local path to the file (for example, C:/AWS/book.pdf). \n" ;
if (args.length != 3) {
System.out.println(usage);
System.exit(1);
}
String bucketName = args[0];
String objectKey = args[1];
String path = args[2];
ProfileCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create();
Region region = Region.US_EAST_1;
S3AsyncClient s3AsyncClient = S3AsyncClient.builder()
.region(region)
.credentialsProvider(credentialsProvider)
.build();
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(objectKey)
.build();
CompletableFuture<GetObjectResponse> futureGet = s3AsyncClient.getObject(objectRequest,
AsyncResponseTransformer.toFile(Paths.get(path)));
futureGet.whenComplete((resp, err) -> {
try {
if (resp != null) {
System.out.println("Object downloaded. Details: "+resp);
} else {
err.printStackTrace();
}
} finally {
// Only close the client when you are completely done with it.
s3AsyncClient.close();
}
});
futureGet.join();
}
}

How do I return a hash map from HttpResponse?

I am new to Java. I am trying to hit an external API and am a little lost of how I translate the response back to a hashmap. All the examples I see online for interacting with APIs assume you want it back as a string or have an existing POJO you want to map things back to. I literally just care about extracting values in the response. In the below example, I am trying to extract out the access token from an API (and, later, in methods I haven't defined yet, use said access token to retrieve information from different endpoints).
I'm not wedded to Java 11's built-in library, either, if there is a library that requires less boilerplate.
I'm a little surprised none of my Googling has surfaced examples of API consumption where a hashmap is returned... Is converting into a self-defined POJO standard for Java? Feels like a lot of unnecessary boilerplate when I don't care about a lot of the JSON's fields.
(Also, if there is anything else wrong with this snippet, I'm open to suggestions!)
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
public class AlationApiClient {
private static final HttpClient client = HttpClient.newBuilder().build();
private static final String alationUrl = "someUrl";
private String refreshToken;
private Integer userId;
private String accessToken;
public AlationApiClient(String refreshToken, Integer userId) {
this.refreshToken = refreshToken;
this.userId = userId;
}
private static String createRequestBody(Map<String, Object> data) throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
return objectMapper
.writerWithDefaultPrettyPrinter()
.writeValueAsString(data);
}
public Object getAccessToken(String refreshToken, Integer userId) throws JsonProcessingException {
String url = alationUrl + "/integration/v1/createAPIAccessToken/";
Map<String, Object> data = Map.of(
"refresh_token", this.refreshToken,
"user_id", this.userId
);
String requestBody = createRequestBody(data);
HttpRequest request = HttpRequest.newBuilder(URI.create(url))
.POST(HttpRequest.BodyPublishers.ofString(requestBody))
.build();
return AlationApiClient.client.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(HttpResponse::body)
.thenApply(???) // what do I do here to put things into a hashmap??
.join();
}
}

Given a Ratpack RequestFixture test, how can I have the fixture invoke Request#beforeSend?

This question is in the context of a Ratpack RequestFixture Spock test, for a Ratpack chain authenticating with RatpackPac4j#requireAuth, and employing a workaround for the missing WWW-Authenticate header (as described in the answer to this question)
The problem I have is, I find that #beforeSend appears to be uncalled when the response is obtained from GroovyRequestFixture#handle (a wrapper for RequestFixture#handle). The work-around depends on this to work, so I can't test it. Is there a way to get #beforeSend called on the response represented by the HandlingResult returned?
For example, this test case fails with the assertion that the WWW-Authenticate header is present, even though the code this is adapted from inserts the header correctly when called in the actual application. The chain under test is testChain, skip to the end for the failing assertion:
package whatever
import com.google.inject.Module
import groovy.transform.Canonical
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import org.pac4j.core.profile.jwt.JwtClaims
import org.pac4j.http.client.direct.HeaderClient
import org.pac4j.jwt.config.encryption.EncryptionConfiguration
import org.pac4j.jwt.config.encryption.SecretEncryptionConfiguration
import org.pac4j.jwt.config.signature.SecretSignatureConfiguration
import org.pac4j.jwt.config.signature.SignatureConfiguration
import org.pac4j.jwt.credentials.authenticator.JwtAuthenticator
import org.pac4j.jwt.profile.JwtGenerator
import org.pac4j.jwt.profile.JwtProfile
import ratpack.groovy.handling.GroovyChainAction
import ratpack.groovy.test.handling.GroovyRequestFixture
import ratpack.guice.Guice
import ratpack.http.Response
import ratpack.http.Status
import ratpack.jackson.Jackson
import ratpack.pac4j.RatpackPac4j
import ratpack.registry.Registry
import ratpack.session.SessionModule
import ratpack.test.handling.HandlerExceptionNotThrownException
import ratpack.test.handling.HandlingResult
import spock.lang.Specification
#CompileStatic
class AuthenticatorTest extends Specification {
static byte[] salt = new byte[32] // dummy salt
static SignatureConfiguration signatureConfiguration = new SecretSignatureConfiguration(salt)
static EncryptionConfiguration encryptionConfiguration = new SecretEncryptionConfiguration(salt)
static JwtAuthenticator authenticator = new JwtAuthenticator(signatureConfiguration, encryptionConfiguration)
static JwtGenerator generator = new JwtGenerator(signatureConfiguration, encryptionConfiguration)
static HeaderClient headerClient = new HeaderClient("Authorization", "bearer ", authenticator)
/** A stripped down user class */
#Canonical
static class User {
final String id
}
/** A stripped down user registry class */
#Canonical
static class UserRegistry {
private final Map<String, String> users = [
'joebloggs': 'sekret'
]
User authenticate(String id, String password) {
if (password != null && users[id] == password)
return new User(id)
return null
}
}
/** Generates a JWT token for a given user
*
* #param userId - the name of the user
* #return A JWT token encoded as a string
*/
static String generateToken(String userId) {
JwtProfile profile = new JwtProfile()
profile.id = userId
profile.addAttribute(JwtClaims.ISSUED_AT, new Date())
String token = generator.generate(profile)
token
}
static void trapExceptions(HandlingResult result) {
try {
Throwable t = result.exception(Throwable)
throw t
}
catch (HandlerExceptionNotThrownException ignored) {
}
}
/** Composes a new registry binding the module class passed
* as per SO question https://stackoverflow.com/questions/50814817/how-do-i-mock-a-session-in-ratpack-with-requestfixture
*/
static Registry addModule(Registry registry, Class<? extends Module> module) {
Guice.registry { it.module(module) }.apply(registry)
}
GroovyChainAction testChain = new GroovyChainAction() {
#Override
#CompileDynamic
void execute() throws Exception {
register addModule(registry, SessionModule)
all RatpackPac4j.authenticator(headerClient)
all {
/*
* This is a workaround for an issue in RatpackPac4j v2.0.0, which doesn't
* add the WWW-Authenticate header by itself.
*
* See https://github.com/pac4j/ratpack-pac4j/issues/3
*
* This handler needs to be ahead of any potential causes of 401 statuses
*/
response.beforeSend { Response response ->
if (response.status.code == 401) {
response.headers.set('WWW-Authenticate', 'bearer realm="authenticated api"')
}
}
next()
}
post('login') { UserRegistry users ->
parse(Jackson.fromJson(Map)).then { Map data ->
// Validate the credentials
String id = data.user
String password = data.password
User user = users.authenticate(id, password)
if (user == null) {
clientError(401) // Bad authentication credentials
} else {
response.contentType('text/plain')
// Authenticates ok. Issue a JWT token to the client which embeds (signed, encrypted)
// certain standardised metadata of our choice that the JWT validation will use.
String token = generateToken(user.id)
render token
}
}
}
get('unprotected') {
render "hello"
}
// All subsequent paths require authentication
all RatpackPac4j.requireAuth(HeaderClient)
get('protected') {
render "hello"
}
notFound()
}
}
#CompileDynamic
def "should be denied protected path, unauthorised..."() {
given:
def result = GroovyRequestFixture.handle(testChain) {
uri 'protected'
method 'GET'
}
expect:
result.status == Status.of(401) // Unauthorized
// THIS FAILS BECAUSE Response#beforeSend WASN'T INVOKED BY GroovyRequestFixture
result.headers['WWW-Authenticate'] == 'bearer realm="authenticated api"'
// If the server threw, rethrow that
trapExceptions(result)
}
}

Best answer so far... or more strictly, a workaround to sidestep the limitations of RequestFixture, is: don't use RequestFixture. Use GroovyEmbeddedApp
(Credit to Dan Hyun on the Ratpack slack channel)
RequestFixture is only meant to check handler behavior, it doesn't do a lot of
things - it won't serialize responses. EmbeddedApp is probably the way to go
for most testing. You'd care more about overall interaction rather than how an
individual handler does a thing, unless it was a highly reused component or is
middleware that is used by other apps
An modified version of the example above follows, I've marked the modified sections in the comments:
package whatever
import com.google.inject.Module
import groovy.transform.Canonical
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import org.pac4j.core.profile.jwt.JwtClaims
import org.pac4j.http.client.direct.HeaderClient
import org.pac4j.jwt.config.encryption.EncryptionConfiguration
import org.pac4j.jwt.config.encryption.SecretEncryptionConfiguration
import org.pac4j.jwt.config.signature.SecretSignatureConfiguration
import org.pac4j.jwt.config.signature.SignatureConfiguration
import org.pac4j.jwt.credentials.authenticator.JwtAuthenticator
import org.pac4j.jwt.profile.JwtGenerator
import org.pac4j.jwt.profile.JwtProfile
import ratpack.groovy.test.embed.GroovyEmbeddedApp
import ratpack.guice.Guice
import ratpack.http.Response
import ratpack.http.Status
import ratpack.http.client.ReceivedResponse
import ratpack.jackson.Jackson
import ratpack.pac4j.RatpackPac4j
import ratpack.registry.Registry
import ratpack.session.SessionModule
import ratpack.test.handling.HandlerExceptionNotThrownException
import ratpack.test.handling.HandlingResult
import ratpack.test.http.TestHttpClient
import spock.lang.Specification
#CompileStatic
class TempTest extends Specification {
static byte[] salt = new byte[32] // dummy salt
static SignatureConfiguration signatureConfiguration = new SecretSignatureConfiguration(salt)
static EncryptionConfiguration encryptionConfiguration = new SecretEncryptionConfiguration(salt)
static JwtAuthenticator authenticator = new JwtAuthenticator(signatureConfiguration, encryptionConfiguration)
static JwtGenerator generator = new JwtGenerator(signatureConfiguration, encryptionConfiguration)
static HeaderClient headerClient = new HeaderClient("Authorization", "bearer ", authenticator)
/** A stripped down user class */
#Canonical
static class User {
final String id
}
/** A stripped down user registry class */
#Canonical
static class UserRegistry {
private final Map<String, String> users = [
'joebloggs': 'sekret'
]
User authenticate(String id, String password) {
if (password != null && users[id] == password)
return new User(id)
return null
}
}
/** Generates a JWT token for a given user
*
* #param userId - the name of the user
* #return A JWT token encoded as a string
*/
static String generateToken(String userId) {
JwtProfile profile = new JwtProfile()
profile.id = userId
profile.addAttribute(JwtClaims.ISSUED_AT, new Date())
String token = generator.generate(profile)
token
}
static void trapExceptions(HandlingResult result) {
try {
Throwable t = result.exception(Throwable)
throw t
}
catch (HandlerExceptionNotThrownException ignored) {
}
}
/** Composes a new registry binding the module class passed
* as per SO question https://stackoverflow.com/questions/50814817/how-do-i-mock-a-session-in-ratpack-with-requestfixture
*/
static Registry addModule(Registry registry, Class<? extends Module> module) {
Guice.registry { it.module(module) }.apply(registry)
}
/*** USE GroovyEmbeddedApp HERE INSTEAD OF GroovyResponseFixture ***/
GroovyEmbeddedApp testApp = GroovyEmbeddedApp.ratpack {
bindings {
module SessionModule
}
handlers {
all RatpackPac4j.authenticator(headerClient)
all {
/*
* This is a workaround for an issue in RatpackPac4j v2.0.0, which doesn't
* add the WWW-Authenticate header by itself.
*
* See https://github.com/pac4j/ratpack-pac4j/issues/3
*
* This handler needs to be ahead of any potential causes of 401 statuses
*/
response.beforeSend { Response response ->
if (response.status.code == 401) {
response.headers.set('WWW-Authenticate', 'bearer realm="authenticated api"')
}
}
next()
}
post('login') { UserRegistry users ->
parse(Jackson.fromJson(Map)).then { Map data ->
// Validate the credentials
String id = data.user
String password = data.password
User user = users.authenticate(id, password)
if (user == null) {
clientError(401) // Bad authentication credentials
} else {
response.contentType('text/plain')
// Authenticates ok. Issue a JWT token to the client which embeds (signed, encrypted)
// certain standardised metadata of our choice that the JWT validation will use.
String token = generateToken(user.id)
render token
}
}
}
get('unprotected') {
render "hello"
}
// All subsequent paths require authentication
all RatpackPac4j.requireAuth(HeaderClient)
get('protected') {
render "hello"
}
notFound()
}
}
/*** THIS NOW ALTERED TO USE testApp ***/
#CompileDynamic
def "should be denied protected path, unauthorised..."() {
given:
TestHttpClient client = testApp.httpClient
ReceivedResponse response = client.get('protected')
expect:
response.status == Status.of(401) // Unauthorized
response.headers['WWW-Authenticate'] == 'bearer realm="authenticated api"'
}
}

Amazon Advertising API: Cannot locate SignedRequestHelper class

I have got code from amazon scratchpad and got the package required. The SignedRequestHelper class is not in the package and i am unable to run the program. I am attempting to get the price of the item using the amazon asin number.
package com.amazon.advertising.api.sample;
import java.util.HashMap;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
/*
* This class shows how to make a simple authenticated call to the
* Amazon Product Advertising API.
*
* See the README.html that came with this sample for instructions on
* configuring and running the sample.
*/
public class lookup {
/*
* Your AWS Access Key ID, as taken from the AWS Your Account page.
*/
private static final String AWS_ACCESS_KEY_ID = "XXXXXX";
/*
* Your AWS Secret Key corresponding to the above ID, as taken from the AWS
* Your Account page.
*/
private static final String AWS_SECRET_KEY = "XXXXXXX";
/*
* Use the end-point according to the region you are interested in.
*/
private static final String ENDPOINT = "webservices.amazon.com";
public static void main(String[] args) {
/*
* Set up the signed requests helper.
*/
SignedRequestsHelper helper;
try {
} catch (Exception e) {
e.printStackTrace();
return;
}
String requestUrl = null;
Map<String, String> params = new HashMap<String, String>();
params.put("Service", "AWSECommerceService");
params.put("Operation", "ItemLookup");
params.put("AWSAccessKeyId", "XXXXXX");
params.put("AssociateTag", "XXXXX");
params.put("ItemId", "B01H57GXUQ");
params.put("IdType", "ASIN");
params.put("ResponseGroup", "Images,ItemAttributes,Offers");
requestUrl = helper.sign(params);
System.out.println("Signed URL: \"" + requestUrl + "\"");
}
}
How would i be able to get the signedrequestshelper method or how would i be able to change the code?

SignedRequestHelper is a class available in one of the code samples from AWS here:
http://docs.aws.amazon.com/AWSECommerceService/latest/DG/AuthJavaSampleSig2.html
You can copy/paste the code in your project to make it run and work.

Google+ API Exception in thread "main" java.lang.NoClassDefFoundError: com/google/common/base/Preconditions

I am running the YouTubeSample given on the google developers website. I have no errors in the code and my imports appear to be fine. But when I run the project I get the aforementioned error.
I have done some searches but to be honest I have been unable to work out what the problem is. I have already tried importing an external jar guava but it didn't help.
Any help is appreciated. Here is the full class
package com.pengilleys.googlesamples;
import java.io.IOException;
import java.util.List;
import com.google.api.client.googleapis.GoogleHeaders;
import com.google.api.client.googleapis.json.JsonCParser;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.client.util.Key;
public class YouTubeSample {
public static class VideoFeed {
#Key List<Video> items;
}
public static class Video {
#Key String title;
#Key String description;
#Key Player player;
}
public static class Player {
#Key("default") String defaultUrl;
}
public static class YouTubeUrl extends GenericUrl {
#Key final String alt = "jsonc";
#Key String author;
#Key("max-results") Integer maxResults;
YouTubeUrl(String url) {
super(url);
}
}
public static void main(String[] args) throws IOException {
// set up the HTTP request factory
HttpTransport transport = new NetHttpTransport();
final JsonFactory jsonFactory = new JacksonFactory();
HttpRequestFactory factory = transport.createRequestFactory(new HttpRequestInitializer() {
#Override
public void initialize(HttpRequest request) {
// set the parser
JsonCParser parser = new JsonCParser();
parser.jsonFactory = jsonFactory;
request.addParser(parser);
// set up the Google headers
GoogleHeaders headers = new GoogleHeaders();
headers.setApplicationName("Google-YouTubeSample/1.0");
headers.gdataVersion = "2";
request.headers = headers;
}
});
// build the YouTube URL
YouTubeUrl url = new YouTubeUrl("https://gdata.youtube.com/feeds/api/videos");
url.author = "searchstories";
url.maxResults = 2;
// build the HTTP GET request
HttpRequest request = factory.buildGetRequest(url);
// execute the request and the parse video feed
VideoFeed feed = request.execute().parseAs(VideoFeed.class);
for (Video video : feed.items) {
System.out.println();
System.out.println("Video title: " + video.title);
System.out.println("Description: " + video.description);
System.out.println("Play URL: " + video.player.defaultUrl);
}
}
}

The setup documentation gives a list of dependencies:
Depending on the application you are building, you may also need these dependencies:
Apache HTTP Client version 4.0.3
Google Guava version r09
Jackson version 1.6.7
Google GSON version 1.6
In this case, it looks like it's Guava which is missing. I don't know what you mean about "exporting" Guava, but if you include the Guava r09 jar file in the classpath when you're running the code, it should be fine.

what's the extra ); for above the // build the YouTube URL and did you mean to close main on that line?