We Keep Coding

Getting partial json response for s3select with aws java sdk v2

I am trying to implement s3select in a spring boot app to query parquet file in s3 bucket, I am only getting partial result from the s3select output, Please help to identify the issue, i have used aws java sdk v2.
Upon checking the json output(printed in the console), overall characters in the output is 65k.
I am using eclipse and tried unchecking "Limit console output" in the console preference, which did not help.
Code is here:-
import java.util.List;
import java.util.concurrent.CompletableFuture;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.async.SdkPublisher;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.CompressionType;
import software.amazon.awssdk.services.s3.model.EndEvent;
import software.amazon.awssdk.services.s3.model.ExpressionType;
import software.amazon.awssdk.services.s3.model.InputSerialization;
import software.amazon.awssdk.services.s3.model.JSONOutput;
import software.amazon.awssdk.services.s3.model.OutputSerialization;
import software.amazon.awssdk.services.s3.model.ParquetInput;
import software.amazon.awssdk.services.s3.model.RecordsEvent;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream;
import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream.EventType;
import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponse;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
public class ParquetSelect {
private static final String BUCKET_NAME = "<bucket-name>";
private static final String KEY = "<object-key>";
private static final String QUERY = "select * from S3Object s";
public static S3AsyncClient s3;
public static void selectObjectContent() {
Handler handler = new Handler();
SelectQueryWithHandler(handler).join();
RecordsEvent recordsEvent = (RecordsEvent) handler.receivedEvents.stream()
.filter(e -> e.sdkEventType() == EventType.RECORDS)
.findFirst()
.orElse(null);
System.out.println(recordsEvent.payload().asUtf8String());
}
private static CompletableFuture<Void> SelectQueryWithHandler(SelectObjectContentResponseHandler handler) {
InputSerialization inputSerialization = InputSerialization.builder()
.parquet(ParquetInput.builder().build())
.compressionType(CompressionType.NONE)
.build();
OutputSerialization outputSerialization = OutputSerialization.builder()
.json(JSONOutput.builder().build())
.build();
SelectObjectContentRequest select = SelectObjectContentRequest.builder()
.bucket(BUCKET_NAME)
.key(KEY)
.expression(QUERY)
.expressionType(ExpressionType.SQL)
.inputSerialization(inputSerialization)
.outputSerialization(outputSerialization)
.build();
return s3.selectObjectContent(select, handler);
}
private static class Handler implements SelectObjectContentResponseHandler {
private SelectObjectContentResponse response;
private List<SelectObjectContentEventStream> receivedEvents = new ArrayList<>();
private Throwable exception;
#Override
public void responseReceived(SelectObjectContentResponse response) {
this.response = response;
}
#Override
public void onEventStream(SdkPublisher<SelectObjectContentEventStream> publisher) {
publisher.subscribe(receivedEvents::add);
}
#Override
public void exceptionOccurred(Throwable throwable) {
exception = throwable;
}
#Override
public void complete() {
}
}
}

I see you are using selectObjectContent(). Have you tried calling the s3AsyncClient.getObject() method. Does that work for you?
For example, here is a code example that gets a PDF file from an Amazon S3 bucket and write the PDF file to a local file.
package com.example.s3.async;
// snippet-start:[s3.java2.async_stream_ops.complete]
// snippet-start:[s3.java2.async_stream_ops.import]
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import java.nio.file.Paths;
import java.util.concurrent.CompletableFuture;
// snippet-end:[s3.java2.async_stream_ops.import]
// snippet-start:[s3.java2.async_stream_ops.main]
/**
* Before running this Java V2 code example, set up your development environment, including your credentials.
*
* For more information, see the following documentation topic:
*
* https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html
*/
public class S3AsyncStreamOps {
public static void main(String[] args) {
final String usage = "\n" +
"Usage:\n" +
" <bucketName> <objectKey> <path>\n\n" +
"Where:\n" +
" bucketName - The name of the Amazon S3 bucket (for example, bucket1). \n\n" +
" objectKey - The name of the object (for example, book.pdf). \n" +
" path - The local path to the file (for example, C:/AWS/book.pdf). \n" ;
if (args.length != 3) {
System.out.println(usage);
System.exit(1);
}
String bucketName = args[0];
String objectKey = args[1];
String path = args[2];
ProfileCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create();
Region region = Region.US_EAST_1;
S3AsyncClient s3AsyncClient = S3AsyncClient.builder()
.region(region)
.credentialsProvider(credentialsProvider)
.build();
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(objectKey)
.build();
CompletableFuture<GetObjectResponse> futureGet = s3AsyncClient.getObject(objectRequest,
AsyncResponseTransformer.toFile(Paths.get(path)));
futureGet.whenComplete((resp, err) -> {
try {
if (resp != null) {
System.out.println("Object downloaded. Details: "+resp);
} else {
err.printStackTrace();
}
} finally {
// Only close the client when you are completely done with it.
s3AsyncClient.close();
}
});
futureGet.join();
}
}

Spring Integration Java DSL SFTP how to get remote SFTP server information in handler

I am trying to download files from multiple SFTP servers then handle those files. But I can not get the information of remote SFTP server such as: IpAddress, remoteDirectory depending on which file MessageHandler handling. Instead Payload only contains the information of the dowloaded files at local. Here the source code I use from the guide:
How to dynamically define file filter pattern for Spring Integration SFTP Inbound Adapter?
SFTIntegration.java
import com.jcraft.jsch.ChannelSftp.LsEntry;
import java.io.File;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.channel.NullChannel;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.dsl.Pollers;
import org.springframework.integration.dsl.SourcePollingChannelAdapterSpec;
import org.springframework.integration.expression.FunctionExpression;
import org.springframework.integration.file.remote.aop.RotatingServerAdvice;
import org.springframework.integration.file.remote.session.DelegatingSessionFactory;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.scheduling.PollerMetadata;
import org.springframework.integration.sftp.dsl.Sftp;
import org.springframework.integration.sftp.dsl.SftpInboundChannelAdapterSpec;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
import org.springframework.messaging.MessageChannel;
import org.springframework.stereotype.Component;
/**
* flow.
*/
#Configuration
#Component
public class SFTIntegration {
public static final String TIMEZONE_UTC = "UTC";
public static final String TIMESTAMP_FORMAT_OF_FILES = "yyyyMMddHHmmssSSS";
public static final String TEMPORARY_FILE_SUFFIX = ".part";
public static final int POLLER_FIXED_PERIOD_DELAY = 60000;
public static final int MAX_MESSAGES_PER_POLL = 100;
private static final Logger LOG = LoggerFactory.getLogger(SFTIntegration.class);
private static final String CHANNEL_INTERMEDIATE_STAGE = "intermediateChannel";
#Autowired
private ImportHandler importHandler;
/** database access repository */
private final SFTPServerConfigRepo SFTPServerConfigRepo;
#Value("${sftp.local.directory.download:${java.io.tmpdir}/localDownload}")
private String localTemporaryPath;
public SFTIntegration(final SFTPServerConfigRepo SFTPServerConfigRepo) {
this.SFTPServerConfigRepo = SFTPServerConfigRepo;
}
/**
* The default poller with 5s, 100 messages, RotatingServerAdvice and transaction.
*
* #return default poller.
*/
#Bean(name = PollerMetadata.DEFAULT_POLLER)
public PollerMetadata poller() {
return Pollers
.fixedDelay(POLLER_FIXED_PERIOD_DELAY)
.advice(advice())
.maxMessagesPerPoll(MAX_MESSAGES_PER_POLL)
.transactional()
.get();
}
/**
* The direct channel for the flow.
*
* #return MessageChannel
*/
#Bean
public MessageChannel stockIntermediateChannel() {
return new DirectChannel();
}
/**
* Get the files from a remote directory. Add a timestamp to the filename
* and write them to a local temporary folder.
*
* #return IntegrationFlow
*/
#Bean
public IntegrationFlow collectionInboundFlowFromSFTPServer() {
// Source definition
final SftpInboundChannelAdapterSpec sourceSpec = Sftp.inboundAdapter(delegatingSFtpSessionFactory())
.preserveTimestamp(true)
.patternFilter("*.*")
.deleteRemoteFiles(true)
.maxFetchSize(MAX_MESSAGES_PER_POLL)
.remoteDirectory("/")
.localDirectory(new File(localTemporaryPath))
.temporaryFileSuffix(TEMPORARY_FILE_SUFFIX)
.localFilenameExpression(new FunctionExpression<String>(s -> {
final int fileTypeSepPos = s.lastIndexOf('.');
return
DateTimeFormatter
.ofPattern(TIMESTAMP_FORMAT_OF_FILES)
.withZone(ZoneId.of(TIMEZONE_UTC))
.format(Instant.now())
+ "_"
+ s.substring(0, fileTypeSepPos)
+ s.substring(fileTypeSepPos);
}));
// Poller definition
final Consumer<SourcePollingChannelAdapterSpec> collectionInboundPoller = endpointConfigurer -> endpointConfigurer
.id("collectionInboundPoller")
.autoStartup(true)
.poller(poller());
return IntegrationFlows
.from(sourceSpec, collectionInboundPoller)
.transform(File.class, p -> {
// log step
LOG.info("flow=collectionInboundFlowFromSFTPServer, message=incoming file: " + p);
return p;
})
.channel(CHANNEL_INTERMEDIATE_STAGE)
.get();
}
#Bean
public IntegrationFlow collectionIntermediateStageChannel() {
return IntegrationFlows
.from(CHANNEL_INTERMEDIATE_STAGE)
.handle(importHandler)
.channel(new NullChannel())
.get();
}
public DefaultSftpSessionFactory createNewSftpSessionFactory(final SFTPServerConfig pc) {
final DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory(
false);
factory.setHost(pc.getServerIp());
factory.setPort(pc.getPort());
factory.setUser(pc.getUsername());
factory.setPassword(pc.getPassword());
factory.setAllowUnknownKeys(true);
return factory;
}
#Bean
public DelegatingSessionFactory<LsEntry> delegatingSFtpSessionFactory() {
final List<SFTPServerConfig> partnerConnections = SFTPServerConfigRepo.findAll();
if (partnerConnections.isEmpty()) {
return null;
}
final Map<Object, SessionFactory<LsEntry>> factories = new LinkedHashMap<>(10);
for (SFTPServerConfig pc : partnerConnections) {
// create a factory for every key containing server type, url and port
if (factories.get(pc.getKey()) == null) {
factories.put(pc.getKey(), createNewSftpSessionFactory(pc));
}
}
// use the first SF as the default
return new DelegatingSessionFactory<>(factories, factories.values().iterator().next());
}
#Bean
public RotatingServerAdvice advice() {
final List<SFTPServerConfig> sftpConnections = SFTPServerConfigRepo.findAll();
final List<RotatingServerAdvice.KeyDirectory> keyDirectories = new ArrayList<>();
for (SFTPServerConfig pc : sftpConnections) {
keyDirectories
.add(new RotatingServerAdvice.KeyDirectory(pc.getKey(), pc.getServerPath()));
}
return new RotatingServerAdvice(delegatingSFtpSessionFactory(), keyDirectories, true);
}
}
ImportHandler.java
import org.springframework.messaging.Message;
import org.springframework.stereotype.Service;
#Service
public class ImportHandler {
public void handle(Message<?> message) {
System.out.println("Hello " + message);
System.out.println(message.getPayload());
System.out.println(message.getHeaders());
//How can I get the information of remote server Ip address, remoteDirectory here where the file comes from
}
}
If you have any ideas, please let me know. Thank you so much!.

It's not currently supported; please open a new feature request.

Given a Ratpack RequestFixture test, how can I have the fixture invoke Request#beforeSend?

This question is in the context of a Ratpack RequestFixture Spock test, for a Ratpack chain authenticating with RatpackPac4j#requireAuth, and employing a workaround for the missing WWW-Authenticate header (as described in the answer to this question)
The problem I have is, I find that #beforeSend appears to be uncalled when the response is obtained from GroovyRequestFixture#handle (a wrapper for RequestFixture#handle). The work-around depends on this to work, so I can't test it. Is there a way to get #beforeSend called on the response represented by the HandlingResult returned?
For example, this test case fails with the assertion that the WWW-Authenticate header is present, even though the code this is adapted from inserts the header correctly when called in the actual application. The chain under test is testChain, skip to the end for the failing assertion:
package whatever
import com.google.inject.Module
import groovy.transform.Canonical
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import org.pac4j.core.profile.jwt.JwtClaims
import org.pac4j.http.client.direct.HeaderClient
import org.pac4j.jwt.config.encryption.EncryptionConfiguration
import org.pac4j.jwt.config.encryption.SecretEncryptionConfiguration
import org.pac4j.jwt.config.signature.SecretSignatureConfiguration
import org.pac4j.jwt.config.signature.SignatureConfiguration
import org.pac4j.jwt.credentials.authenticator.JwtAuthenticator
import org.pac4j.jwt.profile.JwtGenerator
import org.pac4j.jwt.profile.JwtProfile
import ratpack.groovy.handling.GroovyChainAction
import ratpack.groovy.test.handling.GroovyRequestFixture
import ratpack.guice.Guice
import ratpack.http.Response
import ratpack.http.Status
import ratpack.jackson.Jackson
import ratpack.pac4j.RatpackPac4j
import ratpack.registry.Registry
import ratpack.session.SessionModule
import ratpack.test.handling.HandlerExceptionNotThrownException
import ratpack.test.handling.HandlingResult
import spock.lang.Specification
#CompileStatic
class AuthenticatorTest extends Specification {
static byte[] salt = new byte[32] // dummy salt
static SignatureConfiguration signatureConfiguration = new SecretSignatureConfiguration(salt)
static EncryptionConfiguration encryptionConfiguration = new SecretEncryptionConfiguration(salt)
static JwtAuthenticator authenticator = new JwtAuthenticator(signatureConfiguration, encryptionConfiguration)
static JwtGenerator generator = new JwtGenerator(signatureConfiguration, encryptionConfiguration)
static HeaderClient headerClient = new HeaderClient("Authorization", "bearer ", authenticator)
/** A stripped down user class */
#Canonical
static class User {
final String id
}
/** A stripped down user registry class */
#Canonical
static class UserRegistry {
private final Map<String, String> users = [
'joebloggs': 'sekret'
]
User authenticate(String id, String password) {
if (password != null && users[id] == password)
return new User(id)
return null
}
}
/** Generates a JWT token for a given user
*
* #param userId - the name of the user
* #return A JWT token encoded as a string
*/
static String generateToken(String userId) {
JwtProfile profile = new JwtProfile()
profile.id = userId
profile.addAttribute(JwtClaims.ISSUED_AT, new Date())
String token = generator.generate(profile)
token
}
static void trapExceptions(HandlingResult result) {
try {
Throwable t = result.exception(Throwable)
throw t
}
catch (HandlerExceptionNotThrownException ignored) {
}
}
/** Composes a new registry binding the module class passed
* as per SO question https://stackoverflow.com/questions/50814817/how-do-i-mock-a-session-in-ratpack-with-requestfixture
*/
static Registry addModule(Registry registry, Class<? extends Module> module) {
Guice.registry { it.module(module) }.apply(registry)
}
GroovyChainAction testChain = new GroovyChainAction() {
#Override
#CompileDynamic
void execute() throws Exception {
register addModule(registry, SessionModule)
all RatpackPac4j.authenticator(headerClient)
all {
/*
* This is a workaround for an issue in RatpackPac4j v2.0.0, which doesn't
* add the WWW-Authenticate header by itself.
*
* See https://github.com/pac4j/ratpack-pac4j/issues/3
*
* This handler needs to be ahead of any potential causes of 401 statuses
*/
response.beforeSend { Response response ->
if (response.status.code == 401) {
response.headers.set('WWW-Authenticate', 'bearer realm="authenticated api"')
}
}
next()
}
post('login') { UserRegistry users ->
parse(Jackson.fromJson(Map)).then { Map data ->
// Validate the credentials
String id = data.user
String password = data.password
User user = users.authenticate(id, password)
if (user == null) {
clientError(401) // Bad authentication credentials
} else {
response.contentType('text/plain')
// Authenticates ok. Issue a JWT token to the client which embeds (signed, encrypted)
// certain standardised metadata of our choice that the JWT validation will use.
String token = generateToken(user.id)
render token
}
}
}
get('unprotected') {
render "hello"
}
// All subsequent paths require authentication
all RatpackPac4j.requireAuth(HeaderClient)
get('protected') {
render "hello"
}
notFound()
}
}
#CompileDynamic
def "should be denied protected path, unauthorised..."() {
given:
def result = GroovyRequestFixture.handle(testChain) {
uri 'protected'
method 'GET'
}
expect:
result.status == Status.of(401) // Unauthorized
// THIS FAILS BECAUSE Response#beforeSend WASN'T INVOKED BY GroovyRequestFixture
result.headers['WWW-Authenticate'] == 'bearer realm="authenticated api"'
// If the server threw, rethrow that
trapExceptions(result)
}
}

Best answer so far... or more strictly, a workaround to sidestep the limitations of RequestFixture, is: don't use RequestFixture. Use GroovyEmbeddedApp
(Credit to Dan Hyun on the Ratpack slack channel)
RequestFixture is only meant to check handler behavior, it doesn't do a lot of
things - it won't serialize responses. EmbeddedApp is probably the way to go
for most testing. You'd care more about overall interaction rather than how an
individual handler does a thing, unless it was a highly reused component or is
middleware that is used by other apps
An modified version of the example above follows, I've marked the modified sections in the comments:
package whatever
import com.google.inject.Module
import groovy.transform.Canonical
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
import org.pac4j.core.profile.jwt.JwtClaims
import org.pac4j.http.client.direct.HeaderClient
import org.pac4j.jwt.config.encryption.EncryptionConfiguration
import org.pac4j.jwt.config.encryption.SecretEncryptionConfiguration
import org.pac4j.jwt.config.signature.SecretSignatureConfiguration
import org.pac4j.jwt.config.signature.SignatureConfiguration
import org.pac4j.jwt.credentials.authenticator.JwtAuthenticator
import org.pac4j.jwt.profile.JwtGenerator
import org.pac4j.jwt.profile.JwtProfile
import ratpack.groovy.test.embed.GroovyEmbeddedApp
import ratpack.guice.Guice
import ratpack.http.Response
import ratpack.http.Status
import ratpack.http.client.ReceivedResponse
import ratpack.jackson.Jackson
import ratpack.pac4j.RatpackPac4j
import ratpack.registry.Registry
import ratpack.session.SessionModule
import ratpack.test.handling.HandlerExceptionNotThrownException
import ratpack.test.handling.HandlingResult
import ratpack.test.http.TestHttpClient
import spock.lang.Specification
#CompileStatic
class TempTest extends Specification {
static byte[] salt = new byte[32] // dummy salt
static SignatureConfiguration signatureConfiguration = new SecretSignatureConfiguration(salt)
static EncryptionConfiguration encryptionConfiguration = new SecretEncryptionConfiguration(salt)
static JwtAuthenticator authenticator = new JwtAuthenticator(signatureConfiguration, encryptionConfiguration)
static JwtGenerator generator = new JwtGenerator(signatureConfiguration, encryptionConfiguration)
static HeaderClient headerClient = new HeaderClient("Authorization", "bearer ", authenticator)
/** A stripped down user class */
#Canonical
static class User {
final String id
}
/** A stripped down user registry class */
#Canonical
static class UserRegistry {
private final Map<String, String> users = [
'joebloggs': 'sekret'
]
User authenticate(String id, String password) {
if (password != null && users[id] == password)
return new User(id)
return null
}
}
/** Generates a JWT token for a given user
*
* #param userId - the name of the user
* #return A JWT token encoded as a string
*/
static String generateToken(String userId) {
JwtProfile profile = new JwtProfile()
profile.id = userId
profile.addAttribute(JwtClaims.ISSUED_AT, new Date())
String token = generator.generate(profile)
token
}
static void trapExceptions(HandlingResult result) {
try {
Throwable t = result.exception(Throwable)
throw t
}
catch (HandlerExceptionNotThrownException ignored) {
}
}
/** Composes a new registry binding the module class passed
* as per SO question https://stackoverflow.com/questions/50814817/how-do-i-mock-a-session-in-ratpack-with-requestfixture
*/
static Registry addModule(Registry registry, Class<? extends Module> module) {
Guice.registry { it.module(module) }.apply(registry)
}
/*** USE GroovyEmbeddedApp HERE INSTEAD OF GroovyResponseFixture ***/
GroovyEmbeddedApp testApp = GroovyEmbeddedApp.ratpack {
bindings {
module SessionModule
}
handlers {
all RatpackPac4j.authenticator(headerClient)
all {
/*
* This is a workaround for an issue in RatpackPac4j v2.0.0, which doesn't
* add the WWW-Authenticate header by itself.
*
* See https://github.com/pac4j/ratpack-pac4j/issues/3
*
* This handler needs to be ahead of any potential causes of 401 statuses
*/
response.beforeSend { Response response ->
if (response.status.code == 401) {
response.headers.set('WWW-Authenticate', 'bearer realm="authenticated api"')
}
}
next()
}
post('login') { UserRegistry users ->
parse(Jackson.fromJson(Map)).then { Map data ->
// Validate the credentials
String id = data.user
String password = data.password
User user = users.authenticate(id, password)
if (user == null) {
clientError(401) // Bad authentication credentials
} else {
response.contentType('text/plain')
// Authenticates ok. Issue a JWT token to the client which embeds (signed, encrypted)
// certain standardised metadata of our choice that the JWT validation will use.
String token = generateToken(user.id)
render token
}
}
}
get('unprotected') {
render "hello"
}
// All subsequent paths require authentication
all RatpackPac4j.requireAuth(HeaderClient)
get('protected') {
render "hello"
}
notFound()
}
}
/*** THIS NOW ALTERED TO USE testApp ***/
#CompileDynamic
def "should be denied protected path, unauthorised..."() {
given:
TestHttpClient client = testApp.httpClient
ReceivedResponse response = client.get('protected')
expect:
response.status == Status.of(401) // Unauthorized
response.headers['WWW-Authenticate'] == 'bearer realm="authenticated api"'
}
}

Google Pub/Sub Java examples

I'm not able to find a way to read messages from pub/sub using java.
I'm using this maven dependency in my pom
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-pubsub</artifactId>
<version>0.17.2-alpha</version>
</dependency>
I implemented this main method to create a new topic:
public static void main(String... args) throws Exception {
// Your Google Cloud Platform project ID
String projectId = ServiceOptions.getDefaultProjectId();
// Your topic ID
String topicId = "my-new-topic-1";
// Create a new topic
TopicName topic = TopicName.create(projectId, topicId);
try (TopicAdminClient topicAdminClient = TopicAdminClient.create()) {
topicAdminClient.createTopic(topic);
}
}
The above code works well and, indeed, I can see the new topic I created using the google cloud console.
I implemented the following main method to write a message to my topic:
public static void main(String a[]) throws InterruptedException, ExecutionException{
String projectId = ServiceOptions.getDefaultProjectId();
String topicId = "my-new-topic-1";
String payload = "Hellooooo!!!";
PubsubMessage pubsubMessage =
PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(payload)).build();
TopicName topic = TopicName.create(projectId, topicId);
Publisher publisher;
try {
publisher = Publisher.defaultBuilder(
topic)
.build();
publisher.publish(pubsubMessage);
System.out.println("Sent!");
} catch (IOException e) {
System.out.println("Not Sended!");
e.printStackTrace();
}
}
Now I'm not able to verify if this message was really sent.
I would like to implement a message reader using a subscription to my topic.
Could someone show me a correct and working java example about reading messages from a topic?
Anyone can help me?
Thanks in advance!

Here is the version using the google cloud client libraries.
package com.techm.data.client;
import com.google.cloud.pubsub.v1.AckReplyConsumer;
import com.google.cloud.pubsub.v1.MessageReceiver;
import com.google.cloud.pubsub.v1.Subscriber;
import com.google.cloud.pubsub.v1.SubscriptionAdminClient;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.pubsub.v1.ProjectSubscriptionName;
import com.google.pubsub.v1.ProjectTopicName;
import com.google.pubsub.v1.PubsubMessage;
import com.google.pubsub.v1.PushConfig;
/**
* A snippet for Google Cloud Pub/Sub showing how to create a Pub/Sub pull
* subscription and asynchronously pull messages from it.
*/
public class CreateSubscriptionAndConsumeMessages {
private static String projectId = "projectId";
private static String topicId = "topicName";
private static String subscriptionId = "subscriptionName";
public static void createSubscription() throws Exception {
ProjectTopicName topic = ProjectTopicName.of(projectId, topicId);
ProjectSubscriptionName subscription = ProjectSubscriptionName.of(projectId, subscriptionId);
try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create()) {
subscriptionAdminClient.createSubscription(subscription, topic, PushConfig.getDefaultInstance(), 0);
}
}
public static void main(String... args) throws Exception {
ProjectSubscriptionName subscription = ProjectSubscriptionName.of(projectId, subscriptionId);
createSubscription();
MessageReceiver receiver = new MessageReceiver() {
#Override
public void receiveMessage(PubsubMessage message, AckReplyConsumer consumer) {
System.out.println("Received message: " + message.getData().toStringUtf8());
consumer.ack();
}
};
Subscriber subscriber = null;
try {
subscriber = Subscriber.newBuilder(subscription, receiver).build();
subscriber.addListener(new Subscriber.Listener() {
#Override
public void failed(Subscriber.State from, Throwable failure) {
// Handle failure. This is called when the Subscriber encountered a fatal error
// and is
// shutting down.
System.err.println(failure);
}
}, MoreExecutors.directExecutor());
subscriber.startAsync().awaitRunning();
// In this example, we will pull messages for one minute (60,000ms) then stop.
// In a real application, this sleep-then-stop is not necessary.
// Simply call stopAsync().awaitTerminated() when the server is shutting down,
// etc.
Thread.sleep(60000);
} finally {
if (subscriber != null) {
subscriber.stopAsync().awaitTerminated();
}
}
}
}
This is working fine for me.

The Cloud Pub/Sub Pull Subscriber Guide has sample code for reading messages from a topic.

I haven't used google cloud client libraries but used the api client libraries. Here is how I created a subscription.
package com.techm.datapipeline.client;
import java.io.IOException;
import java.security.GeneralSecurityException;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.HttpStatusCodes;
import com.google.api.services.pubsub.Pubsub;
import com.google.api.services.pubsub.Pubsub.Projects.Subscriptions.Create;
import com.google.api.services.pubsub.Pubsub.Projects.Subscriptions.Get;
import com.google.api.services.pubsub.Pubsub.Projects.Topics;
import com.google.api.services.pubsub.model.ExpirationPolicy;
import com.google.api.services.pubsub.model.Subscription;
import com.google.api.services.pubsub.model.Topic;
import com.techm.datapipeline.factory.PubsubFactory;
public class CreatePullSubscriberClient {
private final static String PROJECT_NAME = "yourProjectId";
private final static String TOPIC_NAME = "yourTopicName";
private final static String SUBSCRIPTION_NAME = "yourSubscriptionName";
public static void main(String[] args) throws IOException, GeneralSecurityException {
Pubsub pubSub = PubsubFactory.getService();
String topicName = String.format("projects/%s/topics/%s", PROJECT_NAME, TOPIC_NAME);
String subscriptionName = String.format("projects/%s/subscriptions/%s", PROJECT_NAME, SUBSCRIPTION_NAME);
Topics.Get listReq = pubSub.projects().topics().get(topicName);
Topic topic = listReq.execute();
if (topic == null) {
System.err.println("Topic doesn't exist...run CreateTopicClient...to create the topic");
System.exit(0);
}
Subscription subscription = null;
try {
Get getReq = pubSub.projects().subscriptions().get(subscriptionName);
subscription = getReq.execute();
} catch (GoogleJsonResponseException e) {
if (e.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
System.out.println("Subscription " + subscriptionName + " does not exist...will create it");
}
}
if (subscription != null) {
System.out.println("Subscription already exists ==> " + subscription.toPrettyString());
System.exit(0);
}
subscription = new Subscription();
subscription.setTopic(topicName);
subscription.setPushConfig(null); // indicating a pull
ExpirationPolicy expirationPolicy = new ExpirationPolicy();
expirationPolicy.setTtl(null); // never expires;
subscription.setExpirationPolicy(expirationPolicy);
subscription.setAckDeadlineSeconds(null); // so defaults to 10 sec
subscription.setRetainAckedMessages(true);
Long _week = 7L * 24 * 60 * 60;
subscription.setMessageRetentionDuration(String.valueOf(_week)+"s");
subscription.setName(subscriptionName);
Create createReq = pubSub.projects().subscriptions().create(subscriptionName, subscription);
Subscription createdSubscription = createReq.execute();
System.out.println("Subscription created ==> " + createdSubscription.toPrettyString());
}
}
And once you create the subscription (pull type)...this is how you pull the messages from the topic.
package com.techm.datapipeline.client;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.List;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.HttpStatusCodes;
import com.google.api.client.util.Base64;
import com.google.api.services.pubsub.Pubsub;
import com.google.api.services.pubsub.Pubsub.Projects.Subscriptions.Acknowledge;
import com.google.api.services.pubsub.Pubsub.Projects.Subscriptions.Get;
import com.google.api.services.pubsub.Pubsub.Projects.Subscriptions.Pull;
import com.google.api.services.pubsub.model.AcknowledgeRequest;
import com.google.api.services.pubsub.model.Empty;
import com.google.api.services.pubsub.model.PullRequest;
import com.google.api.services.pubsub.model.PullResponse;
import com.google.api.services.pubsub.model.ReceivedMessage;
import com.techm.datapipeline.factory.PubsubFactory;
public class PullSubscriptionsClient {
private final static String PROJECT_NAME = "yourProjectId";
private final static String SUBSCRIPTION_NAME = "yourSubscriptionName";
private final static String SUBSCRIPTION_NYC_NAME = "test";
public static void main(String[] args) throws IOException, GeneralSecurityException {
Pubsub pubSub = PubsubFactory.getService();
String subscriptionName = String.format("projects/%s/subscriptions/%s", PROJECT_NAME, SUBSCRIPTION_NAME);
//String subscriptionName = String.format("projects/%s/subscriptions/%s", PROJECT_NAME, SUBSCRIPTION_NYC_NAME);
try {
Get getReq = pubSub.projects().subscriptions().get(subscriptionName);
getReq.execute();
} catch (GoogleJsonResponseException e) {
if (e.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
System.out.println("Subscription " + subscriptionName
+ " does not exist...run CreatePullSubscriberClient to create");
}
}
PullRequest pullRequest = new PullRequest();
pullRequest.setReturnImmediately(false); // wait until you get a message
pullRequest.setMaxMessages(1000);
Pull pullReq = pubSub.projects().subscriptions().pull(subscriptionName, pullRequest);
PullResponse pullResponse = pullReq.execute();
List<ReceivedMessage> msgs = pullResponse.getReceivedMessages();
List<String> ackIds = new ArrayList<String>();
int i = 0;
if (msgs != null) {
for (ReceivedMessage msg : msgs) {
ackIds.add(msg.getAckId());
//System.out.println(i++ + ":===:" + msg.getAckId());
String object = new String(Base64.decodeBase64(msg.getMessage().getData()));
System.out.println("Decoded object String ==> " + object );
}
//acknowledge all the received messages
AcknowledgeRequest content = new AcknowledgeRequest();
content.setAckIds(ackIds);
Acknowledge ackReq = pubSub.projects().subscriptions().acknowledge(subscriptionName, content);
Empty empty = ackReq.execute();
}
}
}
Note: This client only waits until it receives at least one message and terminates if it's receives one (up to a max of value - set in MaxMessages) at once.
Let me know if this helps. I'm going to try the cloud client libraries soon and will post an update once I get my hands on them.
And here's the missing factory class ...if you plan to run it...
package com.techm.datapipeline.factory;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.pubsub.Pubsub;
import com.google.api.services.pubsub.PubsubScopes;
public class PubsubFactory {
private static Pubsub instance = null;
private static final Logger logger = Logger.getLogger(PubsubFactory.class.getName());
public static synchronized Pubsub getService() throws IOException, GeneralSecurityException {
if (instance == null) {
instance = buildService();
}
return instance;
}
private static Pubsub buildService() throws IOException, GeneralSecurityException {
logger.log(Level.FINER, "Start of buildService");
HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
JsonFactory jsonFactory = new JacksonFactory();
GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
// Depending on the environment that provides the default credentials (for
// example: Compute Engine, App Engine), the credentials may require us to
// specify the scopes we need explicitly.
if (credential.createScopedRequired()) {
Collection<String> scopes = new ArrayList<>();
scopes.add(PubsubScopes.PUBSUB);
credential = credential.createScoped(scopes);
}
logger.log(Level.FINER, "End of buildService");
// TODO - Get the application name from outside.
return new Pubsub.Builder(transport, jsonFactory, credential).setApplicationName("Your Application Name/Version")
.build();
}
}

The message reader is injected on the subscriber. This part of the code will handle the messages:
MessageReceiver receiver =
new MessageReceiver() {
#Override
public void receiveMessage(PubsubMessage message, AckReplyConsumer consumer) {
// handle incoming message, then ack/nack the received message
System.out.println("Id : " + message.getMessageId());
System.out.println("Data : " + message.getData().toStringUtf8());
consumer.ack();
}
};

remote akka system port conflict

I am working on a play framework 2 system using java .The play framework uses the akka system integrated with play framework to connect with a remote akka system. the remote akka system is made up of a master node and a worker node. Both systems are on the same computer with eclipse juno IDE
I have configured two ports 2552 for the master node and port 2553 for the worker node.the akka node on the play 2 framework is selected by the system itself. the akka system in the play framework is expected to ppass a messgae to the remote master node by remote lookup using the akka configuration. the master node interns also pass the message to the remote worker for processing by a remote lookup as well. The master node and the worker node have thier application.conf files in the following formart :
src/main/resources/application.conf
however on start up both the master node and the worker node decide to use port number 2552 for their communication. I present the code snippets below :
this i the code for the play framewrok application.config file.
localNode {
akka {
actor {
provider = "akka.remote.RemoteActorRefProvider"
}
remote {
transport = "akka.remote.netty.NettyRemoteTransport"
netty {
hostname = "127.0.0.1"
port = 0
}
}
}
}
this is the configuation for the play localNode
package controllers;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import play.libs.F.Callback;
import play.mvc.WebSocket;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import akka.actor.Props;
import akka.serialization.Serialization;
import akka.serialization.SerializationExtension;
import com.typesafe.config.ConfigFactory;
import Com.RubineEngine.GesturePoints.*;
public class LocalNode {
ActorSystem csystem;
ActorRef localActor ;
public LocalNode() {
//We create the actor container and a child upon initialization
csystem = ActorSystem.create("LocalNode", ConfigFactory.load().getConfig("localNode"));
localActor = csystem.actorOf(new Props(LocalActor.class),"localActor");
}
public void connectMaster (final String classname)
{
localActor.tell(classname);
}
public void connectMaster ()
{
}
public void connectMaster (final WebSocket.In<JsonNode> in, final WebSocket.Out<JsonNode> out )
{
in.onMessage(new Callback<JsonNode>() {
public void invoke(JsonNode event) throws JsonParseException, JsonMappingException, IOException {
ObjectMapper mapper = new ObjectMapper();
#SuppressWarnings("unchecked")
Map<String,ArrayList<Object>> jsonMap = mapper.readValue(event, Map.class);
GesturePoints gp = new GesturePoints();
gp.setPoints(jsonMap);
localActor.tell(gp);
}
}); }
}
this is the code for the akka actor in the play framework
package controllers;
import Com.RubineEngine.GesturePoints.*;
import akka.actor.ActorRef;
import akka.actor.UntypedActor;
import akka.event.Logging;
import akka.event.LoggingAdapter;
public class LocalActor extends UntypedActor {
/**
*
*/
ActorRef masterActor; // = getContext().actorFor("akka://MasterNode#127.0.0.1:2552/user/masterActor");
LoggingAdapter log = Logging.getLogger(getContext().system(), this);
#Override
public void onReceive(Object arg) throws Exception {
System.out.println(" Local Actor 1");
if(arg instanceof GesturePoints)
{ System.out.println(" local Actor 2");
masterActor.tell(" Welcome home " , getSelf());
System.out.println(" Local Actor 3");}
else
{unhandled(arg);}
}
public void preStart()
{
masterActor = getContext().actorFor("akka://MasterNode#127.0.0.1:2553/user/masterActor");
}
}
this is the code for the Master node application.conf
masterNode {
akka {
actor {
provider = "akka.remote.RemoteActorRefProvider"
}
remote {
transport = "akka.remote.netty.NettyRemoteTransport"
netty {
hostname = "127.0.0.1"
remote.netty.port = 2553
}
}
}
}
this is the code for the master node
package Rubine_Cluster;
import java.util.Arrays;
import com.typesafe.config.ConfigFactory;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import akka.actor.Props;
import akka.kernel.Bootable;
/**
* Hello world!
*
*/
public class MasterNode implements Bootable
{
final ActorSystem system;
ActorRef masterActor;
public MasterNode() {
//Create a child actor of this actor upon initialization
system = ActorSystem.create("MasterNode", ConfigFactory.load()
.getConfig("masterNode"));
masterActor = system.actorOf(new Props(MasterActor.class),"masterActor");
}
public void startup() {
}
public void shutdown() {
system.shutdown();
}
}
this is the code for the master akka actor
package Rubine_Cluster;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Map;
import Com.RubineEngine.GesturePoints.*;
import akka.actor.*;
import akka.serialization.Serialization;
import akka.serialization.SerializationExtension;
import akka.serialization.Serializer;
public class MasterActor extends UntypedActor {
/**
*
*/
ActorRef worker1;
#Override
public void onReceive(Object message) throws Exception {
System.out.println(" Master Actor 5");
System.out.println(message);
if(message instanceof GesturePoints)
{ //GesturePoints gp = (GesturePoints) message;
System.out.println(" Master Actor 1");
try { worker1.tell(message, getSelf());
System.out.println(" Master Actor 2");
} catch (Exception e) {
getSender().tell(new akka.actor.Status.Failure(e), getSelf());
throw e;
}
}
else{ unhandled(message);}
}
public void preStart()
{
worker1 = getContext().actorFor("akka://WorkerNode#127.0.0.1:2552/user/workerActor");
}
}
this is the code for the worker node application.conf
workerNode {
akka {
actor {
provider = "akka.remote.RemoteActorRefProvider"
}
remote {
transport = "akka.remote.netty.NettyRemoteTransport"
netty {
hostname = "127.0.0.1"
remote.netty.port = 2552
}
}
}
}
this is the code for the worker nide
package com.theta.gesture;
import com.typesafe.config.ConfigFactory;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import akka.actor.Props;
import akka.kernel.Bootable;
public class WorkerNode implements Bootable{
ActorSystem system;
ActorRef worker;
WorkerNode(){
system = ActorSystem.create("WorkerNode", ConfigFactory.load()
.getConfig("workerNode"));
ActorRef workerActor = system.actorOf(new Props(WorkerActor.class),"workerActor");
}
public void shutdown() {
system.shutdown();
}
public void startup() {
}
}
this is the code for the actor in the worker project
package com.theta.gesture;
import java.util.ArrayList;
import java.util.Map;
import Com.RubineEngine.GesturePoints.GesturePoints;
import akka.actor.*;
public class WorkerActor extends UntypedActor {
private static double DIST_SQ_THRESHOLD = 3 * 3; /* threshold to eliminate mouse jitter */
#Override
public void onReceive(Object msg) throws Exception {
if(msg instanceof GesturePoints)
{ GesturePoints message = (GesturePoints) msg;
initial_Theta(message);}
else {unhandled(msg);}
}
public void initial_Theta(GesturePoints p)
{ System.out.println(" Worker Actor 1");
if(p.getPoints().get("X").size() < 3) //The number of x coordinates as size
{ return;}
System.out.println(" Worker Actor 2");
double magsq,dx,dy, recip;
dx = (double) ((Integer)p.getPoints().get("x").get(2) - (Integer)p.getPoints().get("x").get(0)) ;
dy = ((Double)p.getPoints().get("y").get(2)) - ((Double)p.getPoints().get("y").get(0));
magsq = dx * dx + dy * dy;
if(magsq > DIST_SQ_THRESHOLD)
{
recip = 1/Math.sqrt(magsq);
double initial_cos = dx * recip;
System.out.println(" Worker Actor 3");
double initial_sin = dy * recip;
System.out.println("Feature cos " + initial_cos);
System.out.println("Gesture sin " + initial_sin);
}
} }
this is the console information on the Worker Node
[INFO] [10/08/2012 12:12:44.486] [main] [ActorSystem(WorkerNode)] REMOTE:
RemoteServerStarted#akka://WorkerNode#127.0.0.1:2552
this is the console information on the master nODE
[INFO] [10/08/2012 12:13:34.633] [main] [ActorSystem(MasterNode)] REMOTE:
RemoteServerStarted#akka://MasterNode#127.0.0.1:2552
any idea about the possible course of this situation and a suggested solution is dearly sought after

Your "remote.netty.port = X" inside the nested sections should be "port = X"