I have a system with HTTP POST requests and it runs with Spring 5 (standalone tomcat). In short it looks like this:
client (Apache AB) ----> micro service (java or golang) --> RabbitMQ --> Core(spring + tomcat).
The thing is, when I use my Java (Spring) service, it is ok. AB shows this output:
ab -n 1000 -k -s 2 -c 10 -s 60 -p test2.sh -A 113:113 -T 'application/json' https://127.0.0.1:8449/SecureChat/chat/v1/rest-message/send
This is ApacheBench, Version 2.3 <$Revision: 1807734 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 127.0.0.1 (be patient)
Completed 100 requests
...
Completed 1000 requests
Finished 1000 requests
Server Software:
Server Hostname: 127.0.0.1
Server Port: 8449
SSL/TLS Protocol: TLSv1.2,ECDHE-RSA-AES256-GCM-SHA384,2048,256
Document Path: /rest-message/send
Document Length: 39 bytes
Concurrency Level: 10
Time taken for tests: 434.853 seconds
Complete requests: 1000
Failed requests: 0
Keep-Alive requests: 0
Total transferred: 498000 bytes
Total body sent: 393000
HTML transferred: 39000 bytes
Requests per second: 2.30 [#/sec] (mean)
Time per request: 4348.528 [ms] (mean)
Time per request: 434.853 [ms] (mean, across all concurrent
requests)
Transfer rate: 1.12 [Kbytes/sec] received
0.88 kb/s sent
2.00 kb/s total
Connection Times (ms)
min mean[+/-sd] median max
Connect: 4 14 7.6 17 53
Processing: 1110 4317 437.2 4285 8383
Waiting: 1107 4314 437.2 4282 8377
Total: 1126 4332 436.8 4300 8403
That is through TLS.
But when I try to use my Golang service I get timeout:
Benchmarking 127.0.0.1 (be patient)...apr_pollset_poll: The timeout specified has expired (70007)
Total of 92 requests completed
And this output:
ab -n 100 -k -s 2 -c 10 -s 60 -p test2.sh -T 'application/json' http://127.0.0.1:8089/
This is ApacheBench, Version 2.3 <$Revision: 1807734 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 127.0.0.1 (be patient)...^C
Server Software:
Server Hostname: 127.0.0.1
Server Port: 8089
Document Path: /
Document Length: 39 bytes
Concurrency Level: 10
Time taken for tests: 145.734 seconds
Complete requests: 92
Failed requests: 1
(Connect: 0, Receive: 0, Length: 1, Exceptions: 0)
Keep-Alive requests: 91
Total transferred: 16380 bytes
Total body sent: 32200
HTML transferred: 3549 bytes
Requests per second: 0.63 [#/sec] (mean)
Time per request: 15840.663 [ms] (mean)
Time per request: 1584.066 [ms] (mean, across all concurrent requests)
Transfer rate: 0.11 [Kbytes/sec] received
0.22 kb/s sent
0.33 kb/s total
Connection Times (ms)
min mean[+/-sd] median max
Connect: 0 0 0.0 0 0
Processing: 1229 1494 1955.9 1262 20000
Waiting: 1229 1291 143.8 1262 2212
Total: 1229 1494 1955.9 1262 20000
That is through plane tcp.
I guess I have some mistakes in my code. I made it in one file
func initAmqp(rabbitUrl string) {
var err error
conn, err = amqp.Dial(rabbitUrl)
failOnError(err, "Failed to connect to RabbitMQ")
}
func main() {
err := gcfg.ReadFileInto(&cfg, "config.gcfg")
if err != nil {
log.Fatal(err);
}
PrintConfig(cfg)
if cfg.Section_rabbit.RabbitUrl != "" {
initAmqp(cfg.Section_rabbit.RabbitUrl);
}
mux := http.NewServeMux();
mux.Handle("/", NewLimitHandler(1000, newTestHandler()))
server := http.Server {
Addr: cfg.Section_basic.Port,
Handler: mux,
ReadTimeout: 20 * time.Second,
WriteTimeout: 20 * time.Second,
}
defer conn.Close();
log.Println(server.ListenAndServe());
}
func NewLimitHandler(maxConns int, handler http.Handler) http.Handler {
h := &limitHandler{
connc: make(chan struct{}, maxConns),
handler: handler,
}
for i := 0; i < maxConns; i++ {
h.connc <- struct{}{}
}
return h
}
func newTestHandler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
handler(w, r);
})
}
func handler(w http.ResponseWriter, r *http.Request) {
if b, err := ioutil.ReadAll(r.Body); err == nil {
fmt.Println("message is ", string(b));
res := publishMessages(string(b))
w.Write([]byte(res))
w.WriteHeader(http.StatusOK)
counter ++;
}else {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("500 - Something bad happened!"))
}
}
func publishMessages(payload string) string {
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
q, err = ch.QueueDeclare(
"", // name
false, // durable
false, // delete when unused
true, // exclusive
false, // noWait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
corrId := randomString(32)
log.Println("corrId ", corrId)
err = ch.Publish(
"", // exchange
cfg.Section_rabbit.RabbitQeue, // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
DeliveryMode: amqp.Transient,
ContentType: "application/json",
CorrelationId: corrId,
Body: []byte(payload),
Timestamp: time.Now(),
ReplyTo: q.Name,
})
failOnError(err, "Failed to Publish on RabbitMQ")
defer ch.Close();
result := "";
for d := range msgs {
if corrId == d.CorrelationId {
failOnError(err, "Failed to convert body to integer")
log.Println("result = ", string(d.Body))
return string(d.Body);
}else {
log.Println("waiting for result = ")
}
}
return result;
}
Can someone help?
EDIT
here are my variables
type limitHandler struct {
connc chan struct{}
handler http.Handler
}
var conn *amqp.Connection
var q amqp.Queue
EDIT 2
func (h *limitHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
select {
case <-h.connc:
fmt.Println("ServeHTTP");
h.handler.ServeHTTP(w, req)
h.connc <- struct{}{}
default:
http.Error(w, "503 too busy", http.StatusServiceUnavailable)
}
}
EDIT 3
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
Related
I can see in the logs that Spring retry is sending 2 requests to the remote server and both requests return successful responses.
I am not able to get the reason behind the same.
Code:
Class StatusClient{
#CircuitBreaker(maxAttemptsExpression = "#{${remote.broadridge.circuitBreaker.maxAttempts}}",
openTimeoutExpression = "#{${remote.broadridge.circuitBreaker.openTimeout}}", resetTimeoutExpression = "#{${remote.broadridge.circuitBreaker.resetTimeout}}")
public Optional<JobStatusResponseDTO> getStatus(String account, String jobNumber) {
client.post()
.uri(PATH)
.body(BodyInserters.fromValue(request))
.exchangeToMono(response -> {
if (response.statusCode() == HttpStatus.NO_CONTENT) {
return Mono.empty();
} else if (isClientOrServerError(response)) {
return Mono.error(new RemoteClientException(String.format("status is not received: %s", response.statusCode())));
}
stopWatch.stop();
log.info("time taken by the getStatus=[{}] for {}", (stopWatch.getTotalTimeMillis()), request);
return response.bodyToMono(JobStatusResponseDTO.class);
})
.block();
return Optional.ofNullable(block);}
}
Class status{
#Retryable(maxAttemptsExpression = "#{${remote.retry.maxAttempts}}", backoff = #Backoff(delayExpression = "#{${remote.retry.delay}}"))
public Optional<JobStatusResponseDTO> getStatus(String jobNumber, String accountNumber) {
return statusClient.getStatus(accountNumber, jobNumber);
}
}
Config in application.yml
circuitBreaker:
maxAttempts: 3 # defalut 3
openTimeout: 5000 # defalut 5000
resetTimeout: 20000 # defalut 20000
retry:
maxAttempts: 3 # defalut 3
delay: 1000 # defalut 1000
Logs:
792 <14>1 2021-10-26T16:26:32.978917+00:00 - 2021-10-26 16:26:32.978 INFO [batch,ec40b8fe1f6a4cfb,06052e092b3f8e66] : time taken by the getStatus=[582] for JobStatusRequestDTO(account=12
456, jobNumber=S123456)
792 <14>1 2021-10-26T16:26:18.263121+00:00 2021-10-26 16:26:18.262 INFO [batch,ec40b8fe1f6a4cfb,21202725a0002bde] : time taken by the getStatus=[592] for JobStatusRequestDTO(account=12
456, jobNumber=S123456)
Both the request are a few seconds apart.
Edit 1:
changed circuit breaker to the max attempt to 1. Now it is retrying 3 times. There is still an issue. It seems it is calling the remote server only once and not calling after.
The remote call is wrapped in a circuit breaker.
1st Attempt log:
status is not received: 503 SERVICE_UNAVAILABLE
2nd Attempt log:
org.springframework.retry.ExhaustedRetryException: Retry exhausted after last attempt with no recovery path;
3rd Attempt log:
org.springframework.retry.ExhaustedRetryException: Retry exhausted after last attempt with no recovery path;
circuitBreaker:
maxAttempts: 1
openTimeout: 5000 # defalut 5000
resetTimeout: 20000 # defalut 20000
retry:
maxAttempts: 3 # defalut 3
delay: 1000 # defalut 1000
This is because you have placed the default retry.maxAttempts to 3 with a delay of 1000ms. Spring will auto-retry if there is no response within the mentioned delay time. So, replace the retry.maxAttemps to 2 then it won't give multiple responses.
you can simply paste below lines in application.properties.
retry.maxAttempts=2
retry.maxDelay=100
Also, I suggest you go through this.
I am running into a weird problem when reading CAC card, in a multi terminal setup, from Nth (Except 1st) terminal.
Environment - JDK8_u211, Destop without any card reader, Internal card reader for laptop, USB card reader, CAC\PIV card
USE-CASE Tried
When the CAC card is plugged into internal slot of Laptop, everything works fine with and withou USB card terminal connected to laptop.
When the CAC card is plugged into the USB reader and connected to the Desktop, everything works fine.
When the CAC card is plugged into the USB reader and connected to the Laptop, the PKCS store instantiation fails, as the config uses slot=1
When the CAC card is plugged into the USB reader and connected to the Laptop, if I hard code slot=4 in the config, everything works fine.
Problem: Unable to determine the correct slot to use within the config file. Somehow number of terminals is not adding up to determine the slot at runtime.
By enabling JDK debug logs, i could verify that internally "C_GetSlotList" returns 0,4 where as I have only 2 terminals, hence using index 0 or 1.
Does anyone know how to determine the correlation between terminal vs slot or is there something really wrong in the way i am trying to read the card ?
Please advise.
Sample code
public class MultipleTerminals {
private static String DLL_PATH = "C:\\opensc-pkcs11.dll";
public static void main(String[] args) throws Exception {
MyTerminal custTerminalObj = getSmartCardTerminal();
CardTerminal terminal = custTerminalObj.getTerminal();
Integer terminalIdx = custTerminalObj.getIndex();
System.out.println("Using Terminal.Name: " + terminal.getName());
System.out.println("Using Terminal.isCardPresent: " + ((CardTerminal)terminal).isCardPresent());
if (terminal.isCardPresent()) { // duplicate check
registerProvider(4);
KeyStore keyStore = createKeyStore();
printCertificate(keyStore);
}
}
public static MyTerminal getSmartCardTerminal() {
System.out.println("---------------------------------------------------");
MyTerminal terminal = null;
int terminalIdx = -1;
try {
TerminalFactory factory = TerminalFactory.getDefault();
if (factory != null) {
List<CardTerminal> allTerminals = factory.terminals().list();
for(CardTerminal term : allTerminals) {
System.out.println("Terminal Name: " + term.getName());
System.out.println("isCardPresent: " + term.isCardPresent());
System.out.println("----------");
}
for(CardTerminal term : allTerminals) {
terminalIdx++;
if(term.isCardPresent()) {
terminal = new MyTerminal();
terminal.setIndex(terminalIdx);
terminal.setTerminal(term);
System.out.println("Using Terminal Idx: " + terminalIdx);
}
}
}
} catch (CardException e1) {
terminal = null;
System.out.println("SmartCardHelper.getSmartCardTerminal --> "
+ "Exception while accessing smart-card terminal: "
+ e1.getMessage());
e1.printStackTrace();
}
return terminal;
}
public static void registerProvider(Integer idx) {
System.out.println("---------------------------------------------------");
String ext = "attributes(*,*,*)=\n{\nCKA_TOKEN=true\nCKA_LOCAL=true\n}";
String OPENSC_DEFAULT_PROVIDER_NAME = "PKCS#11";
String OPENSC_PKCS11_DLL = DLL_PATH;
String configString = "name = "
+ OPENSC_DEFAULT_PROVIDER_NAME.replace(' ', '_') + "\n"
+ "library = "
+ OPENSC_PKCS11_DLL + "\n slot = " + idx + " "
+ "\n attributes = compatibility \n" + ext;
System.out.println("\tConfigString: " + configString);
ByteArrayInputStream is = new ByteArrayInputStream(
configString.getBytes(Charset.defaultCharset()));
Provider cardSecurityProvider = new sun.security.pkcs11.SunPKCS11(
is);
displayProviders();
Security.addProvider(cardSecurityProvider);
displayProviders();
System.out.println("==================");
}
public static KeyStore createKeyStore() throws KeyStoreException {
KeyStore.CallbackHandlerProtection callbackHandler = new KeyStore.CallbackHandlerProtection(
new PIVCardPinHandler());
KeyStore.Builder builder = KeyStore.Builder.newInstance("PKCS11", null,
callbackHandler);
return builder.getKeyStore();
}
public static void displayProviders() {
System.out.println("---------------------------------------------------");
Provider[] objs = Security.getProviders();
System.out.println("Provider Count: " + objs.length);
String type = "KeyStore";
String algorithm="PKCS11";
ProviderList list = Providers.getProviderList();
Provider provider = list.getProvider("SunPKCS11-PKCS");
System.out.println("Provider[SunPKCS11-PKCS]: " + provider);
if(provider != null) {
Set<Service> svcs = provider.getServices();
System.out.println("Service count: " + svcs.size());
}
Service firstService = list.getService(type, algorithm);
System.out.println("Type[KeyStore], Algo[PKCS11], Service: " + firstService);
}
public static void printCertificate(KeyStore keyStore)
throws KeyStoreException {
for (String alias : Collections.list(keyStore.aliases())) {
System.out.println("Alias: " + alias);
}
}
}
class MyTerminal{
private Integer index;
private CardTerminal terminal;
public MyTerminal() {
}
public Integer getIndex() {
return index;
}
public void setIndex(Integer index) {
this.index = index;
}
public CardTerminal getTerminal() {
return terminal;
}
public void setTerminal(CardTerminal terminal) {
this.terminal = terminal;
}
}
* Output TestCase-1 (WORKING)*
---------------------------------------------------
Terminal Name: Alcor Micro USB Smart Card Reader 0
isCardPresent: true
----------
Terminal Name: Identive SCR33xx v2.0 USB SC Reader 0
isCardPresent: false
----------
Using Terminal Idx: 0
Using Terminal.Name: Alcor Micro USB Smart Card Reader 0
Using Terminal.isCardPresent: true
---------------------------------------------------
ConfigString:
name = PKCS#11
library = C:\opensc-pkcs11.dll
slot = 0
attributes = compatibility
attributes(*,*,*)=
{
CKA_TOKEN=true
CKA_LOCAL=true
}
SunPKCS11 loading ---DummyConfig-1---
sunpkcs11: Initializing PKCS#11 library C:\opensc-pkcs11.dll
Information for provider SunPKCS11-PKCS
Library info:
cryptokiVersion: 2.20
manufacturerID: DUMMY_VALUE
flags: 0
libraryDescription: DUMMY_VALUE
libraryVersion: 0.19
All slots: 0, 4
Slots with tokens: 0, 4
Slot info for slot 0:
slotDescription: Alcor Micro USB Smart Card Reader 0
manufacturerID: DUMMY_VALUE
flags: CKF_TOKEN_PRESENT | CKF_REMOVABLE_DEVICE | CKF_HW_SLOT
hardwareVersion: 0.00
firmwareVersion: 0.00
Token info for token in slot 0:
label: DUMMY_VALUE
manufacturerID: DUMMY_VALUE
model: PKCS#15 emulated
serialNumber: DUMMY_VALUE
flags: CKF_RNG | CKF_LOGIN_REQUIRED | CKF_USER_PIN_INITIALIZED | CKF_TOKEN_INITIALIZED
ulMaxSessionCount: CK_EFFECTIVELY_INFINITE
ulSessionCount: 0
ulMaxRwSessionCount: CK_EFFECTIVELY_INFINITE
ulRwSessionCount: 0
ulMaxPinLen: 8
ulMinPinLen: 4
ulTotalPublicMemory: CK_UNAVAILABLE_INFORMATION
ulFreePublicMemory: CK_UNAVAILABLE_INFORMATION
ulTotalPrivateMemory: CK_UNAVAILABLE_INFORMATION
ulFreePrivateMemory: CK_UNAVAILABLE_INFORMATION
hardwareVersion: 0.00
firmwareVersion: 0.00
utcTime:
---------------------------------------------------
Provider Count: 10
Provider[SunPKCS11-PKCS]: null
Type[KeyStore], Algo[PKCS11], Service: null
---------------------------------------------------
Provider Count: 11
Provider[SunPKCS11-PKCS]: SunPKCS11-PKCS version 1.8
Service count: 26
Type[KeyStore], Algo[PKCS11], Service: SunPKCS11-PKCS: KeyStore.PKCS11 -> sun.security.pkcs11.P11KeyStore
aliases: [PKCS11-PKCS]
(KeyStore)
==================
sunpkcs11: caller passed NULL pin
* Output TestCase-3 (NOT WORKING)*
---------------------------------------------------
Terminal Name: Alcor Micro USB Smart Card Reader 0
isCardPresent: false
----------
Terminal Name: Identive SCR33xx v2.0 USB SC Reader 0
isCardPresent: true
----------
Using Terminal Idx: 1
Using Terminal.Name: Identive SCR33xx v2.0 USB SC Reader 0
Using Terminal.isCardPresent: true
---------------------------------------------------
ConfigString:
name = PKCS#11
library = C:\opensc-pkcs11.dll
slot = 1
attributes = compatibility
attributes(*,*,*)=
{
CKA_TOKEN=true
CKA_LOCAL=true
}
SunPKCS11 loading ---DummyConfig-1---
sunpkcs11: Initializing PKCS#11 library C:\opensc-pkcs11.dll
Information for provider SunPKCS11-PKCS
Library info:
cryptokiVersion: 2.20
manufacturerID: DUMMY_VALUE
flags: 0
libraryDescription: DUMMY_VALUE
libraryVersion: 0.19
All slots: 0, 4
Slots with tokens: 0, 4
---------------------------------------------------
Provider Count: 10
Provider[SunPKCS11-PKCS]: null
Type[KeyStore], Algo[PKCS11], Service: null
---------------------------------------------------
Provider Count: 11
Provider[SunPKCS11-PKCS]: SunPKCS11-PKCS version 1.8
Service count: 0
Type[KeyStore], Algo[PKCS11], Service: null
==================
Exception in thread "main" java.security.KeyStoreException: KeyStore instantiation failed
at java.security.KeyStore$Builder$2.getKeyStore(KeyStore.java:1967)
Caused by: java.security.KeyStoreException: PKCS11 not found
at java.security.KeyStore.getInstance(KeyStore.java:851)
at java.security.KeyStore$Builder$2$1.run(KeyStore.java:1923)
at java.security.KeyStore$Builder$2$1.run(KeyStore.java:1918)
at java.security.AccessController.doPrivileged(Native Method)
at java.security.KeyStore$Builder$2.getKeyStore(KeyStore.java:1964)
... 2 more
Caused by: java.security.NoSuchAlgorithmException: PKCS11 KeyStore not available
at sun.security.jca.GetInstance.getInstance(GetInstance.java:159)
at java.security.Security.getImpl(Security.java:695)
at java.security.KeyStore.getInstance(KeyStore.java:848)
... 6 more
* Output TestCase-4 (WORKING)*
---------------------------------------------------
Terminal Name: Alcor Micro USB Smart Card Reader 0
isCardPresent: false
----------
Terminal Name: Identive SCR33xx v2.0 USB SC Reader 0
isCardPresent: true
----------
Using Terminal Idx: 1
Using Terminal.Name: Identive SCR33xx v2.0 USB SC Reader 0
Using Terminal.isCardPresent: true
---------------------------------------------------
ConfigString:
name = PKCS#11
library = C:\opensc-pkcs11.dll
slot = 4
attributes = compatibility
attributes(*,*,*)=
{
CKA_TOKEN=true
CKA_LOCAL=true
}
SunPKCS11 loading ---DummyConfig-1---
sunpkcs11: Initializing PKCS#11 library C:\opensc-pkcs11.dll
Information for provider SunPKCS11-PKCS
Library info:
cryptokiVersion: DUMMY_VALUE
manufacturerID: DUMMY_VALUE
flags: 0
libraryDescription: DUMMY_VALUE
libraryVersion: DUMMY_VALUE
All slots: 0, 4
Slots with tokens: 0, 4
Slot info for slot 4:
slotDescription: DUMMY_VALUE
manufacturerID: DUMMY_VALUE
flags: CKF_TOKEN_PRESENT | CKF_REMOVABLE_DEVICE | CKF_HW_SLOT
hardwareVersion: 0.00
firmwareVersion: 0.00
Token info for token in slot 4:
label: DUMMY_LABEL
manufacturerID: DUMMY_VALUE
model: PKCS#15 emulated
serialNumber: DUMMY_VALUE
flags: CKF_RNG | CKF_LOGIN_REQUIRED | CKF_USER_PIN_INITIALIZED | CKF_TOKEN_INITIALIZED
ulMaxSessionCount: CK_EFFECTIVELY_INFINITE
ulSessionCount: 0
ulMaxRwSessionCount: CK_EFFECTIVELY_INFINITE
ulRwSessionCount: 0
ulMaxPinLen: 8
ulMinPinLen: 4
ulTotalPublicMemory: CK_UNAVAILABLE_INFORMATION
ulFreePublicMemory: CK_UNAVAILABLE_INFORMATION
ulTotalPrivateMemory: CK_UNAVAILABLE_INFORMATION
ulFreePrivateMemory: CK_UNAVAILABLE_INFORMATION
hardwareVersion: 0.00
firmwareVersion: 0.00
utcTime:
---------------------------------------------------
Provider Count: 10
Provider[SunPKCS11-PKCS]: null
Type[KeyStore], Algo[PKCS11], Service: null
---------------------------------------------------
Provider Count: 11
Provider[SunPKCS11-PKCS]: SunPKCS11-PKCS version 1.8
Service count: 26
Type[KeyStore], Algo[PKCS11], Service: SunPKCS11-PKCS: KeyStore.PKCS11 -> sun.security.pkcs11.P11KeyStore
aliases: [PKCS11-PKCS]
(KeyStore)
==================
sunpkcs11: caller passed NULL pin
You are getting confused between slot id and slot index. They are two different attributes.
The C_GetSlotList gives you an array of slot id's not slot indexes. In your case 0 and 4 are slot id's, not slot indexes. The slot index is the index of the slot id in the array.
In your getSmartCardTerminal() method where you are identifying the terminalIdx you are identifying the index, but in your registerProvider(Integer idx) method you are mapping the index as the id, here:
OPENSC_PKCS11_DLL + "\n slot = " + idx + " "
When you are using the slot index you should be using slotListIndex and when you are using the slot id, you should be using slot.
So to fix your problem change it to:
OPENSC_PKCS11_DLL + "\n slotListIndex = " + idx + " "
Documentation here.
Assume original message size is 500 bytes (before sending it to the Kafka). So what will be the size of the message after sending it to the Kafka? And what if we use any compression?
Additional information: I am putting a ByteBuffer of size 2048 bytes to a topic (with single partition) without any key.
Topic name: ub3
Path: /data/kafka-logs/ub3-0
[hdpusr#hdpdev2 ub3-0]$ $KAFKA_HOME/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list hdpdev2:8092 --topic ub3 --time -1 --offsets 1 | awk -F ":" '{sum += $3} END {print sum}'
184
[hdpusr#hdpdev2 ub3-0]$ du -sh *
10M 00000000000000000000.index
448K 00000000000000000000.log
10M 00000000000000000000.timeindex
4.0K leader-epoch-checkpoint
[hdpusr#hdpdev2 ub3-0]$
[hdpusr#hdpdev2 ub3-0]$
[hdpusr#hdpdev2 ub3-0]$ $KAFKA_HOME/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list hdpdev2:8092 --topic ub3 --time -1 --offsets 1 | awk -F ":" '{sum += $3} END {print sum}'
86284
[hdpusr#hdpdev2 ub3-0]$ du -sh *
10M 00000000000000000000.index
256M 00000000000000000000.log
10M 00000000000000000000.timeindex
4.0K leader-epoch-checkpoint
[hdpusr#hdpdev2 ub3-0]$
[hdpusr#hdpdev2 ub3-0]$ $KAFKA_HOME/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list hdpdev2:8092 --topic ub3 --time -1 --offsets 1 | awk -F ":" '{sum += $3} END {print sum}'
172405
[hdpusr#hdpdev2 ub3-0]$ du -sh *
10M 00000000000000000000.index
512M 00000000000000000000.log
10M 00000000000000000000.timeindex
4.0K leader-epoch-checkpoint
[hdpusr#hdpdev2 ub3-0]$
[hdpusr#hdpdev2 ub3-0]$ $KAFKA_HOME/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list hdpdev2:8092 --topic ub3 --time -1 --offsets 1 | awk -F ":" '{sum += $3} END {print sum}'
258491
[hdpusr#hdpdev2 ub3-0]$ du -sh *
10M 00000000000000000000.index
596M 00000000000000000000.log
10M 00000000000000000000.timeindex
4.0K leader-epoch-checkpoint
[hdpusr#hdpdev2 ub3-0]$
[hdpusr#hdpdev2 ub3-0]$ $KAFKA_HOME/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list hdpdev2:8092 --topic ub3 --time -1 --offsets 1 | awk -F ":" '{sum += $3} END {print sum}'
344563
[hdpusr#hdpdev2 ub3-0]$ du -sh *
10M 00000000000000000000.index
1.1G 00000000000000000000.log
10M 00000000000000000000.timeindex
4.0K leader-epoch-checkpoint
[hdpusr#hdpdev2 ub3-0]$
The short answer is: who knows?
But let's try to find out some numbers. I have started a Kafka in Docker using this guide. Then, wrote a simple producer:
public class App {
public static void main(String[] args) throws Exception {
final Producer<String, byte[]> producer = producer();
producer.send(
new ProducerRecord<>(
"test",
key(),
value()
)
).get();
}
private static Producer<String, byte[]> producer() {
final Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(ProducerConfig.CLIENT_ID_CONFIG, "so57472830");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
return new KafkaProducer<>(props);
}
private static String key() {
return UUID.randomUUID().toString();
}
}
So, will be sending to localhost:9092 with a client id equal to so57472830 into a test topic. The payloads are byte arrays and keys are string UUIDs. As you'll see later all these values (except the host:port) contribute to the "overhead". Here I suppose that overhead is everything except the message payload itself.
Let's start with a "Hello, world!":
private static byte[] value() {
return "Hello, world!".getBytes();
}
Run the app and capture the traffic to localhost:9092. I used WireShark for that.
Here I found the message with the payload. Let's see the whole TCP stream ("Follow TCP stream" in WireShark):
So, the whole stream took 527 bytes of which the client send (highlighted with rose color) 195:
(This also means that Kafka send 527 - 195 == 332 bytes in response):
Our payload was 13 bytes. As you noticed, the outbound traffic contains the client id twice (2 × 10 bytes) and the message key (16 bytes). So, of 195 bytes send 146 are mystery (probably the one that you named as "overhead" in your question).
Let's send 500 random bytes:
private static byte[] value() {
final byte[] result = new byte[500];
new Random().nextBytes(result);
return result;
}
Outbound traffic was 684 bytes (the entire conversation took 1016):
Again, the server send 332 byte in response and the outbound mystery (overhead) made up 684 - (500 + 2 × 10 + 16) = 164 bytes!
All these numbers are not final and may change with producer versions or specific config settings. One of them, you've mentioned, is compression. Let's check it out. Be warned that the compression depends on the data. Random bytes are tougher to compress than the constant ones as they have more entropy. So, let's send 500 repetiting bytes with a GZIP compression. Without the compression, the numbers are the same:
Add props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); to the producer() method and change the value():
private static byte[] value() {
final byte[] result = new byte[500];
Arrays.fill(result, (byte) 'a');
return result;
}
When the compression is enabled, the message (key and value, not the client id and topic) are compressed, and the outbound traffic is only 208 bytes:
I'd say that the overhead is about the same as in the examples above, the compression impacts the size of the message itself.
That all applies to the traffic, but after your edit I see you were interested in the storage size. Nevertheless, I would say that the answer is the same: "who knows". The numbers definitely depend on your configuration.
I used the NSQ as the mq in my project，java producer produce the message to NSQ and go consumer consume it.But the strange things is that the consumer always get the message after few seconds.There is just a few messages,I really don't know how to explain why it's happened.
Here is the test result，please pay attention to the time.They both consume the same topic.You can see in the second time,Go is slower than java for 7s.
Java result:
INFO | jvm 1 | 2018/07/11 17:22:01 | Msg
receive：{"did":"XSQ000200000005","msg":{"id":"5560","type":1,"content":"ZBINh6CBsLw7k2xjr1wslSjY+5QavEgYU6AzzLZn0lOgON9ZYHnNP4UJVUGB+/SpsxZQnrWR9PlULzpSP/p9l9t8wiAwj8qhznRaT8jeyx1/EUrDE0oXJB8GxWaLJUICCbC92j4BMA2HU8vgcfDOp9nSy1KFafi9zgFiCf9Igqo="}}
INFO | jvm 1 | 2018/07/11 17:22:11 | Msg
receive：{"did":"XSQ000200000005","msg":{"id":"5560","type":1,"content":"ZBINh6CBsLw7k2xjr1wslSjY+5QavEgYU6AzzLZn0lOgON9ZYHnNP4UJVUGB+/SpsxZQnrWR9PlULzpSP/p9l9t8wiAwj8qhznRaT8jeyx1/EUrDE0oXJB8GxWaLJUICCbC92j4BMA2HU8vgcfDOp9nSy1KFafi9zgFiCf9Igqo="}}
INFO | jvm 1 | 2018/07/11 17:23:21 | Msg
receive：{"did":"XSQ000200000005","msg":{"id":"5560","type":1,"content":"ZBINh6CBsLw7k2xjr1wslSjY+5QavEgYU6AzzLZn0lOgON9ZYHnNP4UJVUGB+/SpsxZQnrWR9PlULzpSP/p9l9t8wiAwj8qhznRaT8jeyx1/EUrDE0oXJB8GxWaLJUICCbC92j4BMA2HU8vgcfDOp9nSy1KFafi9zgFiCf9Igqo="}}
INFO | jvm 1 | 2018/07/11 17:25:31 | Msg
receive：{"did":"XSQ000200000005","msg":{"id":"5560","type":1,"content":"ZBINh6CBsLw7k2xjr1wslSjY+5QavEgYU6AzzLZn0lOgON9ZYHnNP4UJVUGB+/SpsxZQnrWR9PlULzpSP/p9l9t8wiAwj8qhznRaT8jeyx1/EUrDE0oXJB8GxWaLJUICCbC92j4BMA2HU8vgcfDOp9nSy1KFafi9zgFiCf9Igqo="}}
Go result:
2018-07-11 17:22:03 broker.go DEBUG Ready to send msg 5560 with type 1 to XSQ000200000005
2018-07-11 17:22:28 broker.go DEBUG Ready to send msg 5560 with type 1 to XSQ000200000005
2018-07-11 17:23:21 broker.go DEBUG Ready to send msg 5560 with type 1 to XSQ000200000005
2018-07-11 17:25:38 broker.go DEBUG Ready to send msg 5560 with type 1 to XSQ000200000005
please ignore the other errors,just because of the business.
Here is my go consumer:
func (b *Broker) createConsumer(topic string, vendor int32) error {
config := nsq.NewConfig()
laddr := "127.0.0.1"
// so that the test can simulate binding consumer to specified address
config.LocalAddr, _ = net.ResolveTCPAddr("tcp", laddr+":0")
// so that the test can simulate reaching max requeues and a call to LogFailedMessage
config.DefaultRequeueDelay = 0
// so that the test wont timeout from backing off
config.MaxBackoffDuration = time.Millisecond * 50
c, err := nsq.NewConsumer(topic, "channel_box_" + util.String(vendor), config)
if err != nil {
return log.Error("Failed to new nsq consumers.")
}
c.AddConcurrentHandlers(nsq.HandlerFunc(func(message *nsq.Message) error {
if err := b.handle(message, vendor); err != nil {
log.Errorf("Handle message %v for vendor %d from mq failed.", message.ID, vendor)
}
return nil
}), 5)
if err = c.ConnectToNSQLookupds(b.Opts.Nsq.Lookup); err != nil {
return log.Error("Failed to connect to nsq lookup server.")
}
b.consumers = append(b.consumers, c)
return nil
}
I'm trying to execute a program in eclipse and I when I click run I see this in the console output:
[] is an unknown syslog facility. Defaulting to [USER].
/
"Failed"
Any ideas?
It looks like that error is coming from org.apache.log4j.net.SyslogAppender, and that you've tried to set a bad facility name. Go take a look at your appenders and how you are setting them up.
414 public
415 void setFacility(String facilityName) {
416 if(facilityName == null)
417 return;
418
419 syslogFacility = getFacility(facilityName);
420 if (syslogFacility == -1) {
421 System.err.println("["+facilityName +
422 "] is an unknown syslog facility. Defaulting to [USER].");
423 syslogFacility = LOG_USER;
424 }
425
426 this.initSyslogFacilityStr();
427
428 // If there is already a sqw, make it use the new facility.
429 if(sqw != null) {
430 sqw.setSyslogFacility(this.syslogFacility);
431 }
432 }