We Keep Coding

Mockito 4.6.1 does not work with springboot

When I run Junit tests from Eclipse it all passes, however when I run the test from a gradle task which is literally just a test task it fails with the following error message:
java.lang.IllegalStateException at PluginLoader.java:88
Caused by: java.lang.IllegalStateException at DefaultMockitoPlugins.java:85
Caused by: java.lang.reflect.InvocationTargetException at NativeConstructorAccessorImpl.java:-2
Caused by: org.mockito.exceptions.base.MockitoInitializationException at Reporter.java:1131
Caused by: java.lang.NoClassDefFoundError at SubclassInjectionLoader.java:34
Caused by: java.lang.ClassNotFoundException at BuiltinClassLoader.java:581
Here are my gradle plugins and dependencies:
plugins {
id 'java-library'
id 'project-report'
id 'org.springframework.boot' version '2.6.8'
}
apply plugin: 'java'
apply plugin: 'org.springframework.boot'
apply plugin: 'io.spring.dependency-management'
apply plugin: 'maven-publish'
sourceCompatibility = 1.8
targetCompatibility = 1.8
dependencies {
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', {
exclude module: "spring-boot-starter-jetty"
}
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-actuator'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-thymeleaf'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-tomcat'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-cache'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-security'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-jdbc'
implementation group: 'org.springframework', name: 'spring-oxm', version: '5.3.20'
implementation group: 'org.eclipse.persistence', name: 'org.eclipse.persistence.moxy', version: '2.7.9'
implementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.1'
implementation group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
implementation group: 'org.liquibase', name: 'liquibase-core', version: '4.8.0'
implementation group: 'javax.cache', name: 'cache-api'
implementation group: 'org.ehcache', name: 'ehcache'
implementation group: 'org.apache.commons', name: 'commons-text', version: '1.9'
implementation group: 'net.iharder', name: 'base64', version: '2.3.9'
implementation group: 'org.freemarker', name: 'freemarker', version: '2.3.31'
//implementation group: 'com.nimbusds', name: 'nimbus-jose-jwt', version: '9.15.2'
//implementation group: 'org.apache.cxf', name: 'cxf-rt-rs-security-jose', version: '3.4.5'
implementation group: 'org.apache.httpcomponents', name: 'httpcore', version: '4.4.15'
implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.13', {
exclude module: "commons-codec:commons-codec"
}
implementation group: 'org.apache.poi', name: 'poi-ooxml', version: '5.2.2', {
exclude module: "commons-codec:commons-codec"
}
implementation group: 'commons-codec', name: 'commons-codec', version: '1.15'
implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2'
implementation group: 'commons-io', name: 'commons-io', version: '2.11.0'
implementation group: 'commons-cli', name: 'commons-cli', version: '1.5.0'
implementation group: 'xerces', name: 'xercesImpl', version: '2.12.2'
testImplementation group: 'com.h2database', name: 'h2'
testImplementation group: 'org.springframework.boot', name: 'spring-boot-starter-test', {
exclude group: "org.junit.jupiter"
}
testImplementation group: 'org.springframework.security', name: 'spring-security-test'
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.1'
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.2'
testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.6.1'
testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '2.23.0'
testRuntimeOnly group: 'org.junit.platform', name: 'junit-platform-commons', version: '1.8.2'
testImplementation group: 'org.junit.vintage', name: 'junit-vintage-engine', {
exclude module: "org.hamcrest:hamcrest-core"
}
compileOnly group: 'org.projectlombok', name: 'lombok', version: '1.18.22'
testImplementation group: 'org.projectlombok', name: 'lombok', version: '1.18.22'
testAnnotationProcessor 'org.projectlombok:lombok:1.18.22'
annotationProcessor 'org.projectlombok:lombok:1.18.22'
}
If I delete testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.6.1' it runs successfully, so it seems like somehow that dependency causes the issue, but unfortunately I definitely need that.
I have both JUnit 4 and JUnit 5 tests in my project. There are some test with #RunWith(SpringRunner.class) and some with #ExtendWith(MockitoExtension.class) annotations. I am using Java 11.
Any ideas are welcomed!

You probably use this plugin.
apply plugin: 'io.spring.dependency-management'
and this plugin fix the bytebuddy version to 1.11.22 where mockito 4.6.1 need 1.12.10.
So according to Could not initialize plugin: interface org.mockito.plugins.MockMaker after upgrading Mockito from 4.4.0 to 4.5.0
You need to add
testImplementation 'net.bytebuddy:byte-buddy:1.12.10'
To override the version.

Spring Boot migration from 1.5.22 to 2.0.0

I migrate my app from Spring Boot 1.5.22 to 2.7.0 and I have a problem. Java 8. To migrate to 2.0.0 I have to replace save () with saveAll (), replace the postgres driver: compile group: 'postgresql', name: 'postgresql', version: '9.1-901-1.jdbc4' with implementation ' org.postgresql: postgresql: 42.2.9 ', I also add this line to application.properties:
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation = true. After that, I build without mistakes, but precisely: when I start shooting with Postman, with the correct token I get 401. (Before it was 200). When I started analyzing it, it turned out that after Spring migration, when the project is being built, postgres tables on localhost are created correctly, but they are empty, (that's why I get 401). I don't know how to work around it. Would you have an idea? I'm pasting build.gradle:
import org.apache.tools.ant.filters.ReplaceTokens
buildscript {
repositories {
mavenCentral()
}
dependencies {
classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.0.0.RELEASE'
}
}
plugins {
id "io.franzbecker.gradle-lombok" version "1.11"
}
apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin: 'org.springframework.boot'
apply plugin: "io.spring.dependency-management"
project.version = '1.1.5'
jar {
baseName = 'refurbishment'
version = project.version
}
sourceSets {
util {
compileClasspath += sourceSets.main.runtimeClasspath
}
}
sourceCompatibility = 1.8
targetCompatibility = 1.8
repositories {
maven { url "http://projectlombok.org/mavenrepo" }
maven { url "http://repo.maven.apache.org/maven2" }
}
dependencies {
compile 'org.springframework.boot:spring-boot-starter-web'
compile 'org.springframework.boot:spring-boot-starter-actuator'
compile group: 'org.springframework.security.oauth', name: 'spring-security-oauth2', version: '2.5.2.RELEASE'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-mail', version:'1.5.8.RELEASE'
compile group: 'org.springframework.security', name: 'spring-security-jwt', version:'1.1.1.RELEASE'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version:'1.5.8.RELEASE'
implementation 'org.springframework.boot:spring-boot-properties-migrator:2.1.18.RELEASE'
testCompile 'org.springframework.boot:spring-boot-starter-test'
compile group: 'org.hibernate', name: 'hibernate-validator', version:'5.2.4.Final'
compile group: 'com.google.guava', name: 'guava', version:'16.0.1'
compile group: 'joda-time', name: 'joda-time', version:'2.8.2'
compile group: 'commons-lang', name: 'commons-lang', version:'2.6'
compile group: 'net.sf.ehcache', name: 'ehcache-core', version:'2.6.9'
implementation 'org.postgresql:postgresql:42.2.9'
compile group: 'io.jsonwebtoken', name: 'jjwt', version:'0.6.0'
compile group: 'org.projectlombok', name: 'lombok', version:'1.16.6'
compile group: 'org.apache.commons', name: 'commons-csv', version:'1.2'
compile group: 'com.google.code.gson', name: 'gson', version:'2.6.2'
compile group: 'commons-io', name: 'commons-io', version:'2.4'
compile group: 'org.json', name: 'json', version:'20160212'
compile group: 'io.springfox', name: 'springfox-swagger-ui', version:'2.5.0'
compile group: 'io.springfox', name: 'springfox-swagger2', version:'2.5.0'
compile group: 'com.fasterxml', name: 'classmate', version:'1.3.1'
compile 'com.microsoft.azure:azure-storage:5.4.0'
runtime group: 'com.microsoft.sqlserver', name: 'mssql-jdbc', version: '6.3.4.jre8-preview'
compile files('libs/PDFjet-5.1.jar')
testCompile(group: 'junit', name: 'junit')
//DBUnit
testCompile 'org.dbunit:dbunit:2.5.3'
testCompile 'com.github.springtestdbunit:spring-test-dbunit:1.3.0'
testCompile group: 'org.hamcrest', name: 'hamcrest-all', version:'1.3'
testCompile group: 'org.assertj', name: 'assertj-core', version:'1.7.0'
testCompile(group: 'org.mockito', name: 'mockito-core', version:'1.10.19') {
exclude(module: 'hamcrest-core')
}
testCompile group: 'org.springframework', name: 'spring-test', version:'4.2.6.RELEASE'
testCompile group: 'com.jayway.jsonpath', name: 'json-path', version:'2.0.0'
testCompile group: 'com.jayway.jsonpath', name: 'json-path-assert', version:'0.9.1'
}
bootRun {
def profiles = findProperty('profiles')
if (profiles) {
args = ["--spring.profiles.active=" + profiles]
}
}
task copyWebConfig(type: Copy) {
from('src/main/templates') {
include 'web.config'
}
into "$buildDir/libs"
filter(ReplaceTokens, tokens: [VERSION: project.version])
inputs.property("VERSION", project.version)
}
assemble.dependsOn(copyWebConfig)

Solved :) Added spring.datasource.initialization-mode=always to application.properties (working for Spring Boot 2.0.0)

Spring Boot: Project won't compile after changing gradle dependencies

I have a rather strange issue with my Spring boot application which has been driving me around the bend.
So I have my Spring boot API which is in Github and uses Gradle to manage my dependencies. If I add a dependency and compile the project everything works fine. However, if I decide I do not need said dependency anymore and remove it from my build file, my Project will no longer compile stating it cant find packages / symbols for dependencies unrelated to what I removed.
Has anyone seen something like this before? As an FYI I'm using Intellij.
To give an example, here is my current build file:
plugins {
id "org.springframework.boot" version "2.4.2"
id 'io.spring.dependency-management' version '1.0.8.RELEASE'
id 'java'
id "com.commercehub.gradle.plugin.avro" version "0.21.0"
id "idea"
}
group 'org.example'
version '1.0-SNAPSHOT'
java {
sourceCompatibility = JavaVersion.VERSION_14
targetCompatibility = JavaVersion.VERSION_14
}
ext {
avroVersion = "1.10.1"
}
repositories {
mavenCentral()
jcenter()
maven {
url "https://packages.confluent.io/maven/"
}
}
avro {
createSetters = true
fieldVisibility = "PRIVATE"
}
apply plugin: "war"
dependencies {
providedRuntime 'org.springframework.boot:spring-boot-starter-tomcat'
implementation group: 'net.logstash.logback', name: 'logstash-logback-encoder', version: '6.6'
compile group: 'co.elastic.logging', name: 'logback-ecs-encoder', version: '0.5.2'
compile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.860'
implementation 'org.springframework.boot:spring-boot-starter-data-redis'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-cache', version: '2.4.3'
// compile 'redis.clients:jedis:3.5.1'
implementation group: 'com.mashape.unirest', name: 'unirest-java', version: '1.4.9'
compile group: 'io.springfox', name: 'springfox-swagger-ui', version: '3.0.0'
compile group: 'io.springfox', name: 'springfox-boot-starter', version: '3.0.0'
compile('org.springframework.boot:spring-boot-starter-data-elasticsearch')
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-mongodb', version: '2.3.3.RELEASE'
compile group: 'org.springframework.data', name: 'spring-data-elasticsearch', version: '4.0.4.RELEASE'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-security', version: '2.3.3.RELEASE'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: '2.3.3.RELEASE'
compile group: 'org.springframework.security', name: 'spring-security-oauth2-client', version: '5.4.0'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-validation', version: '2.4.2'
compile group: 'org.springframework.kafka', name: 'spring-kafka', version: '2.6.5'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-oauth2-resource-server', version: '2.4.2'
compile group: 'mysql', name: 'mysql-connector-java', version: '8.0.21'
compile group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
compile group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: '2.11.2'
compile group: 'io.jsonwebtoken', name: 'jjwt', version: '0.9.1'
compile group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.1'
compile group: 'commons-io', name: 'commons-io', version: '2.6'
compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'
compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.11'
compile group: 'org.passay', name: 'passay', version: '1.6.0'
compile group: 'com.google.guava', name: 'guava', version: '30.0-jre'
// https://mvnrepository.com/artifact/com.auth0/java-jwt
compile group: 'com.auth0', name: 'java-jwt', version: '3.12.0'
compile group: 'io.confluent', name: 'kafka-schema-registry-client', version: '6.0.0'
compile group: 'io.confluent', name: 'kafka-avro-serializer', version: '6.0.0'
compile group: 'io.confluent', name: 'monitoring-interceptors', version: '6.0.0'
compile(group: 'io.confluent', name: 'kafka-streams-avro-serde', version: '6.0.0') {
exclude(module: 'log4j-over-slf4j')
}
compile "org.apache.avro:avro:1.10.1"
implementation "org.apache.avro:avro:${avroVersion}"
compileOnly 'org.projectlombok:lombok:1.18.12'
annotationProcessor 'org.projectlombok:lombok:1.18.12'
implementation 'com.amazonaws:aws-java-sdk-s3'
implementation 'org.springframework.boot:spring-boot-starter-web'
testCompile group: 'junit', name: 'junit', version: '4.12'
testCompileOnly 'org.projectlombok:lombok:1.18.12'
testAnnotationProcessor 'org.projectlombok:lombok:1.18.12'
testImplementation('org.springframework.boot:spring-boot-starter-test') {
exclude group: 'org.junit.vintage', module: 'junit-vintage-engine'
}
}
jar {
manifest {
attributes(
'Main-Class': 'com.test.TestApplication'
)
}
from {
configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
}
}
test {
useJUnitPlatform()
}
Now let's assume I don't want logstash so I remove it, this happens with any dependency
implementation group: 'net.logstash.logback', name: 'logstash-logback-encoder', version: '6.6'
Now my build file looks like this:
plugins {
id "org.springframework.boot" version "2.4.2"
id 'io.spring.dependency-management' version '1.0.8.RELEASE'
id 'java'
id "com.commercehub.gradle.plugin.avro" version "0.21.0"
id "idea"
}
group 'org.example'
version '1.0-SNAPSHOT'
java {
sourceCompatibility = JavaVersion.VERSION_14
targetCompatibility = JavaVersion.VERSION_14
}
ext {
avroVersion = "1.10.1"
}
repositories {
mavenCentral()
jcenter()
maven {
url "https://packages.confluent.io/maven/"
}
}
avro {
createSetters = true
fieldVisibility = "PRIVATE"
}
apply plugin: "war"
dependencies {
providedRuntime 'org.springframework.boot:spring-boot-starter-tomcat'
// REMOVED implementation group: 'net.logstash.logback', name: 'logstash-logback-encoder', version: '6.6'
compile group: 'co.elastic.logging', name: 'logback-ecs-encoder', version: '0.5.2'
compile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.860'
implementation 'org.springframework.boot:spring-boot-starter-data-redis'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-cache', version: '2.4.3'
// compile 'redis.clients:jedis:3.5.1'
implementation group: 'com.mashape.unirest', name: 'unirest-java', version: '1.4.9'
compile group: 'io.springfox', name: 'springfox-swagger-ui', version: '3.0.0'
compile group: 'io.springfox', name: 'springfox-boot-starter', version: '3.0.0'
compile('org.springframework.boot:spring-boot-starter-data-elasticsearch')
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-mongodb', version: '2.3.3.RELEASE'
compile group: 'org.springframework.data', name: 'spring-data-elasticsearch', version: '4.0.4.RELEASE'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-security', version: '2.3.3.RELEASE'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: '2.3.3.RELEASE'
compile group: 'org.springframework.security', name: 'spring-security-oauth2-client', version: '5.4.0'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-validation', version: '2.4.2'
compile group: 'org.springframework.kafka', name: 'spring-kafka', version: '2.6.5'
compile group: 'org.springframework.boot', name: 'spring-boot-starter-oauth2-resource-server', version: '2.4.2'
compile group: 'mysql', name: 'mysql-connector-java', version: '8.0.21'
compile group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
compile group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: '2.11.2'
compile group: 'io.jsonwebtoken', name: 'jjwt', version: '0.9.1'
compile group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.1'
compile group: 'commons-io', name: 'commons-io', version: '2.6'
compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'
compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.11'
compile group: 'org.passay', name: 'passay', version: '1.6.0'
compile group: 'com.google.guava', name: 'guava', version: '30.0-jre'
// https://mvnrepository.com/artifact/com.auth0/java-jwt
compile group: 'com.auth0', name: 'java-jwt', version: '3.12.0'
compile group: 'io.confluent', name: 'kafka-schema-registry-client', version: '6.0.0'
compile group: 'io.confluent', name: 'kafka-avro-serializer', version: '6.0.0'
compile group: 'io.confluent', name: 'monitoring-interceptors', version: '6.0.0'
compile(group: 'io.confluent', name: 'kafka-streams-avro-serde', version: '6.0.0') {
exclude(module: 'log4j-over-slf4j')
}
compile "org.apache.avro:avro:1.10.1"
implementation "org.apache.avro:avro:${avroVersion}"
compileOnly 'org.projectlombok:lombok:1.18.12'
annotationProcessor 'org.projectlombok:lombok:1.18.12'
implementation 'com.amazonaws:aws-java-sdk-s3'
implementation 'org.springframework.boot:spring-boot-starter-web'
testCompile group: 'junit', name: 'junit', version: '4.12'
testCompileOnly 'org.projectlombok:lombok:1.18.12'
testAnnotationProcessor 'org.projectlombok:lombok:1.18.12'
testImplementation('org.springframework.boot:spring-boot-starter-test') {
exclude group: 'org.junit.vintage', module: 'junit-vintage-engine'
}
}
jar {
manifest {
attributes(
'Main-Class': 'com.test.TestApplication'
)
}
from {
configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
}
}
test {
useJUnitPlatform()
}
Once I hit build I'll get non related logstash compilation errors such as in various classes:
java: package lombok does not exist
java: package org.openapitools.jackson.nullable does not exist
java: cannot find symbol
symbol: class Getter
java: package org.springframework.data.domain does not exist
java: package org.openapitools.jackson.nullable does not exist
I've tried reimporting my project, I've deleted it and pull down a fresh copy from Git..I'm at a loss now. Please if anyone has come across this let me know as google is failing me now :-/
Many thanks
EDIT: Please note for clarification, this seems to only happen when I have Intellij performing the build and Run.
If I switch the below to Gradle I've no issues remove dependencies as described above. However, I'd like to keep Intellij performing the build and run.
Edit:
Adding gradle-wrapper.properties in case it helps
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
Also, following a suggestion below, I purged the cache in Intellij which Did solve the problem, However, once I remove another dependency I'm back to getting the errors above.
Second Edit
I'm not sure it matters but I noticed since Lombok is the main error I checked the annotation settings.
Why is there a default & a gradle imported, Would that be causing any of the issues?

Publish Messages on Kafka using Spark

I m currently trying to write a Publisher for Kafka Messages using Spark and Java. The requirement is I will have a Dataset with few records. I need to only publish these records on Kafka queue on a topic.
Following is the publisher code written in Java:
Dataset dataset = messageDataSet.select("value");
dataset.selectExpr("CAST(value AS STRING)")
.write()
.format("kafka")
.option("topic","batch")
.option("kafka.bootstrap.servers","kafka01.qa.com:9092")
.save();
I have added following things is gradle , since the deployment guide of Streaming Kafka Integration says to have the gradle dependecies as provided I have also tried that.
dependencies {
compile group: 'com.util', name:'core-utilities-jdk8',version: '0.0.2-spark'
compile group: 'org.postgresql', name: 'postgresql', version: '42.2.8'
compile group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.11.687'
compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
compile group: 'org.apache.spark', name: 'spark-core_2.12', version: "$sparkVersion"
compile group: 'org.apache.spark', name: 'spark-sql_2.12', version: "$sparkVersion"
compile group: 'com.databricks', name: 'spark-csv_2.11', version: '1.5.0'
compile group: 'commons-logging', name: 'commons-logging', version: '1.2'
compile group: 'org.apache.spark', name: 'spark-streaming-kafka-0-10_2.11', version: "$sparkVersion"
compile group: 'org.apache.spark', name: 'spark-streaming_2.12', version: "$sparkVersion"
compile group: 'org.apache.spark', name: 'spark-sql-kafka-0-10_2.12', version: "$sparkVersion"
annotationProcessor 'org.projectlombok:lombok:1.18.10'
compile group: 'org.projectlombok', name: 'lombok', version: '1.18.10'
testCompile group: 'junit', name: 'junit', version: '4.12'
testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19'
compile 'org.apache.httpcomponents:httpmime:4.3.6'
compile group: 'org.apache.commons', name: 'commons-csv', version: '1.7'
compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.12', version: '2.10.1'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1'
compile group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: '2.10.1'
compile group: 'com.jcraft', name: 'jsch', version: '0.1.55'
}
The dependecy I have tried as ProvidedCompile for Kafka and Spark Core but the error continues to be following :
User class threw exception: java.util.ServiceConfigurationError:
org.apache.spark.sql.sources.DataSourceRegister: Provider
org.apache.spark.sql.kafka010.KafkaSourceProvider could not be instantiated
at java.util.ServiceLoader.fail(ServiceLoader.java:232)
at java.util.ServiceLoader.access$100(ServiceLoader.java:185)
at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384)
at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:43)
at scala.collection.Iterator$class.foreach(Iterator.scala:891)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at scala.collection.TraversableLike$class.filterImpl(TraversableLike.scala:247)
at scala.collection.TraversableLike$class.filter(TraversableLike.scala:259)
at scala.collection.AbstractTraversable.filter(Traversable.scala:104)
at org.apache.spark.sql.execution.datasources.DataSource$.lookupDataSource(DataSource.scala:630)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:245)
at com.fileprocessor.jobs.Runner.publishMessage(Runner.java:200)
at com.fileprocessor.SparkFileProcessorMain.main(SparkFileProcessorMain.java:12)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:684)
Caused by: java.lang.NoSuchMethodError:
org.apache.spark.internal.Logging.$init$(Lorg/apache/spark/internal/Logging;)V
at org.apache.spark.sql.kafka010.KafkaSourceProvider.<init>(KafkaSourceProvider.scala:44)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)
I want to simply publish by dataset to Kafka Queues . Spark version is 2.4.4
I have went through the different results of Having transformenrs in Gradle , or downgrading the things for Kafka versions and scala . But nothing has helped . Anyone if have faced similar situation ?
Tried : spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.4 --class com.fileprocessor.SparkFileProcessorMain --master local[*] file-processor-spark-jobs-1.0-SNAPSHOT.jar
Or If any one have anyother way to write the similar code ?

Following gradle worked:
dependencies {
compile group: 'org.postgresql', name: 'postgresql', version: '42.2.8'
compile group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.11.687'
compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
compile group: 'org.apache.spark', name: 'spark-core_2.12', version: "$sparkVersion"
compile group: 'org.apache.spark', name: 'spark-sql_2.12', version: "$sparkVersion"
compile group: 'com.databricks', name: 'spark-csv_2.11', version: '1.5.0'
compile group: 'commons-logging', name: 'commons-logging', version: '1.2'
compile group: 'org.apache.spark', name: 'spark-streaming-kafka-0-10_2.11', version: "$sparkVersion"
providedCompile group: 'org.apache.spark', name: 'spark-streaming_2.12', version: "$sparkVersion"
providedCompile group: 'org.apache.spark', name: 'spark-sql-kafka-0-10_2.11', version: "2.2.0"
annotationProcessor 'org.projectlombok:lombok:1.18.10'
compile group: 'org.projectlombok', name: 'lombok', version: '1.18.10'
testCompile group: 'junit', name: 'junit', version: '4.12'
testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19'
compile 'org.apache.httpcomponents:httpmime:4.3.6'
compile group: 'org.apache.commons', name: 'commons-csv', version: '1.7'
compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.12', version: '2.10.1'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1'
compile group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: '2.10.1'
compile group: 'com.jcraft', name: 'jsch', version: '0.1.55'
}
And spark submit : spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.0 --class com.fileprocessor.SparkFileProcessorMain --master local[*] file-processor-spark-jobs-1.0-SNAPSHOT.jar test bottomline false

Gradle Package does not exist

I am trying to build Gradle using ./gradlew build but during compilation getting following error package not found.In eclipse I am able to run refresh gradle but in command prompt facing the below issue :
> Task :compileJava FAILED
/Users/Documents/em-cedm-integ-test/src/main/java/com/BaseTest.java:3: error: package io.restassured does not exist
import static io.restassured.RestAssured.given;
^
/Users/Documents/em-cedm-integ-test/src/main/java/com/BaseTest.java:3: error: static import only from classes and interfaces
import static io.restassured.RestAssured.given;
^
/Users/Documents/em-cedm-integ-test/src/main/java/com/BaseTest.java:21: error: package org.apache.commons.lang3 does not exist
import org.apache.commons.lang3.StringUtils;
^
/Users/Documents/em-cedm-integ-test/src/main/java/com/BaseTest.java:34: error: package io.restassured.response does not exist
import io.restassured.response.ExtractableResponse;
^
/Users/Documents/em-cedm-integ-test/src/main/java/com/BaseTest.java:35: error: package io.restassured.response does not exist
import io.restassured.response.Response;
^
/Users/Documents/em-cedm-integ-test/src/main/java/com/CedmTest.java:3: error: package org.junit.runner does not exist
import org.junit.runner.JUnitCore;
^
^
Build.gradle file I am using
// Apply the java-library plugin to add support for Java Library
apply plugin: 'java'
allprojects {
repositories {
// You can declare any Maven/Ivy/file repository here.
maven {
url "http://repo.hortonworks.com/content/repositories/releases"
}
maven {
url "${artifactory_contextUrl}"
credentials {
username = "${artifactory_user}"
password = "${artifactory_password}"
}
}
maven {
url "${artifactory_contextUrl}/ip-fci-maven-virtual"
credentials {
username = "${artifactory_user}"
password = "${artifactory_password}"
}
}
}
configurations.all {
transitive = false
}
}
version = '1.0'
task fatJar(type: Jar) {
manifest {
attributes 'Implementation-Title': 'Gradle Jar File Example',
'Implementation-Version': version,
'Main-Class': 'com.ibm.cedm.CedmTest'
}
baseName = project.name + '-all'
from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
with jar
}
defaultTasks 'downloadFile'
dependencies {
compile group: 'com.google.guava', name: 'guava', version: '12.0.1'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.9.7'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.7'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.9.7'
compile group: 'org.apache.kafka', name: 'kafka-clients', version: '0.11.0.3'
compile group: 'org.apache.hbase', name: 'hbase-client', version: '1.1.2.2.6.4.0-91'
compile group: 'org.apache.hbase', name: 'hbase-common', version: '1.1.2.2.6.4.0-91'
compile group: 'org.apache.hbase', name: 'hbase-client', version: '1.1.2.2.6.4.0-91'
compile group: 'org.apache.hadoop', name: 'hadoop-common', version: '2.7.3.2.6.4.0-91'
compile group: 'com.tdunning', name: 'json', version: '1.8'
compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.6'
compile group: 'org.apache.httpcomponents', name: 'httpcore', version: '4.4.10'
compile group: 'org.apache.oozie', name: 'oozie-client', version: '4.2.0'
compileOnly group: 'io.swagger', name: 'swagger-annotations', version: '1.5.12'
compileOnly group: 'javax', name: 'javaee-api', version: '7.0'
compile group: 'ip-fci-generic-local.fcco-core', name: 'fci-core-utils', version: 'master'
compile(group: 'ip-fci-generic-local.media', name: 'db2jcc4', version: '11.1.3')
testImplementation 'junit:junit:4.12'
testCompile group: 'commons-codec', name: 'commons-codec', version: '1.9'
testCompile group: 'com.tdunning', name: 'json', version: '1.8'
testCompile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.6'
testCompile group: 'org.apache.httpcomponents', name: 'httpmime', version: '4.5.6'
testCompile group: 'org.apache.httpcomponents', name: 'httpcore', version: '4.4.10'
testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2'
testCompile group: 'commons-logging', name: 'commons-logging', version: '1.2'
testCompile group: 'io.rest-assured', name: 'rest-assured', version: '3.0.2'
testCompile group: 'io.rest-assured', name: 'rest-assured-common', version: '3.0.2'
testCompile group: 'io.rest-assured', name: 'json-path', version: '3.0.2'
testCompile group: 'io.rest-assured', name: 'xml-path', version: '3.0.2'
testCompile group: 'net.javacrumbs.json-unit', name: 'json-unit', version: '1.5.2'
testCompile group: 'net.javacrumbs.json-unit', name: 'json-unit-core', version: '1.5.2'
testCompile group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3'
testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3'
testCompile group: 'org.codehaus.groovy', name: 'groovy', version: '2.4.4'
testCompile group: 'org.codehaus.groovy', name: 'groovy-json', version: '2.4.4'
testCompile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25'
testCompile group: 'org.slf4j', name: 'slf4j-jdk14', version: '1.7.25'
}
/*
test {
filter {
//include specific method in any of the tests
includeTestsMatching "*createBasicIndividualParty"
}
}
*/
Can someone please tell me why it is so? and what changes I should I make it to work?

How dependency mapped
dependencies mapped with testCompile will be compiled with src/test/java classes
dependencies mapped with compile will be compiled with src/main/java classes
Issue with the gradle config
Few dependencies are mapped as testCompile but it could be referenced in production code src/main/java.
testCompile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2'
Should be
compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.2'