I am getting this following error (Caused by: java.lang.ClassNotFoundException: play.api.libs.functional.syntax.package) while I am trying to run my code I have right dependencies and added right Jar in my spark lib folder still unable to figure it out.
play-json_2.11-2.7.4.jar --> /usr/lib/spark/jars/
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.0</version>
</dependency>
spark-submit --jars /usr/lib/spark/jars/play-json_2.11-2.7.4.jar
--class com.MyClass
${jar_path}
Thanks Sri
Advertisement
Answer
Fixed the error I have shaded few Jars which worked perfectly well.
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.blah</groupId>
<artifactId>blah</artifactId>
<version>2</version>
<properties>
<jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
<jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
<sonar.language>scala</sonar.language>
<sonar.sources>src/main/scala</sonar.sources>
<sonar.tests>src/test/scala</sonar.tests>
<sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
<sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
<sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
<sonar.host.url>https://sonar.cloud.blah.com </sonar.host.url>
<sonar.projectKey>blah</sonar.projectKey>
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<jacoco.plugin.version>0.8.3</jacoco.plugin.version>
<java.version>1.8</java.version>
<encoding>UTF-8</encoding>
<scala.version>2.11.8</scala.version>
<jackson.version>2.10.3</jackson.version>
<sonar.version>3.0</sonar.version>
<sonar.pluginKey>scala</sonar.pluginKey>
<stream-poc.version>0.2.4-23d5a6b</stream-poc.version>
<grpc.version>1.24.0</grpc.version>
<protobuf.version>3.6.1</protobuf.version>
<guava.version>26.0-android</guava.version>
<spark.version>2.4.5</spark.version>
</properties>
<distributionManagement>
<repository>
<id>central</id>
<name>blah Artifactory-releases</name>
<url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
</repository>
<snapshotRepository>
<id>snapshots</id>
<name>blah Artifactory-snapshots</name>
<url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>com.holdenkarau</groupId>
<artifactId>spark-testing-base_2.11</artifactId>
<version>2.4.5_0.14.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.sl4j</groupId>
<artifactId>sl4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.579</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.7</version>
</dependency>
<dependency>
<groupId>org.scalamock</groupId>
<artifactId>scalamock_2.11</artifactId>
<version>4.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-plugin-api</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-surefire-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-testing-harness</artifactId>
<version>${sonar.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.sonar.plugins</groupId>
<artifactId>sonar-cobertura-plugin</artifactId>
<version>${sonar.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.softwaremill.macwire</groupId>
<artifactId>macros_2.11</artifactId>
<version>2.3.7</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_2.11</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-json_2.11</artifactId>
<version>2.7.4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.11</artifactId>
<version>0.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.lihaoyi</groupId>
<artifactId>os-lib_2.11</artifactId>
<version>0.2.9</version>
</dependency>
<dependency>
<groupId>org.wvlet.airframe</groupId>
<artifactId>airframe-log_2.11</artifactId>
<version>20.10.3</version>
</dependency>
<!--
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.11</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>io.github.embeddedkafka</groupId>
<artifactId>embedded-kafka_2.11</artifactId>
<version>2.4.0</version>
</dependency>
https://github.com/scallop/scallop
<dependency>
<groupId>org.rogach</groupId>
<artifactId>scallop</artifactId>
<version>4.0.1</version>
</dependency>
https://github.com/ben-manes/caffeine
https://code.google.com/archive/p/findbugs-tutorials/
//find bugs
-->
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.6.0.1398</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<!-- Scala -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>com</pattern>
<shadedPattern>shaded.com</shadedPattern>
<includes>
<include>com.google.common.**</include>
<include>com.google.protobuf.**</include>
</includes>
</relocation>
</relocations>
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>
<exclude>junit:junit</exclude>
<exclude>jmock:*</exclude>
<exclude>*:xml-apis</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.scalatest:*</exclude>
<exclude>org.scalactic:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.5</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
<artifactId>maven-pdfreport-plugin</artifactId>
<version>1.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.sonatype.plexus</groupId>
<artifactId>plexus-cipher</artifactId>
<version>1.4</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Scoverage -->
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<aggregate>true</aggregate>
<highlighting>true</highlighting>
<failOnMinimumCoverage>false</failOnMinimumCoverage>
<excludedPackages>
misc
</excludedPackages>
</configuration>
<executions>
<execution>
<id>scoverage-report</id>
<goals>
<!-- Needed for Sonar -->
<goal>report-only</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- For all projects, the nexus-staging-maven-plugin provides the ability
to deploy the final packaged artifact in Nexus. -->
</plugins>
<finalName>
${project.artifactId}
<!-- Don't change this. -->
</finalName>
</build>
</project>