Skip to content

How to specify different user library path for different actions in an oozie workflow

How to specify different user library path for different actions in an oozie workflow

I have a spark action and a java action

<workflow-app name="batch-publisher" xmlns="uri:oozie:workflow:0.5">
    <start to="spark-a5b5"/>
    <kill name="Kill">
        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <action name="spark-a5b5">
        <spark xmlns="uri:oozie:spark-action:0.1">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <configuration>
                <property>
                    <name>oozie.libpath</name>
                    <value>${nameNode}/user/hdfs/sps-batch-bi/Jars/</value>
                </property>
                <property>
                    <name>oozie.use.system.libpath</name>
                    <value>true</value>
                </property>
            </configuration>
            <master>yarn</master>
            <mode>client</mode>
            <name>MySpark2</name>
              <class>com.sps.analytics.batch.bi.BatchApp</class>
            <jar>${nameNode}/user/hdfs/sps-batch-bi/Jars/sps-analytics-batch-bi-assembly-0.1.jar</jar>
              <spark-opts>--driver-memory 4G --executor-memory 4G --num-executors 2</spark-opts>
              <arg>/user/hdfs/sps-batch-bi/config.yaml</arg>
              <arg>/user/hdfs/sps-batch-bi/Query_sps_txn_Detail.xlsx</arg>
              <arg>spark</arg>
        </spark>
        <ok to="java-8283"/>
        <error to="Kill"/>
    </action>
    <action name="java-8283">
        <java>
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <configuration>
                <property>
                    <name>oozie.launcher.mapreduce.map.env</name>
                    <value>LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/tableau/lib64/tableausdk/</value>
                </property>
                <property>
                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
                    <value>true</value>
                </property>
            </configuration>
            <main-class>com.sps.analytics.bi.publisher.PublisherApp</main-class>
            <arg>sps-data-bi-dev</arg>
            <arg>default</arg>
            <arg>default</arg>
            <file>/user/sps/bi-publisher/config/application.properties#application.properties</file>
        </java>
        <ok to="End"/>
        <error to="Kill"/>
    </action>
    <end name="End"/>
</workflow-app>

How can I specify different library paths for two actions. I have conflicitng jars in these two assembly jars.

Answer

Making the action s sub-workflow helps resolves the jar incosistencies