diff --git a/hadoop/CDH/defaults/main.yml b/hadoop/CDH/defaults/main.yml index f11b270e..6a153589 100644 --- a/hadoop/CDH/defaults/main.yml +++ b/hadoop/CDH/defaults/main.yml @@ -5,6 +5,7 @@ cdh_version: 5.9.3 cdh_packages_repo: 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm5 contrib' cdh_manager_repo_url: 'https://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/cloudera.list' #cdh_manager_download_url: 'http://archive.cloudera.com/cm5/installer/5.9.3/cloudera-manager-installer.bin' +cdh_csd_directory: /opt/cloudera/csd # Set it to true on the oozie server nodes cdh_oozie_server: False @@ -25,3 +26,7 @@ postgresql_jdbc_driver_version: 42.1.4 # To integrate spark2 and oozie: # https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.0/bk_spark-component-guide/content/ch_oozie-spark-action.html#spark-config-oozie-spark2 # https://community.cloudera.com/t5/Advanced-Analytics-Apache-Spark/Spark2-classpath-issues-with-Oozie/td-p/59782 + +cdh_spark2_enabled: True +cdh_spark2_jar: SPARK2_ON_YARN-2.2.0.cloudera2.jar +cdh_spark2_csd_url: 'http://archive.cloudera.com/spark2/csd/{{ cdh_spark2_jar }}' diff --git a/hadoop/CDH/handlers/main.yml b/hadoop/CDH/handlers/main.yml new file mode 100644 index 00000000..d7d88002 --- /dev/null +++ b/hadoop/CDH/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart cloudera-scm-server + service: name=cloudera-scm-server state=restarted diff --git a/hadoop/CDH/tasks/cdm.yml b/hadoop/CDH/tasks/cdm.yml index c4f34638..dfcfebb8 100644 --- a/hadoop/CDH/tasks/cdm.yml +++ b/hadoop/CDH/tasks/cdm.yml @@ -17,7 +17,7 @@ tags: [ 'cdh', 'hadoop', 'cdh_m_conf' ] - name: Ensure that the cloudera manager daemon is running and enabled - service: name=cloudera-scm-server state=started enabled=yes + service: name=cloudera-scm-server state=restarted enabled=yes when: cdh_manager_install tags: [ 'cdh', 'hadoop' ] diff --git a/hadoop/CDH/tasks/main.yml b/hadoop/CDH/tasks/main.yml index d923d282..28b7138e 100644 --- a/hadoop/CDH/tasks/main.yml +++ b/hadoop/CDH/tasks/main.yml @@ -1,6 +1,7 @@ --- - import_tasks: cdm.yml when: cdh_manager_install +- import_tasks: spark2.yml + when: cdh_manager_install - import_tasks: oozie.yml when: cdh_oozie_server - \ No newline at end of file diff --git a/hadoop/CDH/tasks/spark2.yml b/hadoop/CDH/tasks/spark2.yml new file mode 100644 index 00000000..7c8c1f0a --- /dev/null +++ b/hadoop/CDH/tasks/spark2.yml @@ -0,0 +1,8 @@ +--- +- block: + - name: Download the Spark2 jar file + get_url: url={{ cdh_spark2_csd_url }} dest={{ cdh_csd_directory }} owner=cloudera-scm group=cloudera-scm + notify: Restart cloudera-scm-server + + when: cdh_spark2_enabled + tags: [ 'cdh', 'hadoop', 'spark' ] \ No newline at end of file