From 4fb268447c7d96fb6add2d9d78c68958a2aac93d Mon Sep 17 00:00:00 2001
From: Andrea Dell'Amico <adellam@isti.cnr.it>
Date: Tue, 3 Apr 2018 18:44:49 +0200
Subject: [PATCH] Add a task to install some bits of spark2. Fix the oozie
 task. Move the README into the internal wiki.

---
 hadoop/CDH/defaults/main.yml | 5 +++++
 hadoop/CDH/handlers/main.yml | 3 +++
 hadoop/CDH/tasks/cdm.yml     | 2 +-
 hadoop/CDH/tasks/main.yml    | 3 ++-
 hadoop/CDH/tasks/spark2.yml  | 8 ++++++++
 5 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 hadoop/CDH/handlers/main.yml
 create mode 100644 hadoop/CDH/tasks/spark2.yml

diff --git a/hadoop/CDH/defaults/main.yml b/hadoop/CDH/defaults/main.yml
index f11b270e..6a153589 100644
--- a/hadoop/CDH/defaults/main.yml
+++ b/hadoop/CDH/defaults/main.yml
@@ -5,6 +5,7 @@ cdh_version: 5.9.3
 cdh_packages_repo: 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm5 contrib'
 cdh_manager_repo_url: 'https://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/cloudera.list'
 #cdh_manager_download_url: 'http://archive.cloudera.com/cm5/installer/5.9.3/cloudera-manager-installer.bin'
+cdh_csd_directory: /opt/cloudera/csd
 
 # Set it to true on the oozie server nodes
 cdh_oozie_server: False
@@ -25,3 +26,7 @@ postgresql_jdbc_driver_version: 42.1.4
 # To integrate spark2 and oozie:
 # https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.0/bk_spark-component-guide/content/ch_oozie-spark-action.html#spark-config-oozie-spark2
 # https://community.cloudera.com/t5/Advanced-Analytics-Apache-Spark/Spark2-classpath-issues-with-Oozie/td-p/59782
+
+cdh_spark2_enabled: True
+cdh_spark2_jar: SPARK2_ON_YARN-2.2.0.cloudera2.jar
+cdh_spark2_csd_url: 'http://archive.cloudera.com/spark2/csd/{{ cdh_spark2_jar }}'
diff --git a/hadoop/CDH/handlers/main.yml b/hadoop/CDH/handlers/main.yml
new file mode 100644
index 00000000..d7d88002
--- /dev/null
+++ b/hadoop/CDH/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+- name: Restart cloudera-scm-server
+  service: name=cloudera-scm-server state=restarted
diff --git a/hadoop/CDH/tasks/cdm.yml b/hadoop/CDH/tasks/cdm.yml
index c4f34638..dfcfebb8 100644
--- a/hadoop/CDH/tasks/cdm.yml
+++ b/hadoop/CDH/tasks/cdm.yml
@@ -17,7 +17,7 @@
         tags: [ 'cdh', 'hadoop', 'cdh_m_conf' ]
   
       - name: Ensure that the cloudera manager daemon is running and enabled
-        service: name=cloudera-scm-server state=started enabled=yes
+        service: name=cloudera-scm-server state=restarted enabled=yes
   
     when: cdh_manager_install
     tags: [ 'cdh', 'hadoop' ]
diff --git a/hadoop/CDH/tasks/main.yml b/hadoop/CDH/tasks/main.yml
index d923d282..28b7138e 100644
--- a/hadoop/CDH/tasks/main.yml
+++ b/hadoop/CDH/tasks/main.yml
@@ -1,6 +1,7 @@
 ---
 - import_tasks: cdm.yml
   when: cdh_manager_install
+- import_tasks: spark2.yml
+  when: cdh_manager_install
 - import_tasks: oozie.yml
   when: cdh_oozie_server
-  
\ No newline at end of file
diff --git a/hadoop/CDH/tasks/spark2.yml b/hadoop/CDH/tasks/spark2.yml
new file mode 100644
index 00000000..7c8c1f0a
--- /dev/null
+++ b/hadoop/CDH/tasks/spark2.yml
@@ -0,0 +1,8 @@
+---
+- block:
+  - name: Download the Spark2 jar file
+    get_url: url={{ cdh_spark2_csd_url }} dest={{ cdh_csd_directory }} owner=cloudera-scm group=cloudera-scm
+    notify: Restart cloudera-scm-server
+
+  when: cdh_spark2_enabled
+  tags: [ 'cdh', 'hadoop', 'spark' ]
\ No newline at end of file