d4science-ghn-cluster: CKAN, fixes to the smartgears/ckan-connector installation.

library/roles/smartgears/ckan_connector: New version, fix the upgrade phase.
library/roles/solr-tomcat-instance: Configure a multicore instance when requested.

ckan/ckan-solr/defaults/main.yml

@@ -1,2 +1,6 @@
 ---
 ckan_solr_port: 8983
+solr_multicore: True
+solr_cores:
+  - collection1
+

ckan/ckan-solr/tasks/main.yml

@@ -1,12 +1,14 @@
 ---
 - name: Install the solr schema used by CKAN
-  file: src=/usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml dest={{ tomcat_m_instances_base_path }}/{{ ckan_solr_port }}/solr/data/solr/collection1/conf/schema.xml state=link force=yes
+  file: src=/usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml dest={{ solr_collections_base_dir }}/{{ item }}/conf/schema.xml state=link force=yes
+  with_items: '{{ solr_cores }}'
   when: not ckan_geonetwork_harvester
   notify: Solr Restart
   tags: [ 'ckan', 'solr', 'solr_schema' ]
 
 - name: Install the solr schema used by CKAN, modified with the spatial fields
-  copy: src=schema.xml dest={{ tomcat_m_instances_base_path }}/{{ ckan_solr_port }}/solr/data/solr/collection1/conf/schema.xml force=yes
+  copy: src=schema.xml dest={{ solr_collections_base_dir }}/{{ item }}/conf/schema.xml force=yes
+  with_items: '{{ solr_cores }}'
   when: ckan_geonetwork_harvester
   notify: Solr Restart
   tags: [ 'ckan', 'solr', 'solr_schema' ]

smartgears/ckan_connector/defaults/main.yml

@@ -2,7 +2,8 @@
 gcube_repository: 'gcube-staging'
 ckan_connector_plugin_install: False
 ckan_connector_ver: 1.0.0-4.0.0-129609
-ckan_connector_name: 'ckan_connector-{{ ckan_connector_ver }}-{{ ckan_connector_ver }}.war'
+ckan_connector_name: ckan-connector
-ckan_connector_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/data/access/ckan_connector/{{ ckan_connector_ver}}/{{ ckan_connector_name }}'
+ckan_connector_filename: '{{ ckan_connector_name }}-{{ ckan_connector_ver }}.war'
-ckan_connector_war_file: ckan-connector.war
+ckan_connector_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/data/access/{{ ckan_connector_name }}/{{ ckan_connector_ver}}/{{ ckan_connector_filename }}'
+ckan_connector_war_file: '{{ ckan_connector_name }}.war'
 ckan_connector_user: ckan_connector

smartgears/ckan_connector/tasks/main.yml

@@ -2,7 +2,10 @@
 - block:
 
     - name: Remove the installed CKAN connector before upgrading
-      file: dest={{ smartgears_instance_path }}/webapps/ckan-connector state=absent
+      file: dest={{ item }} state=absent
+      with_items:
+        - '{{ smartgears_instance_path }}/webapps/ckan-connector'
+        - '{{ smartgears_instance_path }}/webapps/ckan-connector.war'
       when: smartgears_upgrade
 
     - name: Get the CKAN connector war file

solr-tomcat-instance/defaults/main.yml

@@ -23,11 +23,17 @@ solr_outside_tomcat_dir: False
 # We need to define this one because we are using the tomcat multiple instances role
 solr_tomcat_instance_dir: '{{ tomcat_m_instances_base_path }}/{{ solr_http_port }}'
 solr_data_dir: '{{ solr_tomcat_instance_dir }}/solr'
+solr_collections_base_dir: '{{ solr_data_dir }}/data/solr'
 solr_zookeeper_data_dir: '{{ solr_data_dir }}/zoo_data'
 solr_install_collection1: False
 # Stand alone
 solr_opts: "-DzkRun -DnumShards={{ solr_shards }}"
 # This is for the replica/sharded version
 # We need to pass a lot of options to the jdk for zookeeper and the solr shard configuration
-#solr_opts: "-DzkRun={{ ansible_fqdn}}:{{ solr_zoo_port }} -DnumShards={{ solr_shards }} -DzkHost=index1.t.hadoop.research-infrastructures.eu:{{ solr_zoo_port }},index2.t.hadoop.research-infrastructures.eu:{{ solr_zoo_port }},index3.t.hadoop.research-infrastructures.eu:{{ solr_zoo_port }} -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port={{ solr_jmx_port_1 }} -Dcom.sun.management.jmxremote.password.file={{ tomcat_conf_dir }}/jmxremote.passwd -Dcom.sun.management.jmxremote.access.file={{ tomcat_conf_dir }}/jmxremote.access"
+#solr_opts: "-DzkRun={{ ansible_fqdn}}:{{ solr_zoo_port }} -DnumShards={{ solr_shards }} -DzkHost=index1:{{ solr_zoo_port }},index2:{{ solr_zoo_port }},index3:{{ solr_zoo_port }}"
+
+# Define the following if you want a multicore installation
+#solr_multicore: True
+solr_cores:
+  - collection1
 

solr-tomcat-instance/tasks/main.yml

@@ -5,23 +5,20 @@
     - data/solr
     - webapps
     - zoo_data
-  tags:
+    - collection_data
-    - solr
+  tags: solr
 
 - name: Create the link from the tomcat instance for solr to the solr data directory
   file: src={{ solr_data_dir }}/data/solr dest={{ solr_tomcat_instance_dir }}/solr state=link
   when: solr_outside_tomcat_dir
-  tags:
+  tags: solr
-    - solr
 
 - name: Solr needs some additional packages
   apt: pkg={{ item }} state={{ pkg_state }}
   with_items:
     - libslf4j-java
     - libcommons-logging-java
-  tags:
+  tags: [ solr, tomcat ]
-    - solr
-    - tomcat
 
 - name: Let the additional packages jar files visible to tomcat
   file: src=/usr/share/java/{{ item }} dest={{ tomcat_catalina_home_dir }}/lib/{{ item }} state=link
@@ -32,51 +29,61 @@
     - jcl-over-slf4j.jar
     - commons-logging.jar
   notify: tomcat solr restart
-  tags:
+  tags: [ solr, tomcat ]
-    - solr
-    - tomcat
 
 - name: Install the solr webapp under <solr_data_mountpoint>/webapps
   copy: src=solr-{{ solr_version }}.war dest={{ solr_data_dir }}/webapps/solr-{{ solr_version }}.war owner={{ solr_user }} group={{ solr_user }} mode=0644
   register: solr_war_installed
   notify: tomcat solr restart
-  tags:
+  tags: [ solr, tomcat ]
-    - solr
-    - tomcat
 
 - name: Install the solr catalina definition
   template: src=catalina-{{ item }}.j2 dest={{ solr_tomcat_instance_dir }}/conf/Catalina/localhost/{{ item }} owner=root group=root mode=0444
   with_items:
     - solr.xml
   notify: tomcat solr restart
-  tags:
+  tags: [ solr, tomcat ]
-    - solr
+
+- name: Copy the solr collection1 and solr_core_base archives on the target machine
+  copy: src={{ item }} dest={{ solr_data_dir }}/collection_data owner={{ solr_user }} group={{ solr_user }}
+  with_items:
+    - collection1.tar.gz
+    - solr_core_base.tar.gz
+  tags: [ solr, tomcat ]
 
 - name: Install the solr collection1 example
-  unarchive: src=collection1.tar.gz dest={{ solr_data_dir }}/data/solr/
+  unarchive: src={{ solr_data_dir }}/collection_data/collection1.tar.gz dest={{ solr_collections_base_dir }} copy=no
   args:
-    creates: '{{ solr_data_dir }}/data/solr/collection1'
+    creates: '{{ solr_data_dir }}/data/solr/collection1/conf/solrconfig.xml'
   when: solr_install_collection1
   notify: tomcat solr restart
-  tags:
+  tags: [ solr, tomcat ]
-    - solr
-    - tomcat
 
-- name: Fix the collection1 permissions
+- name: Create the solr cores data directories
-  file: path={{ solr_data_dir }}/data/solr/ owner={{ solr_user }} group={{ solr_user }} recurse=yes
+  file: dest={{ solr_collections_base_dir }}/{{ item }} state=directory owner={{ solr_user }} group={{ solr_user }}
-  when: solr_install_collection1
+  with_items: '{{ solr_cores }}'
-  tags:
+  when: solr_multicore is defined and solr_multicore
-    - solr
+  tags: [ solr, tomcat ]
-    - tomcat
+  
+- name: Install the solr cores data on a multicore system
+  unarchive: src={{ solr_data_dir }}/collection_data/solr_core_base.tar.gz dest={{ solr_data_dir }}/data/solr/{{ item }} copy=no
+  args:
+    creates: '{{ solr_data_dir }}/data/solr/{{ item }}/conf/solrconfig.xml'
+  with_items: '{{ solr_cores }}'
+  when: solr_multicore is defined and solr_multicore
+  notify: tomcat solr restart
+  tags: [ solr, tomcat ]
+
+- name: Fix the cores permissions
+  file: path={{ solr_collections_base_dir }} owner={{ solr_user }} group={{ solr_user }} recurse=yes
+  tags: [ solr, tomcat ]
 
 - name: Install the tomcat.local default file
   template: src={{ item }}.j2 dest=/etc/default/tomcat-instance-{{ solr_http_port }}.local owner=root group={{ solr_user }} mode=0440
   with_items:
     - tomcat.local
   notify: tomcat solr restart
-  tags:
+  tags: [ solr, tomcat ]
-    - solr
-    - tomcat
 
 - name: Install the solr.xml and zookeeper conf files
   template: src={{ item }}.j2 dest={{ solr_data_dir }}/data/solr/{{ item }} owner=root group={{ solr_user }} mode=0440
@@ -84,7 +91,5 @@
     - solr.xml
     - zoo.cfg
   notify: tomcat solr restart
-  tags:
+  tags: [ solr, tomcat ]
-    - solr
-    - tomcat
 

solr-tomcat-instance/templates/solr.xml.j2

@@ -29,11 +29,13 @@
 <solr>
 
   <solrcloud>
+    {% if solr_multicore is not defined or not solr_multicore %}
     <str name="host">{{ ansible_fqdn }}</str>
     <int name="hostPort">{{ solr_http_port_1 }}</int>
     <str name="hostContext">${hostContext:solr}</str>
     <int name="zkClientTimeout">${zkClientTimeout:30000}</int>
     <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
+    {% endif %}
   </solrcloud>
 
   <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
@@ -41,4 +43,17 @@
     <int name="connTimeout">${connTimeout:0}</int>
   </shardHandlerFactory>
 
+  {% if solr_multicore is defined or solr_multicore %}
+  <cores adminPath="/admin/cores">
+    {% for core in solr_cores %}
+    
+    <core name="{{ core }}" instanceDir="{{ core }}">
+      <property name="dataDir" value="{{ solr_collections_base_dir }}/{{ core }}" />
+    </core>
+    
+    {% endfor %}
+  </cores>
+  {% endif %}
+
+
 </solr>