diff --git a/README.md b/README.md index 3637db8..7fc0696 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,20 @@ Role Name ========= -A brief description of the role goes here. - -Requirements ------------- - -Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. +A role that installs the CKAN catalogue portal, Role Variables -------------- -A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. +The most important variables are listed below: + +``` yaml +``` Dependencies ------------ -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. - -Example Playbook ----------------- - -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: - - - hosts: servers - roles: - - { role: username.rolename, x: 42 } +None License ------- @@ -35,4 +24,4 @@ EUPL-1.2 Author Information ------------------ -An optional section for the role authors to include contact information, or a website (HTML is not allowed). +Andrea Dell'Amico, diff --git a/defaults/main.yml b/defaults/main.yml index 95d3c70..d93f712 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,2 +1,212 @@ --- -# defaults file for ansible-role-template \ No newline at end of file +# To create the first sysadmin user: +# . /usr/lib/ckan/default/bin/activate +# cd /usr/lib/ckan/default/src/ckan +# You have to create your first CKAN sysadmin user from the command line. For example, to create a user called seanh and make him a # sysadmin: + +# paster sysadmin add seanh -c /etc/ckan/default/production.ini +# +# To create some test data: +# paster create-test-data -c /etc/ckan/default/production.ini + +ckan_version: 2.6 +ckan_deb_file: 'python-ckan_{{ ckan_version }}-{{ ansible_distribution_release }}_amd64.deb' +ckan_package_url: 'http://packaging.ckan.org/{{ ckan_deb_file }}' +ckan_libdir: /usr/lib/ckan +ckan_confdir: /etc/ckan/default +ckan_virtenv: '{{ ckan_libdir }}/default' +ckan_file_harvesting_dir: /var/lib/ckan +ckan_file_storage_dir: '{{ ckan_file_harvesting_dir }}/dev' +ckan_config_file: '{{ ckan_confdir }}/production.ini' +ckan_webapp_port: 8080 +ckan_solr_port: 8983 +ckan_shell_user: ckan +ckan_logdir: /var/log/ckan +ckan_db_name: ckan +ckan_db_user: ckan + +# By default, initialize the db and solr. Disable if you want to reinstall and maintain the old data +ckan_init_db_and_solr: True +# +ckan_clear_harvesting_enabled: True +ckan_clear_harvesting_freq: 'weekly' + +# CKAN plugins +ckan_plugins_state: present +# yes: update the repository. no, do not update +ckan_git_plugins_state: 'no' +# Order is important +ckan_geonetwork_harvester: False +ckan_ckanext_harvester_url: 'git+https://github.com/ckan/ckanext-harvest.git#egg=ckanext-harvest' +ckan_ckanext_spatial_url: 'git+https://github.com/okfn/ckanext-spatial.git#egg=ckanext-spatial' +ckan_geonetwork_harvester_url: 'https://github.com/geosolutions-it/ckanext-geonetwork.git' +ckan_geoview: False +ckan_geoview_url: ckanext-geoview +ckan_geoview_name: resource_proxy +ckan_dcat: False +ckan_dcat_url: 'git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat' +ckan_dcat_1_0_0_url: 'git+https://github.com/ckan/ckanext-dcat.git@v1.0.0#egg=ckanext-dcat' +# dcat implement harvesters too. +# ckan_dcat_name: 'dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface' +ckan_dcat_name: 'dcat dcat_json_interface' + +# Set this to true to install a cron job that regularly runs the harvesters +ckan_harvester_run: False +ckan_pdfview: False +ckan_ckanext_pdfview_url: ckanext-pdfview +ckan_privatedatasets: False +ckan_privatedatasets_url: ckanext-privatedatasets +ckan_privatedatasets_name: privatedatasets +ckan_hierarchy: False +ckan_hierarchy_url: 'git+https://github.com/datagovuk/ckanext-hierarchy.git#egg=ckanext-hierarchy' +ckan_hierarchy_name: hierarchy_display hierarchy_form +ckan_pages: False +ckan_pages_url: 'git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages' +ckan_pages_name: pages +ckan_ldap: False +#ckan_ldap_url: 'git+https://github.com/NaturalHistoryMuseum/ckanext-ldap' +ckan_ldap_url: 'https://github.com/NaturalHistoryMuseum/ckanext-ldap' +ckan_ldap_name: ldap +ckan_ldap_uri: 'ldap://ldap.example.org' +ckan_ldap_base_dn: '' +ckan_ldap_search_filter: 'uid={login}' +ckan_ldap_user_fullname: 'cn' +ckan_ldap_username: uid +ckan_ldap_email: mail +ckan_ldap_prevent_edits: True +ckan_ldap_fallback: True +ckan_ckanext_lire: False +ckan_ckanext_lire_n: lire +ckan_ckanext_lire_url: 'https://github.com/milicp/ckanext-lire.git' +# Kata OAI-PMH +ckan_kata_oai_pmh: False +ckan_oai_pmh_name: oaipmh +ckan_oai_pmh_state: absent +ckan_oai_pmh_url: 'git+https://github.com/kata-csc/ckanext-oaipmh#egg=ckanext-oaipmh' +ckan_oai_pmh_kata_plugin_url: 'git+https://github.com/kata-csc/ckanext-kata.git#egg=ckanext-kata' +ckan_oai_pmh_kata_ini_state: 'present' +ckan_oai_pmh_kata_ini_options: + - { section: 'app:main', option: 'kata.storage.malware_scan', value: 'false', state: '{{ ckan_oai_pmh_kata_ini_state }}' } + - { section: 'app:main', option: 'kata.ldap.enabled', value: 'false', state: '{{ ckan_oai_pmh_kata_ini_state }}' } + - { section: 'app:main', option: 'kata.disable_contact', value: 'true', state: '{{ ckan_oai_pmh_kata_ini_state }}' } + +# OLD OAI-PMH +ckan_oai_pm: False +ckan_oai_pm_name: oaipmh +ckan_oai_pm_state: absent +ckan_oai_pm_url: 'git+https://github.com/florenthemmi/ckanext-oaipmh#egg=ckanext-oaipm' +# Google analytics +ckan_google_analytics: False +ckan_ga_plugin_state: '{{ ckan_plugins_state }}' +ckan_google_analytics_name: googleanalytics +ckan_google_analytics_url: 'git+https://github.com/ckan/ckanext-googleanalytics.git#egg=ckanext-googleanalytics' +ckan_google_analytics_fixed_file: 'http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-catalogue/ckan-d4science-extension/{{ ckan_version }}/ckan-default/plugins/googleanalytics/plugin.py' +#CKANEXT-RATING +ckan_star_ratings: False +ckan_star_ratings_state: present +ckan_star_ratings_name: rating +ckan_star_ratings_url: 'git+https://github.com/6aika/ckanext-rating.git#egg=ckanext-rating' + +ckan_memcache_sessions: False +ckan_memcache_deb_pkgs: + - libmemcached10 + - libmemcached-dev + +ckan_memcache_ini_opts: + - { section: 'app:main', option: 'beaker.session.type', value: 'ext:memcached', state: 'present' } + - { section: 'app:main', option: 'beaker.session.url ', value: "{{ mc_ipaddress | default('127.0.0.1') }}:{{ mc_port | default('11211') }}", state: 'present' } + +# Google analytics reports +ckan_ga_reports: False +ckan_ga_reports_name: ga-report +ckan_ga_reports_url: 'git+https://github.com/datagovuk/ckanext-ga-report.git#egg=ckanext-ga-report' +ckan_profiler: False +ckan_profiler_url: 'git+https://github.com/morty/ckanext-profile.git#egg=ckanext-profile' +# CKAN-DATESEARCH +ckan_datesearch: False +ckan_datesearch_name: datesearch +ckan_datesearch_state: present +ckan_datesearch_url: 'https://github.com/EUDAT-B2FIND/ckanext-datesearch' + + +# Needed to install some CKAN plugins +ckan_additional_packages: + - git + - libxslt1-dev + - gcc + - python-dev + - libffi-dev + - libxml2-dev + - zlib1g-dev + - libxslt1-dev + - libgeos-c1 + - libldap2-dev + - libsasl2-dev + - libssl-dev + +ckan_pip_dependencies: + - lxml + - factory + - python-ldap + - rdflib + - 'urllib3[secure]' + - bleach + - pyOpenSSL + - idna + - certifi + - xmltodict + - ndg-httpsclient + - pyasn1 + - enum + - ipaddress + - x509 + +ckan_pip_versioned_dependencies: + - { name: 'SQLAlchemy', version: '0.9.6', state: 'present' } + - { name: 'cryptography', version: '2.8', state: 'present' } + +# +apache_additional_packages: + - libapache2-mod-uwsgi + - libpq5 + +apache_additional_modules: + - uwsgi + +ckan_production_ini_opts: + - { section: 'app:main', option: 'ckan.site_id', value: 'ckan_installation', state: 'present' } + - { section: 'app:main', option: 'sqlalchemy.url', value: 'postgresql://{{ ckan_db_user }}:{{ ckan_db_pwd }}@{{ psql_db_host }}/{{ ckan_db_name }}', state: 'present' } + - { section: 'app:main', option: 'ckan.site_url', value: 'http://{{ ansible_fqdn }}', state: 'present' } + - { section: 'app:main', option: 'solr_url', value: 'http://127.0.0.1:{{ ckan_solr_port }}/solr', state: 'present' } + - { section: 'app:main', option: 'ckan.datastore.write_url', value: 'postgresql://{{ ckan_db_user }}:{{ ckan_db_pwd }}@{{ psql_db_host }}/{{ ckan_datastore_db_name }}', state: 'present' } + - { section: 'app:main', option: 'ckan.datastore.read_url', value: 'postgresql://{{ ckan_datastore_db_reader }}:{{ ckan_db_pwd }}@{{ psql_db_host }}/{{ ckan_datastore_db_name }}', state: 'present' } + - { section: 'app:main', option: 'ckan.site_title', value: 'D4Science CKAN development installation', state: 'present' } + - { section: 'app:main', option: 'ckan.site_logo', value: '/base/images/ckan-logo.png', state: 'present' } + - { section: 'app:main', option: 'ckan.max_resource_size', value: '10', state: 'present' } + - { section: 'app:main', option: 'ckan.max_image_size', value: '2', state: 'present' } + - { section: 'app:main', option: 'ckan.tracking_enabled', value: 'true', state: 'present' } + - { section: 'app:main', option: 'ckan.privatedatasets.show_acquire_url_on_create', value: 'true', state: 'present' } + - { section: 'app:main', option: 'ckan.privatedatasets.show_acquire_url_on_edit', value: 'true', state: 'present' } + +ckan_production_ini_plugins_opts: + - { section: 'app:main', option: 'ckan.plugins', value: 'stats text_view image_view recline_view datastore datapusher harvest', state: 'present' } + - { section: 'app:main', option: 'ckan.datapusher.url', value: 'http://127.0.0.1:8800', state: 'present' } + - { section: 'app:main', option: 'ckan.datapusher.formats', value: 'csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', state: 'present' } + - { section: 'app:main', option: 'ckan.storage_path', value: '{{ ckan_file_storage_dir }}', state: 'present' } + - { section: 'app:main', option: 'ckan.harvest.mq.type', value: 'redis', state: 'present' } + - { section: 'app:main', option: 'ckan.harvest.mq.hostname', value: 'localhost', state: 'present' } + - { section: 'app:main', option: 'ckan.harvest.mq.port', value: '6379', state: 'present' } + - { section: 'app:main', option: 'ckan.harvest.mq.db', value: '0', state: 'present' } + - { section: 'app:main', option: 'ckanext.spatial.search_backend', value: 'solr', state: 'present' } + - { section: 'app:main', option: 'ckanext.pages.organization', value: 'true', state: 'present' } + - { section: 'app:main', option: 'ckanext.pages.group', value: 'true', state: 'present' } + - { section: 'app:main', option: 'ckanext.pages.about_menu', value: 'false', state: 'absent' } + - { section: 'app:main', option: 'ckanext.pages.group_menu', value: 'false', state: 'absent' } + - { section: 'app:main', option: 'ckanext.pages.organization_menu', value: 'false', state: 'absent' } + +ckan_gather_fetch_pkgs: + - supervisor + +ckan_gather_fetch_apps: + - ckan_gather_consumer + - ckan_fetch_consumer diff --git a/files/schema.xml b/files/schema.xml new file mode 100644 index 0000000..6707914 --- /dev/null +++ b/files/schema.xml @@ -0,0 +1,187 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +index_id +text + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/handlers/main.yml b/handlers/main.yml index 27474e0..1c03360 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,2 +1,10 @@ --- -# handlers file for ansible-role-template \ No newline at end of file +- name: Restart CKAN + service: name=apache2 state=restarted sleep=10 + +- name: Reconfigure the supervisor daemon + shell: supervisorctl reread ; supervisorctl add ckan_gather_consumer ; supervisorctl add ckan_fetch_consumer ; supervisorctl start ckan_gather_consumer ; supervisorctl start ckan_fetch_consumer + +- name: Restart fetch and gather consumers + supervisorctl: name={{ item }} state=restarted + with_items: '{{ ckan_gather_fetch_apps }}' diff --git a/meta/main.yml b/meta/main.yml index 1126a5e..11cdaba 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,61 +1,23 @@ galaxy_info: - author: your name - description: your description + author: Andrea Dell'Amico + description: Systems Architect company: ISTI-CNR - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value issue_tracker_url: https://redmine-s2i2s.isti.cnr.it/projects/provisioning - # Some suggested licenses: - # - BSD (default) - # - MIT - # - GPLv2 - # - GPLv3 - # - Apache - # - CC-BY - license: EUPL-1.2 + license: EUPL 1.2+ min_ansible_version: 2.8 - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - # Optionally specify the branch Galaxy will use when accessing the GitHub - # repo for this role. During role install, if no tags are available, - # Galaxy will use this branch. During import Galaxy will access files on - # this branch. If Travis integration is configured, only notifications for this - # branch will be accepted. Otherwise, in all cases, the repo's default branch - # (usually master) will be used. - #github_branch: - - # - # Provide a list of supported platforms, and for each platform a list of versions. - # If you don't wish to enumerate all versions for a particular platform, use 'all'. # To view available platforms and versions (or releases), visit: # https://galaxy.ansible.com/api/v1/platforms/ # - # platforms: - # - name: Fedora - # versions: - # - all - # - 25 - # - name: SomePlatform - # versions: - # - all - # - 1.0 - # - 7 - # - 99.99 - - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. + platforms: + - name: Ubuntu + versions: + - trusty + galaxy_tags: + - ckan dependencies: [] - # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. diff --git a/tasks/ckan-config.yml b/tasks/ckan-config.yml new file mode 100644 index 0000000..fef8961 --- /dev/null +++ b/tasks/ckan-config.yml @@ -0,0 +1,17 @@ +--- +- name: Configure the CKAN plugins list into the configuration file + ini_file: dest={{ ckan_config_file }} section={{ item.section }} option={{ item.option }} value={{ item.value }} state={{ item.state }} backup=no + with_items: '{{ ckan_production_ini_plugins_opts }}' + notify: + - Restart CKAN + - Restart fetch and gather consumers + tags: [ 'ckan', 'ckan_ini', 'ckan_plugins' ] + +- name: Configure the CKAN options used by the KATA plugin + ini_file: dest={{ ckan_config_file }} section={{ item.section }} option={{ item.option }} value={{ item.value }} state={{ item.state }} backup=no + with_items: '{{ ckan_oai_pmh_kata_ini_options }}' + notify: + - Restart CKAN + - Restart fetch and gather consumers + tags: [ 'ckan', 'ckan_ini', 'ckan_plugins', 'ckan_oai_pmh' ] + diff --git a/tasks/ckan-memcache.yml b/tasks/ckan-memcache.yml new file mode 100644 index 0000000..5267313 --- /dev/null +++ b/tasks/ckan-memcache.yml @@ -0,0 +1,50 @@ +--- +- block: + - name: Install the memcache library deb package + apt: pkg={{ ckan_memcache_deb_pkgs }} state=present cache_valid_time=1800 + + when: ckan_memcache_sessions is defined and ckan_memcache_sessions + tags: [ 'ckan', 'ckan_sessions', 'ckan_memcache' ] + +- block: + - name: Install the memcache library + pip: name=pylibmc virtualenv={{ ckan_virtenv }} state=present + + become: True + become_user: '{{ ckan_shell_user }}' + when: ckan_memcache_sessions is defined and ckan_memcache_sessions + tags: [ 'ckan', 'ckan_sessions', 'ckan_memcache' ] + +- block: + - name: Configure CKAN so that it uses memcache for its sessions + ini_file: dest={{ ckan_config_file }} section={{ item.section }} option={{ item.option }} value={{ item.value }} state={{ item.state }} + with_items: '{{ ckan_memcache_ini_opts }}' + notify: Restart fetch and gather consumers + register: ckan_use_memcache + tags: [ 'ckan', 'ckan_ini', 'ckan_plugins', 'ckan_sessions', 'ckan_memcache' ] + + - name: Restart CKAN after enabling the memcache sessions configuration + service: name=apache2 state=reloaded + when: ckan_use_memcache is changed + + - name: Remove the CKAN session files + file: dest=/tmp/{{ ckan_site_id }}/sessions state=absent + ignore_errors: True + + when: ckan_memcache_sessions is defined and ckan_memcache_sessions + tags: [ 'ckan', 'ckan_sessions', 'ckan_memcache' ] + +- block: + - name: Configure CKAN to not use memcache for its sessions + ini_file: dest={{ ckan_config_file }} section={{ item.section }} option={{ item.option }} value={{ item.value }} state=absent + with_items: '{{ ckan_memcache_ini_opts }}' + notify: + - Restart CKAN + - Restart fetch and gather consumers + tags: [ 'ckan', 'ckan_ini', 'ckan_plugins', 'ckan_sessions', 'ckan_memcache' ] + + when: + - ckan_memcache_sessions is defined + - not ckan_memcache_sessions + tags: [ 'ckan', 'ckan_sessions', 'ckan_memcache' ] + diff --git a/tasks/ckan-plugins.yml b/tasks/ckan-plugins.yml new file mode 100644 index 0000000..9aa50b0 --- /dev/null +++ b/tasks/ckan-plugins.yml @@ -0,0 +1,276 @@ +--- +- block: + - name: Install some packages dependencies + apt: name={{ ckan_additional_packages }} state=latest update_cache=yes cache_valid_time=3600 + + tags: [ 'ckan', 'geonetwork', 'ckan_plugins', 'ckan_pip_deps' ] + +- block: + - name: Upgrade pip inside the virtualenv + pip: name=pip virtualenv={{ ckan_virtenv }} state=latest + tags: [ 'ckan', 'geonetwork', 'ckan_plugins', 'ckan_pip_deps' ] + + - name: Install some python versioned plugins dependencies inside the CKAN virtualenv + pip: name={{ item.name }} virtualenv={{ ckan_virtenv }} version={{ item.version }} state={{ item.state }} + with_items: '{{ ckan_pip_versioned_dependencies }}' + tags: [ 'ckan', 'geonetwork', 'ckan_plugins', 'ckan_pip_deps' ] + + - name: Install some python plugins dependencies inside the CKAN virtualenv + pip: name={{ ckan_pip_dependencies }} virtualenv={{ ckan_virtenv }} state=present + tags: [ 'ckan', 'geonetwork', 'ckan_plugins', 'ckan_pip_deps' ] + + - name: Download the CKAN ckanext-harvest plugin + pip: name='{{ ckan_ckanext_harvester_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_geonetwork_harvester | bool + register: ckanext_harvest_install + notify: + - Restart CKAN + - Restart fetch and gather consumers + tags: [ 'ckan', 'geonetwork', 'ckan_plugins' ] + + - name: Download the CKAN ckanext-harvest requirements + pip: requirements={{ ckan_virtenv }}/src/ckanext-harvest/pip-requirements.txt virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} + when: ckan_geonetwork_harvester | bool + notify: Restart fetch and gather consumers + tags: [ 'ckan', 'geonetwork', 'ckan_plugins' ] + + - name: Initialize the CKAN ckanext-harvest plugin + shell: . /usr/lib/ckan/default/bin/activate ; paster --plugin=ckanext-harvest harvester initdb --config={{ ckan_config_file }} + when: + - ckanext_harvest_install is changed + - ckan_init_db_and_solr | bool + notify: Restart fetch and gather consumers + tags: [ 'ckan', 'geonetwork', 'ckan_plugins' ] + + - name: Download the CKAN ckanext-spatial plugin + pip: name='{{ ckan_ckanext_spatial_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + notify: Restart CKAN + when: ckan_geonetwork_harvester | bool + register: ckanext_spatial_install + tags: [ 'ckan', 'ckan_spatial', 'ckan_plugins' ] + + - name: Download the CKAN ckanext-spatial requirements + pip: requirements={{ ckan_virtenv }}/src/ckanext-spatial/pip-requirements.txt virtualenv={{ ckan_virtenv }} state=present + when: ckan_geonetwork_harvester | bool + tags: [ 'ckan', 'ckan_spatial', 'ckan_plugins' ] + + - name: Initialize the CKAN ckanext-spatial plugin + shell: . /usr/lib/ckan/default/bin/activate ; paster --plugin=ckanext-spatial spatial initdb --config={{ ckan_config_file }} + when: + - ckanext_spatial_install is changed + - ckan_init_db_and_solr | bool + tags: [ 'ckan', 'ckan_spatial', 'ckan_plugins' ] + + - name: Download the CKAN ckanext-geoview plugin + pip: name='{{ ckan_geoview_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} + notify: Restart CKAN + when: ckan_geoview | bool + tags: [ 'ckan', 'ckan_geoview', 'ckan_plugins' ] + + - name: Download the latest version of the CKAN ckanext-dcat plugin code on CKAN version >= 2.8 + pip: name={{ ckan_dcat_url }} virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: + - ckan_dcat | bool + - ckan_version is version_compare('2.8', '>=') + notify: Restart CKAN + tags: [ 'ckan', 'ckan_dcat', 'ckan_plugins' ] + + - name: Download the CKAN ckanext-dcat plugin code. Stick to version 1.0.0 on CKAN < 2.8 + pip: name={{ ckan_dcat_1_0_0_url }} virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: + - ckan_dcat | bool + - ckan_version is version_compare('2.8', '<') + notify: Restart CKAN + tags: [ 'ckan', 'ckan_dcat', 'ckan_plugins' ] + + - name: Download the CKAN ckanext-dcat requirements + pip: requirements={{ ckan_virtenv }}/src/ckanext-dcat/requirements.txt virtualenv={{ ckan_virtenv }} state=present + when: ckan_dcat | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_dcat', 'ckan_plugins' ] + + - name: Download the CKAN Geonetwork plugin code + git: repo={{ ckan_geonetwork_harvester_url }} dest=/usr/lib/ckan/default/src/ckanext-geonetwork force=yes update={{ ckan_git_plugins_state }} + when: ckan_geonetwork_harvester | bool + register: install_geonetwork_harvester + tags: [ 'ckan', 'ckan_geonetwork', 'ckan_plugins' ] + + - name: Install the CKAN Geonetwork plugin code + shell: . /usr/lib/ckan/default/bin/activate ; cd /usr/lib/ckan/default/src/ckanext-geonetwork ; python setup.py develop + when: install_geonetwork_harvester is changed + notify: Restart CKAN + tags: [ 'ckan', 'ckan_geonetwork', 'ckan_plugins' ] + + - name: Install the script that updates the tracking data + template: src=tracker_update.sh.j2 dest={{ ckan_virtenv }}/bin/tracker_update owner={{ ckan_shell_user }} group={{ ckan_shell_user }} mode=0555 + when: ckan_geonetwork_harvester | bool + tags: [ 'ckan', 'ckan_geonetwork', 'ckan_plugins', 'tracker' ] + + - name: Install the cron job that runs the tracker update script + cron: name="tracker update" minute="0" hour="3" job="{{ ckan_virtenv }}/bin/tracker_update > {{ ckan_logdir }}/tracker_update.log 2>&1" user={{ ckan_shell_user }} + when: ckan_geonetwork_harvester | bool + tags: [ 'ckan', 'ckan_geonetwork', 'ckan_plugins', 'tracker' ] + + - name: Download the CKAN PDF viewer plugin + pip: name='{{ ckan_ckanext_pdfview_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} + when: ckan_pdfview | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_pdfview', 'ckan_plugins' ] + + - name: Download the CKAN Privatedatasets extension for CKAN 2.8 + pip: name='{{ ckan_privatedatasets_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} + when: + - ckan_privatedatasets | bool + - ckan_version is version_compare('2.8', '>=') + notify: Restart CKAN + tags: [ 'ckan', 'ckan_privdatasets', 'ckan_plugins' ] + + - name: Download the CKAN Privatedatasets extension for CKAN 2.6 + pip: name='{{ ckan_privatedatasets_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} version=0.2.19 + when: + - ckan_privatedatasets | bool + - ckan_version is version_compare('2.8', '<') + notify: Restart CKAN + tags: [ 'ckan', 'ckan_privdatasets', 'ckan_plugins' ] + + - name: Download the CKAN hierarchy plugin code + pip: name='{{ ckan_hierarchy_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_hierarchy | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_hierarchy', 'ckan_plugins' ] + + - name: Download the CKAN pages plugin code + pip: name='{{ ckan_pages_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_pages | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_pages', 'ckan_plugins' ] + + - name: Download the CKAN LDAP plugin code + git: repo={{ ckan_ldap_url }} dest=/usr/lib/ckan/default/src/ckanext-ldap force=yes update={{ ckan_git_plugins_state }} + when: ckan_ldap | bool + register: install_ldap_plugin + tags: [ 'ckan', 'ckan_ldap', 'ckan_plugins' ] + + - name: Enable the CKAN ldap plugin code + shell: . /usr/lib/ckan/default/bin/activate ; cd /usr/lib/ckan/default/src/ckanext-ldap ; python setup.py develop + when: install_ldap_plugin is changed + notify: Restart CKAN + tags: [ 'ckan', 'ckan_ldap', 'ckan_plugins' ] + + - name: Download the CKAN LIRE plugin code + git: repo={{ ckan_ckanext_lire_url }} dest={{ ckan_virtenv }}/src/ckanext-lire force=yes update={{ ckan_git_plugins_state }} + when: ckan_ckanext_lire | bool + register: install_lire_plugin + tags: [ 'ckan', 'ckan_lire', 'ckan_plugins' ] + + - name: Activate the CKAN Lire plugin code + shell: . /usr/lib/ckan/default/bin/activate ; cd {{ ckan_virtenv }}/src/ckanext-lire ; pip install -e ./ + when: install_lire_plugin is changed + notify: Restart CKAN + tags: [ 'ckan', 'ckan_lire', 'ckan_plugins' ] + + - name: Download the KATA CKAN OAI-PMH plugin + pip: name='{{ ckan_oai_pmh_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_kata_oai_pmh | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_oai_pmh', 'ckan_plugins' ] + + - name: Download the KATA CKAN ckanext-oaiphm requirements + pip: requirements={{ ckan_virtenv }}/src/ckanext-oaipmh/requirements.txt virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} + when: ckan_kata_oai_pmh | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_oai_pmh', 'ckan_plugins' ] + + - name: Download the KATA CKAN plugin + pip: name='{{ ckan_oai_pmh_kata_plugin_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_kata_oai_pmh | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_oai_pmh', 'ckan_plugins' ] + + - name: Download the KATA CKAN requirements + pip: requirements={{ ckan_virtenv }}/src/ckanext-kata/requirements.txt virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} + when: ckan_kata_oai_pmh | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_oai_pmh', 'ckan_plugins' ] + + - name: Download the opendatasoft CKAN OAI-PMH plugin + pip: name='{{ ckan_oai_pm_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_oai_pm | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_oai_pm', 'ckan_plugins' ] + + - name: Download the CKAN google analytics plugin python requirements + pip: name='genshi' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} + when: ckan_google_analytics | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_google_analytics', 'ckan_plugins' ] + + - name: Download the CKAN google analytics plugin + pip: name='{{ ckan_google_analytics_url }}' virtualenv={{ ckan_virtenv }} editable=true state={{ ckan_ga_plugin_state }} + when: ckan_google_analytics | bool + register: install_ckan_google_analytics + notify: Restart CKAN + tags: [ 'ckan', 'ckan_google_analytics', 'ckan_plugins' ] + + - name: Setup the CKAN google analytics plugin + shell: . /usr/lib/ckan/default/bin/activate ; cd /usr/lib/ckan/default/src/ckanext-googleanalytics ; python setup.py develop + when: install_ckan_google_analytics is changed + notify: Restart CKAN + tags: [ 'ckan', 'ckan_google_analytics', 'ckan_plugins' ] + + - name: Download the CKAN google analytics reports plugin + pip: name='{{ ckan_ga_reports_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_ga_reports | bool + register: install_ckan_ga_reports + tags: [ 'ckan', 'ckan_google_analytics', 'ckan_ga_reports', 'ckan_plugins' ] + + - name: Setup the CKAN google analytics reports plugin + shell: . /usr/lib/ckan/default/bin/activate ; cd /usr/lib/ckan/default/src/ckanext-ga-report ; paster initdb --config={{ ckan_config_file }} + when: + - install_ckan_ga_reports is changed + - ckan_init_db_and_solr | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_google_analytics', 'ckan_ga_reports', 'ckan_plugins' ] + + - name: Download the CKAN star ratings plugin + pip: name='{{ ckan_star_ratings_url }}' virtualenv={{ ckan_virtenv }} editable=true state={{ ckan_star_ratings_state }} + notify: Restart CKAN + when: ckan_star_ratings | bool + register: install_ckan_star_ratings + tags: [ 'ckan', 'ckan_star_ratings', 'ckan_plugins' ] + + - name: Setup the CKAN star ratings plugin + shell: . /usr/lib/ckan/default/bin/activate ; paster --plugin=ckanext-rating rating init --config={{ ckan_config_file }} + notify: Restart CKAN + when: + - install_ckan_star_ratings is changed + - ckan_star_ratings | bool + tags: [ 'ckan', 'ckan_star_ratings', 'ckan_plugins' ] + + - name: Install the CKAN profiler plugin + pip: name='{{ ckan_profiler_url }}' virtualenv={{ ckan_virtenv }} state={{ ckan_plugins_state }} editable=True + when: ckan_profiler | bool + notify: Restart CKAN + tags: [ 'ckan', 'ckan_profiler', 'ckan_plugins' ] + + - name: Create the profiler plugin log directory + become_user: root + file: dest=/var/log/ckan-profiler owner=www-data group=www-data state=directory + when: ckan_profiler | bool + tags: [ 'ckan', 'ckan_profiler', 'ckan_plugins' ] + + - name: Download the CKAN-DATESEARCH plugin code + git: repo={{ ckan_datesearch_url }} dest=/usr/lib/ckan/default/src/ckanext-datesearch force=yes update={{ ckan_git_plugins_state }} + when: ckan_datesearch | bool + register: install_datesearch_plugin + tags: [ 'ckan', 'ckan_datesearch', 'ckan_plugins' ] + + - name: Enable the CKAN-DATESEARCH plugin code + shell: . /usr/lib/ckan/default/bin/activate ; cd /usr/lib/ckan/default/src/ckanext-datesearch ; python setup.py develop + when: install_datesearch_plugin is changed + notify: Restart CKAN + tags: [ 'ckan', 'ckan_datesearch', 'ckan_plugins' ] + + become: True + become_user: '{{ ckan_shell_user }}' + tags: [ 'ckan', 'ckan_plugins' ] diff --git a/tasks/ckan.yml b/tasks/ckan.yml new file mode 100644 index 0000000..dc6d99e --- /dev/null +++ b/tasks/ckan.yml @@ -0,0 +1,99 @@ +--- +- name: Download the CKAN distribution + get_url: url='{{ ckan_package_url }}' dest=/srv/{{ ckan_deb_file }} force=yes + tags: [ 'ckan', 'ckan_pkg' ] + +- name: Install the CKAN deb package + apt: deb=/srv/{{ ckan_deb_file }} + register: ckan_install + tags: [ 'ckan', 'ckan_pkg' ] + +- name: Create the CKAN user + user: name={{ ckan_shell_user }} home={{ ckan_libdir }} createhome=no shell=/usr/sbin/nologin system=yes + +- name: Configure the CKAN production configuration file excluding the plugins list + ini_file: dest={{ ckan_config_file }} section={{ item.section }} option={{ item.option }} value={{ item.value }} state={{ item.state }} backup=yes + with_items: '{{ ckan_production_ini_opts }}' + notify: Restart CKAN + tags: [ 'ckan', 'ckan_ini' ] + +- name: Install the apache.wsgi + template: src=apache.wsgi.j2 dest={{ ckan_confdir }}/apache.wsgi + notify: Restart CKAN + tags: [ 'ckan', 'apache', 'ckan_pkg' ] + +- name: Create the base directory for the CKAN file storage + file: dest={{ ckan_file_storage_dir }} state=directory owner={{ apache_user }} group={{ ckan_shell_user }} mode=2770 + tags: ckan + +- name: Fix the CKAN harvesting storage permissions + file: dest={{ ckan_file_harvesting_dir }} state=directory owner={{ apache_user }} group={{ ckan_shell_user }} mode=2770 recurse=yes + tags: ckan + +- name: authorization file for the psql command, if the database is on a remote server + template: src=pgpass.j2 dest=/root/.pgpass owner=root mode=0600 + when: psql_db_host != 'localhost' + tags: [ 'pg_backup', 'postgresql', 'postgres' ] + +- name: Initialize the CKAN database + shell: ckan db init && touch {{ ckan_libdir }}/.ckan_db_initialized + args: + creates: '{{ ckan_libdir }}/.ckan_db_initialized' + when: ckan_init_db_and_solr + tags: ckan + +- name: Initialize the CKAN datastore database + shell: ckan datastore set-permissions | psql --set ON_ERROR_STOP=1 -h {{ psql_db_host }} -U {{ ckan_db_user }} -w {{ ckan_datastore_db_name }} && touch {{ ckan_libdir }}/.ckan_datastore_db_initialized + args: + creates: '{{ ckan_libdir }}/.ckan_datastore_db_initialized' + when: ckan_init_db_and_solr + tags: ckan + +- name: Create the pip cache directory with the right permissions + file: dest={{ ckan_libdir }}/.cache owner={{ ckan_shell_user }} group={{ ckan_shell_user }} state=directory + tags: [ 'ckan', 'ckan_user' ] + +- name: Assign the CKAN virtenv dir to the ckan user + file: dest={{ ckan_virtenv }} recurse=yes owner={{ ckan_shell_user }} group={{ ckan_shell_user }} + tags: [ 'ckan', 'ckan_user', 'ckan_permissions' ] + +- name: Create a log directory for the jobs run by the ckan user + file: dest={{ ckan_logdir }} state=directory owner={{ ckan_shell_user }} group={{ ckan_shell_user }} + tags: [ 'ckan', 'ckan_user' ] + +- name: Job that clears the havesting history + block: + - name: Install a script that find the source ids and cleans up the history for each of them + template: src=clear_harvesting_history.sh.j2 dest=/usr/local/bin/ckan_clear_harvesting_history owner=root group=ckan mode='0750' + + - name: Cron job that clears the harvesting history + cron: + cron_file: 'clear_ckan_harvesting_history' + disabled: no + job: "/usr/local/bin/ckan_clear_harvesting_history > /var/log/ckan/clear-harvesting-history.log 2>&1" + special_time: "{{ ckan_clear_harvesting_freq }}" + user: '{{ ckan_shell_user }}' + name: Clear the CKAN harvesting history + state: present + + when: ckan_clear_harvesting_enabled + tags: [ 'ckan', 'ckan_d4s', 'ckan_clear_harvesting' ] + +- name: Job that clears the havesting history + block: + - name: Remove the script that clears the harvesting history + file: dest=/usr/local/bin/ckan_clear_harvesting_history state=absent + + - name: Remove the cron job that clears the harvesting history + cron: + cron_file: 'clear_ckan_harvesting_history' + disabled: no + job: "/usr/local/bin/ckan_clear_harvesting_history > /var/log/ckan/clear-harvesting-history.log 2>&1" + special_time: "{{ ckan_clear_harvesting_freq }}" + user: '{{ ckan_shell_user }}' + name: Clear the CKAN harvesting history + state: absent + + when: not ckan_clear_harvesting_enabled + tags: [ 'ckan', 'ckan_d4s', 'ckan_clear_harvesting' ] + diff --git a/tasks/enable-ckan.yml b/tasks/enable-ckan.yml new file mode 100644 index 0000000..19b3ab5 --- /dev/null +++ b/tasks/enable-ckan.yml @@ -0,0 +1,11 @@ +--- +- name: Restart apache + service: name=apache2 state=restarted enabled=yes + when: ckan_install is changed + tags: ckan + +- name: Restart nginx + service: name=nginx state=restarted enabled=yes + when: ckan_install is changed + tags: ckan + diff --git a/tasks/main.yml b/tasks/main.yml index 53c6cae..cb5f8a9 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,2 +1,7 @@ --- -# tasks file for ansible-role-template \ No newline at end of file +- import_tasks: ckan.yml +- import_tasks: ckan-plugins.yml +- import_tasks: ckan-memcache.yml +- import_tasks: ckan-config.yml +- import_tasks: enable-ckan.yml +- import_tasks: supervisor.yml diff --git a/tasks/supervisor.yml b/tasks/supervisor.yml new file mode 100644 index 0000000..d40b3db --- /dev/null +++ b/tasks/supervisor.yml @@ -0,0 +1,18 @@ +--- +- name: Install the supervisor daemon needed to automate the gather and fetch operations + apt: pkg={{ ckan_gather_fetch_pkgs }} state=present + tags: [ 'ckan', 'ckan_harvest' ] + +- name: Install the gather and fetch supervisor configuration + template: src=ckan_harvesting.conf.j2 dest=/etc/supervisor/conf.d/ckan_harvesting.conf owner=root group=root mode=0644 + notify: Reconfigure the supervisor daemon + tags: [ 'ckan', 'ckan_harvest' ] + +- name: Install a cron job that run the harvesters + cron: name="CKAN harvester" minute="0" job="{{ ckan_virtenv }}/bin/paster --plugin=ckanext-harvest harvester run --config={{ ckan_config_file }} > {{ ckan_logdir }}/harvester_run.log 2>&1" user={{ ckan_shell_user }} + when: ckan_harvester_run + tags: [ 'ckan', 'ckan_harvest', 'ckan_harvest_cron' ] + +- name: Ensure that supervisord is running and enabled + service: name=supervisor state=started enabled=yes + tags: [ 'ckan', 'ckan_harvest' ] diff --git a/templates/apache.wsgi.j2 b/templates/apache.wsgi.j2 new file mode 100644 index 0000000..4418d33 --- /dev/null +++ b/templates/apache.wsgi.j2 @@ -0,0 +1,14 @@ +import os +activate_this = os.path.join('/usr/lib/ckan/default/bin/activate_this.py') +execfile(activate_this, dict(__file__=activate_this)) + +from paste.deploy import loadapp + +config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'production.ini') +from paste.script.util.logging_config import fileConfig +fileConfig(config_filepath) +_application = loadapp('config:%s' % config_filepath) + +def application(environ, start_response): + environ['wsgi.url_scheme'] = environ.get('HTTP_X_URL_SCHEME', 'http') + return _application(environ, start_response) diff --git a/templates/ckan_harvesting.conf.j2 b/templates/ckan_harvesting.conf.j2 new file mode 100644 index 0000000..0cb7708 --- /dev/null +++ b/templates/ckan_harvesting.conf.j2 @@ -0,0 +1,31 @@ +; =============================== +; ckan harvester +; =============================== + +[program:ckan_gather_consumer] + +command={{ ckan_virtenv }}/bin/paster --plugin=ckanext-harvest harvester gather_consumer --config={{ ckan_config_file }} + +; user that owns virtual environment. +user={{ ckan_shell_user }} + +numprocs=1 +stdout_logfile={{ ckan_logdir }}/gather_consumer.log +stderr_logfile={{ ckan_logdir }}/gather_consumer.log +autostart=true +autorestart=true +startsecs=10 + +[program:ckan_fetch_consumer] + +command={{ ckan_virtenv }}/bin/paster --plugin=ckanext-harvest harvester fetch_consumer --config={{ ckan_config_file }} + +; user that owns virtual environment. +user={{ ckan_shell_user }} + +numprocs=1 +stdout_logfile={{ ckan_logdir }}/fetch_consumer.log +stderr_logfile={{ ckan_logdir }}/fetch_consumer.log +autostart=true +autorestart=true +startsecs=10 diff --git a/templates/clear_harvesting_history.sh.j2 b/templates/clear_harvesting_history.sh.j2 new file mode 100644 index 0000000..ebe1aed --- /dev/null +++ b/templates/clear_harvesting_history.sh.j2 @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ "$( id -u -n )" != "{{ ckan_shell_user }}" ] ; then + logger "ckan_clear_harvesting_history: must be executed by the ckan user" + exit 1 +fi +# shellcheck disable=SC1091 +source "/usr/lib/ckan/default/bin/activate" + +ckan_source_ids=$( paster --plugin=ckanext-harvest harvester sources --config={{ ckan_config_file }} | grep -E "^Source\ id:" | awk '{ print $3 }' ) + +for id in $ckan_source_ids ; do + echo "----------" + echo "Source id: ${id}" + echo "----------" + paster --plugin=ckanext-harvest harvester clearsource_history "${id}" --config={{ ckan_config_file }} +done + +exit 0 diff --git a/templates/pgpass.j2 b/templates/pgpass.j2 new file mode 100644 index 0000000..1586807 --- /dev/null +++ b/templates/pgpass.j2 @@ -0,0 +1,8 @@ +# Loop psql_db_data to add multiple databases +{% if psql_db_data is defined %} +{% for db in psql_db_data %} +{%if db.pwd is defined %} +{{ psql_db_host }}:{{ psql_db_port }}:{{ db.name }}:{{ db.user }}:{{ db.pwd }} +{% endif %} +{% endfor %} +{% endif %} diff --git a/templates/tracker_update.sh.j2 b/templates/tracker_update.sh.j2 new file mode 100644 index 0000000..fa93312 --- /dev/null +++ b/templates/tracker_update.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +LOCK_DIR={{ ckan_logdir }} +LOCK_FILE=$LOCK_DIR/.index_rebuild.lock +. {{ ckan_virtenv }}/bin/activate + +if [ -f $LOCK_FILE ] ; then + echo 'A lock file is present, exiting' + exit 2 +fi + +echo "cron pid: ${$}" > $LOCK_FILE +paster --plugin=ckan tracking update -c {{ ckan_config_file }} +paster --plugin=ckan search-index rebuild -r -c {{ ckan_config_file }} + +rm -f $LOCK_FILE +exit 0