Polynote role.

README.md

@@ -1,31 +1,20 @@
 Role Name
 =========
 
-A brief description of the role goes here.
-
-Requirements
-------------
-
-Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+A role that installs the Polynote notebook, <https://polynote.org>
 
 Role Variables
 --------------
 
-A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+The most important variables are listed below:
+
+``` yaml
+```
 
 Dependencies
 ------------
 
-A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
-
-Example Playbook
-----------------
-
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
-    - hosts: servers
-      roles:
-         - { role: username.rolename, x: 42 }
+* openjdk
 
 License
 -------
@@ -35,4 +24,4 @@ EUPL-1.2
 Author Information
 ------------------
 
-An optional section for the role authors to include contact information, or a website (HTML is not allowed).
+Andrea Dell'Amico, <andrea.dellamico@isti.cnr.it>

defaults/main.yml

@@ -1,2 +1,13 @@
 ---
-# defaults file for ansible-role-template
+polynote_version: '0.3.11'
+polynote_with_scala_2_12: False
+polynote_is_hadoop_node: True
+polynote_letsencrypt_ssl: False
+polynote_base_uri: '/'
+polynote_listen_ip: '127.0.0.1'
+polynote_listen_port: '8192'
+polynote_data_basedir: '/srv/polynote'
+polynote_storage_location: '{{ polynote_data_basedir }}/data'
+polynote_cache_dir: '{{ polynote_data_basedir }}/tmp'
+polynote_logrotate_maxfilesize: '1G'
+polynote_logrotate_retention: '7'

handlers/main.yml

@@ -1,2 +1,4 @@
 ---
-# handlers file for ansible-role-template
+- name: Restart polynote
+  service: name=polynote state=restarted
+

meta/main.yml

@@ -1,61 +1,48 @@
 galaxy_info:
-  author: your name
-  description: your description
+  author: Andrea Dell'Amico
+  description: Systems Architect
   company: ISTI-CNR
 
-  # If the issue tracker for your role is not on github, uncomment the
-  # next line and provide a value
   issue_tracker_url: https://redmine-s2i2s.isti.cnr.it/projects/provisioning
 
-  # Some suggested licenses:
-  # - BSD (default)
-  # - MIT
-  # - GPLv2
-  # - GPLv3
-  # - Apache
-  # - CC-BY
-  license: EUPL-1.2
+  license: EUPL 1.2+
 
   min_ansible_version: 2.8
 
-  # If this a Container Enabled role, provide the minimum Ansible Container version.
-  # min_ansible_container_version:
-
-  # Optionally specify the branch Galaxy will use when accessing the GitHub
-  # repo for this role. During role install, if no tags are available,
-  # Galaxy will use this branch. During import Galaxy will access files on
-  # this branch. If Travis integration is configured, only notifications for this
-  # branch will be accepted. Otherwise, in all cases, the repo's default branch
-  # (usually master) will be used.
-  #github_branch:
-
-  #
-  # Provide a list of supported platforms, and for each platform a list of versions.
-  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
   # To view available platforms and versions (or releases), visit:
   # https://galaxy.ansible.com/api/v1/platforms/
   #
-  # platforms:
-  # - name: Fedora
-  #   versions:
-  #   - all
-  #   - 25
-  # - name: SomePlatform
-  #   versions:
-  #   - all
-  #   - 1.0
-  #   - 7
-  #   - 99.99
+  platforms:
+    - name: Ubuntu
+      versions:
+        - bionic
+    - name: EL
+      versions:
+        - 7
+        - 8
 
-  galaxy_tags: []
-    # List tags for your role here, one per line. A tag is a keyword that describes
-    # and categorizes the role. Users find roles by searching for tags. Be sure to
-    # remove the '[]' above, if you add tags to this list.
-    #
-    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
-    #       Maximum 20 tags per role.
+  galaxy_tags:
+    - polynote
+    - notebook
+    - spark
+    - scala
 
-dependencies: []
-  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
-  # if you add dependencies to this list.
+dependencies:
+  - src: git+https://gitea-s2i2s.isti.cnr.it/ISTI-ansible-roles/ansible-role-openjdk.git
+    version: master
+    name: openjdk
+    state: latest
+  - src: git+https://gitea-s2i2s.isti.cnr.it/ISTI-ansible-roles/ansible-role-python3-environment.git
+    version: master
+    name: python3-environment
+    state: latest
+  - src: git+https://gitea-s2i2s.isti.cnr.it/ISTI-ansible-roles/ansible-role-python3-environment.git
+    version: master
+    name: python3-environment
+    state: latest
+  - src: git+https://gitea-s2i2s.isti.cnr.it/ISTI-ansible-roles/ansible-role-hadoop-cdh.git
+    version: master
+    name: hadoop-cdh
+    state: latest
+    #when: polynote_is_hadoop_node
 

tasks/main.yml

@@ -1,2 +1,59 @@
 ---
-# tasks file for ansible-role-template
+- name: Polynote distribution
+  block:
+  - name: Create a user for the polynote service
+    user: name={{ polynote_service_user }} home={{ polynote_data_basedir }} createhome=no shell=/usr/sbin/nologin system=yes
+
+  - name: Create the polynote base directory
+    file: dest={{ polynote_data_basedir }} owner=root group=root mode='0755'
+    
+  - name: Create the polynote data directories
+    file: dest={{ item }} owner={{ polynote_service_user }} group={{ polynote_service_user }} mode='0750'
+    with_items:
+      - '{{ polynote_storage_location }}'
+      - '{{ polynote_cache_dir }}'
+
+  - name: Download the polynote distribution
+    unarchive: remote_src=yes src={{ polynote_distribution_url }} dest={{ polynote_install_base_dir }} owner=root group=root
+    args:
+      creates: '{{ polynote_install_dir }}/polynote.jar'
+    when: not polynote_with_scala_2_12
+
+  - name: Download the polynote distribution with scala 2.12
+    unarchive: remote_src=yes src={{ polynote_distribution_scala_2_12_url }} dest={{ polynote_install_base_dir }} owner=root group=root
+    args:
+      creates: '{{ polynote_install_dir }}/polynote.jar'
+    when: polynote_with_scala_2_12
+
+  tags: [ 'polynote', 'polynote_distribution' ]
+
+- name: Polynote configuration
+  block:
+  - name: Install the polynote configuration file
+    template: src=configuration.py.j2 dest={{ polynote_install_dir }}/configuration.py owner=root group={{ polynote_service_user }} mode='0440'
+    notify: Restart polynote
+
+  tags: [ 'polynote', 'polynote_configuration' ]
+
+- name: Manage the Polynote service
+  block:
+  - name: Install the rsyslog configuration for polynote
+    template: src=polynote-rsyslog.conf.j2 dest=/etc/rsyslog.d/50-polynote.conf mode='0644'
+    notify: Restart rsyslog
+
+  - name: Install the logrotate configuration for polynote
+    template: src=polynote.logrotate.j2 dest=/etc/logrotate.d/polynote mode='0644'
+
+  - name: Install the polynote systemd unit
+    template: src=polynote.service.j2 dest=/etc/systemd/system/polynote.service owner=root group=root mode='0644'
+    register: reload_systemd
+
+  - name: Reload the systemd configuration
+    systemd:
+      daemon_reload: yes
+    when: reload_systemd is changed
+
+  - name: Ensure that polynote is running and enabled
+    service: name=polynote state=started enabled=yes
+
+  tags: [ 'polynote', 'polynote_configuration' ]

templates/config.yml.j2

@@ -0,0 +1,162 @@
+######### Polynote Configuration Template ##############################################################################
+# To use, copy this template and name it `config.yml`
+#
+#    cp ./config-template.yml ./config.yml
+#
+# Now, uncomment the section of the config that you'd like to change.
+#
+########################################################################################################################
+
+########## Server-level Configuration ################################
+###
+### These configurations are applied immediately when Polynote starts.
+###
+######################################################################
+
+###############################################################################
+# The host and port can be set by uncommenting and editing the following lines.
+###############################################################################
+
+listen:
+  host: {{ polynote_listen_ip }}
+  port: {{ polynote_listen_port }}
+
+
+############################################################################################
+# Storage-related configuration. Locations are all relative to Polynote's working directory.
+############################################################################################
+
+storage:
+#  # The base directory Polynote should use when looking for notebooks. This location can be absolute or
+#  # relative to Polynote's working directory.
+  dir: {{ polynote_storage_location }}
+#  # These are additional mount points. Folders specified here will be 'mounted' into the base directory
+#  # and visible by their keys - so `foo: dir: bar` will show a folder `foo` in the UI that shows the contents of the
+#  # file system under `bar/`
+#  mounts:
+#    examples:
+#      dir: examples
+#    team_notebooks:
+#      dir: /shared/notebooks
+#  # The location Polynote puts various caches, such as virtual environments created for your notebooks.
+  cache: {{ polynote_cache_dir }}
+
+##########################################################
+# Configure the way Polynote runs Kernels and Interpreters
+##########################################################
+
+behavior:
+  dependency_isolation: false
+  kernel_isolation: always
+  shared_packages:
+    - com.esoteric.kryo
+
+########## Notebook Creation Configuration #######################################################
+###
+### These configurations are applied to *new notebooks* only.
+### When a *new notebook* is created, its Notebook Configuration is copied from this configuration.
+### They do *not* affect existing notebooks.
+###
+###################################################################################################
+
+############################################################################################
+# Default repositories. Uncommenting the following lines would add four default repositories
+# which are inherited by new notebooks.
+############################################################################################
+
+#repositories:
+#  - ivy:
+#      base: https://my-artifacts.org/artifacts/
+#  - ivy:
+#      base: https://my-custom-ivy-repo.org/artifacts/
+#      artifact_pattern: [orgPath]/[module](_[scalaVersion])(_[sbtVersion])/[revision]/[artifact]-[revision](-[classifier]).[ext]
+#      metadata_pattern: [orgPath]/[module](_[scalaVersion])(_[sbtVersion])/[revision]/[module](_[scalaVersion])(_[sbtVersion])-[revision]-ivy.xml
+#      changing: true
+#  - maven:
+#      base: http://central.maven.org/maven2/
+#  - maven:
+#      base: http://oss.sonatype.org/content/repositories/snapshots
+#      changing: true
+
+############################################################################################
+# Default dependencies. Uncommenting the following lines would add some default dependencies
+# which are inherited by new notebooks.
+############################################################################################
+
+#dependencies:
+#  scala:
+#    - org.typelevel:cats-core_2.11:1.6.0
+#    - com.mycompany:my-library:jar:all:1.0.0
+#  python:
+#    - requests
+#    - urllib3==1.25.3
+
+########################################################################################
+# Default exclusions. Uncommenting the following lines would add some default exclusions
+# which are inherited by new notebooks.
+########################################################################################
+
+#exclusions:
+#  - com.google.guava:guava  # spark, update your guava already!!!
+
+########## Notebook Runtime Configuration #########################################################################
+###
+### These configurations are applied to *all notebooks*.
+### When a *new notebook* is created, its Notebook Configuration is copied from this configuration.
+###
+### When an existing notebook is *run*, these configurations are applied at runtime. Values present in the Notebook
+### Configuration take precedence over values here. The Notebook Configuration is *not* modified.
+###
+### **Note** that this means they can affect reproducibility of your notebooks when you share them with others!
+###
+###################################################################################################################
+
+###################################################################################################################
+# Spark config params can be set by uncommenting and editing the following lines.
+# To add additional spark configuration properties, add them as keys under `properties`.
+# To set additional arguments to `spark-submit`, you can use the `spark_submit_args` property.
+#
+# By default, not all jars in SPARK_DIST_CLASSPATH are available to the Polynote compiler (i.e., to user-code
+# inside cells). To use these jars (for example, if you want to use Hadoop APIs in your notebook without adding
+# a Hadoop dependency), set the `dist_classpath_filter` to a valid regular expression that selects which JARs
+# you want to use (there can be a huge number of JARs in SPARK_DIST_CLASSPATH, so it would be burdensome to
+# include all of them by default.)
+#
+#################################################################################
+
+#spark:
+#  properties:
+#    spark.driver.memory: 28g
+#    spark.executor.memory: 60g
+#    spark.driver.userClasspathFirst: true
+#    spark.executor.userClasspathFirst: true
+#  spark_submit_args: "--some --arguments"
+#  dist_classpath_filter: hadoop.*\.jar
+
+
+########## Front-end Configuration ################################################################################
+###
+### These configurations affect the behavior of the front-end. You can modify the behavior by uncommenting the
+### following lines and editing the properties.
+###
+###################################################################################################################
+
+ui:
+#  # The URI relative to the server host where Polynote is mounted. You can edit this if mounting Polynote at a
+#  # different location behind a reverse proxy. This value is placed in the <base> tag.
+  base_uri: {{ polynote_base_uri }}
+
+############################################################################################
+# Credentials. This list contains the list of credentials used to access the repositories
+############################################################################################
+
+#credentials:
+#  coursier:
+#    path: ~/.config/coursier/credentials.properties
+
+#############################################################################################################
+# Environment variables. This map gets merged with the notebook config's environment variable map at runtime.
+#############################################################################################################
+
+#env:
+#  foo: bar

templates/polynote-rsyslog.conf.j2

@@ -0,0 +1,2 @@
+if $programname == 'polynote' then {{ polynote_log_dir }}/polynote.log
+&~

templates/polynote.logrotate.j2

@@ -0,0 +1,10 @@
+{{ polynote_log_dir }}/polynote.log {
+    rotate {{ polynote_logrotate_retention }}
+    maxsize {{ polynote_logrotate_maxfilesize }}
+    missingok
+    notifempty
+    compress
+    daily
+    copytruncate
+    create 644 {{ polynote_service_user }} {{ polynote_service_user }}
+}

templates/polynote.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Polynote
+After=syslog.target network.target
+
+[Service]
+User={{ polynote_service_user }}
+{% for env in polynote_startup_env %}
+Environment="{{ env }}"
+{% endfor %}
+ExecStart={{ polynote_install_dir }}/polynote.py
+Restart=on-failure
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=polynote
+
+[Install]
+WantedBy=multi-user.target

vars/main.yml

@@ -1,2 +1,29 @@
 ---
-# vars file for ansible-role-template
+polynote_distribution_url: 'https://github.com/polynote/polynote/releases/download/{{ polynote_version }}/polynote-dist.tar.gz'
+polynote_distribution_scala_2_12_url: 'https://github.com/polynote/polynote/releases/download/{{ polynote_version }}/polynote-dist-2.12.tar.gz'
+
+polynote_service_user: polynote
+polynote_install_base_dir: /opt
+polynote_install_dir: '{{ polynote_install_base_dir }}/polynote'
+polynote_log_dir: '/var/log/polynote'
+polynote_startup_env:
+  - 'JAVA_HOME={{ jdk_java_home }}'
+  - 'YARN_CONF_DIR=/opt/cloudera/parcels/CDH/lib/hadoop/etc/hadoop'
+  - 'SPARK_HOME=/opt/cloudera/parcels/SPARK2/lib/spark2'
+  - 'HADOOP_CONF_DIR=/etc/spark2/conf/yarn-conf'
+
+py3_env_install: True
+
+py3_env_pip_pkgs:
+  - jep
+  - jedi
+  - virtualenv
+  - ipython
+  - nbconvert
+  - jedi
+  - numpy
+  - pandas
+
+py3_env_versioned_pip_pkgs:
+  - { pkg: 'jep', version: '3.9.0' }
+  - { pkg: 'pyspark', version: '2.4.0' }