From 38e2cde3563afa215f6f7440d9f7c786bc2d5638 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Wed, 9 May 2018 18:19:43 +0200 Subject: [PATCH] Make the mongodb-org-3.2 more generic (it should be renamed too) --- mongodb-org-3.2/defaults/main.yml | 8 + mongodb-org-3.2/files/mongodb.py | 496 ------------------- mongodb-org-3.2/tasks/ganglia-plugin.yml | 16 - mongodb-org-3.2/tasks/main.yml | 4 +- mongodb-org-3.2/tasks/mongodb.yml | 13 +- mongodb-org-3.2/templates/mongod-3.6.conf.j2 | 44 ++ 6 files changed, 58 insertions(+), 523 deletions(-) delete mode 100755 mongodb-org-3.2/files/mongodb.py delete mode 100644 mongodb-org-3.2/tasks/ganglia-plugin.yml create mode 100644 mongodb-org-3.2/templates/mongod-3.6.conf.j2 diff --git a/mongodb-org-3.2/defaults/main.yml b/mongodb-org-3.2/defaults/main.yml index b4be8fc..4606da1 100644 --- a/mongodb-org-3.2/defaults/main.yml +++ b/mongodb-org-3.2/defaults/main.yml @@ -1,9 +1,17 @@ --- mongodb_install_from_external_repo: True +mongodb_repo_key: 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 +mongodb_apt_repository: "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/{{ mongodb_version }} multiverse" + +mongodb_old_repositories: + - "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/3.0 multiverse" + - "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" + mongodb_install_packages: True mongodb_install_server: True mongodb_install_conf: True mongodb_upgrade_from_older_version: False +mongodb_version: 3.2 # Set to 'latest' if you want to get the latest available package mongodb_pkg_state: present mongodb_server_pkgs: diff --git a/mongodb-org-3.2/files/mongodb.py b/mongodb-org-3.2/files/mongodb.py deleted file mode 100755 index 7ef34be..0000000 --- a/mongodb-org-3.2/files/mongodb.py +++ /dev/null @@ -1,496 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# MongoDB gmond module for Ganglia -# -# Copyright (C) 2011 by Michael T. Conigliaro . -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# - -import json -import os -import re -import socket -import string -import time -import copy - -NAME_PREFIX = 'mongodb_' -PARAMS = { - 'server_status' : '~/mongodb-osx-x86_64-1.8.1/bin/mongo --host mongodb04.example.com --port 27018 --quiet --eval "printjson(db.serverStatus())"', - 'rs_status' : '~/mongodb-osx-x86_64-1.8.1/bin/mongo --host mongodb04.example.com --port 27018 --quiet --eval "printjson(rs.status())"' -} -METRICS = { - 'time' : 0, - 'data' : {} -} -LAST_METRICS = copy.deepcopy(METRICS) -METRICS_CACHE_TTL = 3 - - -def flatten(d, pre = '', sep = '_'): - """Flatten a dict (i.e. dict['a']['b']['c'] => dict['a_b_c'])""" - - new_d = {} - for k,v in d.items(): - if type(v) == dict: - new_d.update(flatten(d[k], '%s%s%s' % (pre, k, sep))) - else: - new_d['%s%s' % (pre, k)] = v - return new_d - - -def get_metrics(): - """Return all metrics""" - - global METRICS, LAST_METRICS - - if (time.time() - METRICS['time']) > METRICS_CACHE_TTL: - - metrics = {} - for status_type in PARAMS.keys(): - - # get raw metric data - io = os.popen(PARAMS[status_type]) - - # clean up - metrics_str = ''.join(io.readlines()).strip() # convert to string - metrics_str = re.sub('\w+\((.*)\)', r"\1", metrics_str) # remove functions - - # convert to flattened dict - try: - if status_type == 'server_status': - metrics.update(flatten(json.loads(metrics_str))) - else: - metrics.update(flatten(json.loads(metrics_str), pre='%s_' % status_type)) - except ValueError: - metrics = {} - - # update cache - LAST_METRICS = copy.deepcopy(METRICS) - METRICS = { - 'time': time.time(), - 'data': metrics - } - - return [METRICS, LAST_METRICS] - - -def get_value(name): - """Return a value for the requested metric""" - - # get metrics - metrics = get_metrics()[0] - - # get value - name = name[len(NAME_PREFIX):] # remove prefix from name - try: - result = metrics['data'][name] - except StandardError: - result = 0 - - return result - - -def get_rate(name): - """Return change over time for the requested metric""" - - # get metrics - [curr_metrics, last_metrics] = get_metrics() - - # get rate - name = name[len(NAME_PREFIX):] # remove prefix from name - - try: - rate = float(curr_metrics['data'][name] - last_metrics['data'][name]) / \ - float(curr_metrics['time'] - last_metrics['time']) - if rate < 0: - rate = float(0) - except StandardError: - rate = float(0) - - return rate - - -def get_opcounter_rate(name): - """Return change over time for an opcounter metric""" - - master_rate = get_rate(name) - repl_rate = get_rate(name.replace('opcounters_', 'opcountersRepl_')) - - return master_rate + repl_rate - - -def get_globalLock_ratio(name): - """Return the global lock ratio""" - - try: - result = get_rate(NAME_PREFIX + 'globalLock_lockTime') / \ - get_rate(NAME_PREFIX + 'globalLock_totalTime') * 100 - except ZeroDivisionError: - result = 0 - - return result - - -def get_indexCounters_btree_miss_ratio(name): - """Return the btree miss ratio""" - - try: - result = get_rate(NAME_PREFIX + 'indexCounters_btree_misses') / \ - get_rate(NAME_PREFIX + 'indexCounters_btree_accesses') * 100 - except ZeroDivisionError: - result = 0 - - return result - - -def get_connections_current_ratio(name): - """Return the percentage of connections used""" - - try: - result = float(get_value(NAME_PREFIX + 'connections_current')) / \ - float(get_value(NAME_PREFIX + 'connections_available')) * 100 - except ZeroDivisionError: - result = 0 - - return result - - -def get_slave_delay(name): - """Return the replica set slave delay""" - - # get metrics - metrics = get_metrics()[0] - - # no point checking my optime if i'm not replicating - if 'rs_status_myState' not in metrics['data'] or metrics['data']['rs_status_myState'] != 2: - result = 0 - - # compare my optime with the master's - else: - master = {} - slave = {} - try: - for member in metrics['data']['rs_status_members']: - if member['state'] == 1: - master = member - if member['name'].split(':')[0] == socket.getfqdn(): - slave = member - result = max(0, master['optime']['t'] - slave['optime']['t']) / 1000 - except KeyError: - result = 0 - - return result - - -def get_asserts_total_rate(name): - """Return the total number of asserts per second""" - - return float(reduce(lambda memo,obj: memo + get_rate('%sasserts_%s' % (NAME_PREFIX, obj)), - ['regular', 'warning', 'msg', 'user', 'rollovers'], 0)) - - -def metric_init(lparams): - """Initialize metric descriptors""" - - global PARAMS - - # set parameters - for key in lparams: - PARAMS[key] = lparams[key] - - # define descriptors - time_max = 60 - groups = 'mongodb' - descriptors = [ - { - 'name': NAME_PREFIX + 'opcounters_insert', - 'call_back': get_opcounter_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Inserts/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Inserts', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'opcounters_query', - 'call_back': get_opcounter_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Queries/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Queries', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'opcounters_update', - 'call_back': get_opcounter_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Updates/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Updates', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'opcounters_delete', - 'call_back': get_opcounter_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Deletes/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Deletes', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'opcounters_getmore', - 'call_back': get_opcounter_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Getmores/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Getmores', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'opcounters_command', - 'call_back': get_opcounter_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Commands/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Commands', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'backgroundFlushing_flushes', - 'call_back': get_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Flushes/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Flushes', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'mem_mapped', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'MB', - 'slope': 'both', - 'format': '%u', - 'description': 'Memory-mapped Data', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'mem_virtual', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'MB', - 'slope': 'both', - 'format': '%u', - 'description': 'Process Virtual Size', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'mem_resident', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'MB', - 'slope': 'both', - 'format': '%u', - 'description': 'Process Resident Size', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'extra_info_page_faults', - 'call_back': get_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Faults/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Page Faults', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'globalLock_ratio', - 'call_back': get_globalLock_ratio, - 'time_max': time_max, - 'value_type': 'float', - 'units': '%', - 'slope': 'both', - 'format': '%f', - 'description': 'Global Write Lock Ratio', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'indexCounters_btree_miss_ratio', - 'call_back': get_indexCounters_btree_miss_ratio, - 'time_max': time_max, - 'value_type': 'float', - 'units': '%', - 'slope': 'both', - 'format': '%f', - 'description': 'BTree Page Miss Ratio', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'globalLock_currentQueue_total', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Operations', - 'slope': 'both', - 'format': '%u', - 'description': 'Total Operations Waiting for Lock', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'globalLock_currentQueue_readers', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Operations', - 'slope': 'both', - 'format': '%u', - 'description': 'Readers Waiting for Lock', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'globalLock_currentQueue_writers', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Operations', - 'slope': 'both', - 'format': '%u', - 'description': 'Writers Waiting for Lock', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'globalLock_activeClients_total', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Clients', - 'slope': 'both', - 'format': '%u', - 'description': 'Total Active Clients', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'globalLock_activeClients_readers', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Clients', - 'slope': 'both', - 'format': '%u', - 'description': 'Active Readers', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'globalLock_activeClients_writers', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Clients', - 'slope': 'both', - 'format': '%u', - 'description': 'Active Writers', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'connections_current', - 'call_back': get_value, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Connections', - 'slope': 'both', - 'format': '%u', - 'description': 'Open Connections', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'connections_current_ratio', - 'call_back': get_connections_current_ratio, - 'time_max': time_max, - 'value_type': 'float', - 'units': '%', - 'slope': 'both', - 'format': '%f', - 'description': 'Percentage of Connections Used', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'slave_delay', - 'call_back': get_slave_delay, - 'time_max': time_max, - 'value_type': 'uint', - 'units': 'Seconds', - 'slope': 'both', - 'format': '%u', - 'description': 'Replica Set Slave Delay', - 'groups': groups - }, - { - 'name': NAME_PREFIX + 'asserts_total', - 'call_back': get_asserts_total_rate, - 'time_max': time_max, - 'value_type': 'float', - 'units': 'Asserts/Sec', - 'slope': 'both', - 'format': '%f', - 'description': 'Asserts', - 'groups': groups - } - ] - - return descriptors - - -def metric_cleanup(): - """Cleanup""" - - pass - - -# the following code is for debugging and testing -if __name__ == '__main__': - descriptors = metric_init(PARAMS) - while True: - for d in descriptors: - print (('%s = %s') % (d['name'], d['format'])) % (d['call_back'](d['name'])) - print '' - time.sleep(METRICS_CACHE_TTL) diff --git a/mongodb-org-3.2/tasks/ganglia-plugin.yml b/mongodb-org-3.2/tasks/ganglia-plugin.yml deleted file mode 100644 index 0f0d777..0000000 --- a/mongodb-org-3.2/tasks/ganglia-plugin.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# -# The ganglia plugin comes from https://github.com/ganglia/gmond_python_modules -# -- name: Install the ganglia plugin for MongoDB - copy: src=mongodb.py dest=/usr/lib/ganglia/python_modules/mongodb.py owner=root group=root mode=0444 - when: ganglia_enabled - notify: Restart ganglia monitor - tags: [ 'ganglia', 'mongodb' ] - -- name: Distribute the ganglia (gmond) configuration for the MongoDB plugin - template: src=mongodb.pyconf.j2 dest=/etc/ganglia/conf.d/mongodb.pyconf owner=root group=root mode=0444 - when: ganglia_enabled - notify: Restart ganglia monitor - tags: [ 'ganglia', 'mongodb' ] - diff --git a/mongodb-org-3.2/tasks/main.yml b/mongodb-org-3.2/tasks/main.yml index c3ad3de..c574a39 100644 --- a/mongodb-org-3.2/tasks/main.yml +++ b/mongodb-org-3.2/tasks/main.yml @@ -1,4 +1,2 @@ --- -- include: mongodb.yml -- include: ganglia-plugin.yml - when: ganglia_enabled +- import_tasks: mongodb.yml diff --git a/mongodb-org-3.2/tasks/mongodb.yml b/mongodb-org-3.2/tasks/mongodb.yml index 8235733..35da606 100644 --- a/mongodb-org-3.2/tasks/mongodb.yml +++ b/mongodb-org-3.2/tasks/mongodb.yml @@ -18,26 +18,23 @@ - block: - name: Install the mongodb apt key - #apt_key: id=7F0CEB10 state=present - raw: apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 + apt_key: keyserver="hkp://keyserver.ubuntu.com:80" id={{ mongodb_repo_key }} state=present when: mongodb_install_from_external_repo - name: Remove the old mongo apt repositories apt_repository: repo="{{ item }}" state=absent - with_items: - - "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/3.0 multiverse" - - "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" + with_items: '{{ mongodb_old_repositories }}' when: mongodb_upgrade_from_older_version - name: Install the mongodb repository - apt_repository: repo="deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/3.2 multiverse" update_cache=yes state=present + apt_repository: repo="{{ mongodb_apt_repository }}" update_cache=yes state=present when: mongodb_install_from_external_repo tags: [ 'mongodb', 'mongodb_client', 'mongodb_repo' ] - block: - - name: Install the mongodb 3.2 configuration - template: src=mongod-3.2.conf.j2 dest=/etc/mongod.conf owner=root group=root mode=0444 + - name: Install the mongodb configuration + template: src=mongod-{{ mongodb_version }}.conf.j2 dest=/etc/mongod.conf owner=root group=root mode=0444 backup=yes when: mongodb_install_conf notify: Restart mongodb diff --git a/mongodb-org-3.2/templates/mongod-3.6.conf.j2 b/mongodb-org-3.2/templates/mongod-3.6.conf.j2 new file mode 100644 index 0000000..b709965 --- /dev/null +++ b/mongodb-org-3.2/templates/mongod-3.6.conf.j2 @@ -0,0 +1,44 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +# Where and how to store data. +storage: + dbPath: {{ mongodb_dbpath }} + journal: + enabled: true + directoryPerDB: {{ mongodb_directoryperdb }} + engine: {{ mongodb_storage_engine }} +# mmapv1: +# wiredTiger: + +# where to write logging data. +systemLog: + destination: {{ mongodb_systemlog_destination }} + path: {{ mongodb_logpath }} +{% if not mongodb_systemlog_external_logrotate %} + logRotate: rename + logAppend: false +{% else %} + logRotate: reopen + logAppend: true +{% endif %} +# network interfaces +net: + port: {{ mongodb_tcp_port }} + bindIp: {{ mongo_bind_ip }} + +#processManagement: + +{%if mongodb_cluster_enabled %} +security: + keyFile: /data/mongo_home/dev-d4science-keyfile + +replication: + oplogSizeMB: 2000 + replSetName: {{ mongodb_replicaset }} +{% endif %} + +#sharding: +