Support for Hortonworks HDP

PNDA-2445

Author: jeclarke

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## Unreleased
+### Added
+- PNDA-2445: Support for Hortonworks HDP hadoop distro
+
+### Changed
+- PNDA-2965: Rename `cloudera_*` role grains to `hadoop_*`
+
 ## [2.0.0] 2017-05-23
 ### Added
 - PNDA-2375: Isolate PNDA from breaking dependency changes
@@ -9,9 +16,10 @@ All notable changes to this project will be documented in this file.
 - PNDA-2682: review console backend deployment
 - Add a simple jupyter notebook
 - Allow salt mine for all interfaces
+
 ### Changed
 - PNDA-2446: Download java with wget
-- PNDA-2517: If Cloudera setup (cm_setup.py) fails, orchestrate can be re-run and cm_setup.py will attempt to continue from where it completed up to last time. Progress is recorded in /root/.CM_SETUP_SUCCESS which can be edited if manual control is required over the point to continue from.
+- PNDA-2517: If Cloudera setup (`cm_setup.py`) fails, orchestrate can be re-run and `cm_setup.py` will attempt to continue from where it completed up to last time. Progress is recorded in `/root/.CM_SETUP_SUCCESS` which can be edited if manual control is required over the point to continue from.
 - PNDA-2577: Use spur 0.3.20 for cm_setup.py
 - PNDA-2596: Stop ingesting internal PNDA testbot topic
 - PNDA-2672: Explicitly set CM API version number

pillar/flavors/bmstandard.sls

@@ -12,6 +12,9 @@
         "cdh.setup_hadoop": {
             "template_file": "cfg_bmstandard.py"
         },
+        "hdp.setup_hadoop": {
+            "template_file": "cfg_bmstandard.py"
+        },
         "curator": {
             "days_to_keep": 6
         },

pillar/flavors/pico.sls

@@ -15,6 +15,9 @@
         "cdh.setup_hadoop": {
             "template_file": "cfg_pico.py"
         },
+        "hdp.setup_hadoop": {
+            "template_file": "cfg_pico.py"
+        },
         "curator": {
             "days_to_keep": 1
         }

pillar/flavors/standard.sls

@@ -15,6 +15,9 @@
         "cdh.setup_hadoop": {
             "template_file": "cfg_standard.py"
         },
+        "hdp.setup_hadoop": {
+            "template_file": "cfg_standard.py"
+        },
         "curator": {
             "days_to_keep": 6
         }

pillar/packages/RedHat.sls

@@ -1,9 +1,18 @@
 acl:
   package-name: acl
   version: ""
+ambari-agent:
+  package-name: ambari-agent
+  version: "2.5.0.3-7"
+ambari-server:
+  package-name: ambari-server
+  version: "2.5.0.3-7"
 at:
   package-name: at
   version: ""
+bzip2:
+  package-name: bzip2
+  version: ""
 cloudera-manager-agent:
   package-name: cloudera-manager-agent
   version: "5.9.0-1.cm590.p0.249.el7"
@@ -16,6 +25,12 @@ cloudera-manager-server:
 cyrus-sasl-devel:
   package-name: cyrus-sasl-devel
   version: ""
+cyrus-sasl-gssapi:
+  package-name: cyrus-sasl-gssapi
+  version: ""
+cyrus-sasl-plain:
+  package-name: cyrus-sasl-plain
+  version: ""
 g++:
   package-name: gcc-c++
   version: ""
@@ -33,6 +48,9 @@ libssl-dev:
 libffi-dev:
   package-name: libffi-devel
   version:  ""
+libtirpc-devel:
+  package-name: libtirpc-devel
+  version:  ""
 libsasl:
   package-name: libgsasl-devel
   version: ""

pillar/packages/Ubuntu.sls

@@ -1,6 +1,12 @@
 acl:
   package-name: acl
   version: ""
+ambari-agent:
+  package-name: ambari-agent
+  version: "2.5.0.3-7"
+ambari-server:
+  package-name: ambari-server
+  version: "2.5.0.3-7"
 cloudera-manager-agent:
   package-name: cloudera-manager-agent
   version: "5.9.0-1.cm590.p0.249~trusty-cm5"

pillar/services.sls

@@ -58,7 +58,7 @@ gobblin:
   release_version: 0.1.5
 
 console_frontend:
-  release_version: 0.2.0
+  release_version: hdp
 
 console_backend_data_logger:
   release_version: 0.4.0
@@ -72,23 +72,23 @@ console_backend_utils:
   release_version: 0.4.0
 
 deployment_manager:
-  release_version: 0.4.0
+  release_version: hdp
 
 package_repository:
   release_version: 0.3.1
 
 data-service:
-  release_version: 0.2.0
+  release_version: hdp
 
 hdfs_cleaner:
-  release_version: 0.2.0
+  release_version: hdp
 
 platform_testing:
   release_directory: /opt/pnda
-  release_version: 0.3.3
+  release_version: hdp
 
 platformlib:
-  release_version: RELEASE_0.1.3
+  release_version: hdp
   target_directory: "/opt/pnda"
 
 jmxproxy:
@@ -102,4 +102,4 @@ anaconda:
 java:
   version: "jdk-8u131-linux-x64"
   version_name: "jdk1.8.0_131"
-  source_url: "http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz"
+  source_url: "http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz"

salt/_modules/pnda.py

@@ -1,39 +1,75 @@
 import requests
 
-def get_named_service(cm_host, cluster_name, service_name):
-    """ Returns named service for HA Cluster """
-    user_name = __salt__['pillar.get']('admin_login:user')
-    password = __salt__['pillar.get']('admin_login:password')
-    request_url = 'http://%s:7180/api/v11/clusters/%s/services/%s/nameservices' % (cm_host,
-                                                                                   cluster_name,
-                                                                                   service_name)
+def get_name_service():
+    """ Returns name service for HA Cluster """
+    user_name = hadoop_manager_username()
+    password = hadoop_manager_password()
+    request_url = 'http://%s:7180/api/v11/clusters/%s/services/%s/nameservices' % (hadoop_manager_ip(), cluster_name(), 'hdfs01')
     r = requests.get(request_url, auth=(user_name, password))
-    named_service = ""
+    name_service = ""
     if r.status_code == 200:
         response = r.json()
         if 'items' in response:
-            named_service = response['items'][0]['name']
-    return named_service
+            name_service = response['items'][0]['name']
+    return name_service
 
 
 def cluster_name():
     """Returns PNDA cluster name of the minion"""
     cname = __grains__['pnda_cluster']
     return cname
 
-def namenodes_ips():
-    """Returns hadoop name nodes ip addresses"""
-    cm_name = cluster_name()
-    cm_host = cloudera_manager_ip()
-    service_name = 'hdfs01'
-    named_service = get_named_service(cm_host, cm_name, service_name)
-    if named_service:
-        return [named_service]
-    return ip_addresses('cloudera_namenode')
-
-def cloudera_manager_ip():
+def hadoop_manager_username():
+    """Returns username to log into the hadoop cluster manager"""
+    uname = __salt__['pillar.get']('admin_login:user')
+    return uname
+
+def hadoop_manager_password():
+    """Returns password to log into the hadoop cluster manager"""
+    pwrd = __salt__['pillar.get']('admin_login:password')
+    return pwrd
+
+def hadoop_distro():
+    """Returns hadoop distro"""
+    distro = __salt__['pillar.get']('hadoop.distro')
+    return distro
+
+def ambari_request(uri):
+    full_uri = 'http://%s:8080/api/v1%s' % (hadoop_manager_ip(), uri)
+    headers = {'X-Requested-By': hadoop_manager_username()}
+    auth = (hadoop_manager_username(), hadoop_manager_password())
+    return requests.get(full_uri, auth=auth, headers=headers).json()
+
+def get_namenode_from_ambari():
+    """Returns hadoop namenode IP address"""
+    core_site = ambari_request('/clusters/%s?fields=Clusters/desired_configs/core-site' % cluster_name())
+    config_version = core_site['Clusters']['desired_configs']['core-site']['tag']
+    core_site_config = ambari_request('/clusters/%s/configurations/?type=core-site&tag=%s' % (cluster_name(), config_version))
+    return core_site_config['items'][0]['properties']['fs.defaultFS']
+
+def hadoop_namenode():
+    """Returns the hadoop namenode host or nameservice name in case of HA namenode"""
+    if hadoop_distro() == 'CDH':
+        namenode_host = None
+        name_service = get_name_service()
+        if name_service:
+            namenode_host = name_service
+        else:
+            namenode_host = cloudera_get_hosts_by_role('hdfs01', 'NAMENODE')[0]
+        return 'hdfs://%s:8020' % namenode_host
+    else:
+        return get_namenode_from_ambari()
+
+def hbase_master_host():
+    """Returns host name of an hbase master host"""
+    if hadoop_distro() == 'CDH':
+        return 'todo'
+    else:
+        return ambari_request('/clusters/%s/services/HBASE/components/HBASE_MASTER' % (cluster_name()))['host_components'][0]['HostRoles']['host_name']
+
+def hadoop_manager_ip():
     """ Returns the Cloudera Manager ip address"""
-    cm = ip_addresses('cloudera_manager')
+    cm = ip_addresses('hadoop_manager')
     if cm is not None and len(cm) > 0:
         return cm[0]
     else:
@@ -70,9 +106,9 @@ def generate_http_link(role, suffix):
         return ''
 
 def cloudera_get_hosts_by_role(service, role_type):
-    user = __salt__['pillar.get']('admin_login:user')
-    password = __salt__['pillar.get']('admin_login:password')
-    endpoint = cloudera_manager_ip() + ':7180'
+    user = hadoop_manager_username()
+    password = hadoop_manager_password()
+    endpoint = hadoop_manager_ip() + ':7180'
     cluster = cluster_name()
 
     request_url = 'http://{}/api/v14/clusters/{}/services/{}/roles'.format(endpoint, cluster, service)
@@ -93,3 +129,12 @@ def cloudera_get_hosts_by_role(service, role_type):
         hosts_ips.append(ip_address)
 
     return hosts_ips
+
+def ambari_get_hosts_by_role(service, role_type):
+    return [host['HostRoles']['host_name'] for host in ambari_request('/clusters/%s/services/%s/components/%s' % (cluster_name(),service,role_type))['host_components']]
+
+def get_hosts_by_role(service, role_type):
+    if hadoop_distro() == 'CDH':
+        return cloudera_get_hosts_by_role(service, role_type)
+    else:
+        return ambari_get_hosts_by_role(service, role_type)

salt/ambari/agent.sls

@@ -0,0 +1,45 @@
+{%- set ambari_server_host = salt['pnda.ip_addresses']('hadoop_manager')[0] -%}
+
+ambari-agent-user:
+  user.present:
+    - name: ambari
+    - groups:
+      - root
+
+{% if grains['os'] == 'RedHat' %}
+ambari-agent-libtirpc:
+  pkg.installed:
+    - name: {{ pillar['libtirpc-devel']['package-name'] }}
+    - version: {{ pillar['libtirpc-devel']['version'] }}
+    - ignore_epoch: True
+{%- endif %}
+
+ambari-agent-pkg:
+  pkg.installed:
+    - name: {{ pillar['ambari-agent']['package-name'] }}
+    - version: {{ pillar['ambari-agent']['version'] }}
+    - ignore_epoch: True
+
+ambari-agent-properties:
+  file.managed:
+    - name: /etc/ambari-agent/conf/ambari-agent.ini
+    - source: salt://ambari/templates/ambari-agent.ini.tpl
+    - template: jinja
+    - permission: 0644
+    - defaults:
+        ambari_server_host: {{ ambari_server_host }}
+
+ambari-agent-create_log_dir:
+  file.directory:
+    - name: /var/log/pnda/ambari/
+    - makedirs: True
+
+{% if grains['os'] == 'RedHat' %}
+ambari-agent-systemctl_reload:
+  cmd.run:
+    - name: /bin/systemctl daemon-reload; /bin/systemctl enable ambari-agent
+{%- endif %}
+
+ambari-agent-start_service:
+  cmd.run:
+    - name: 'service ambari-agent stop || echo already stopped; service ambari-agent start'

salt/ambari/files/ambari-server-log4j.properties

@@ -0,0 +1,57 @@
+
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is managed by Salt
+
+# Define some default values that can be overridden by system properties
+# Root logger option
+ambari.log.dir=/var/log/pnda/ambari/
+ambari.log.file=ambari-server.log
+ambari.config-changes.file=ambari-config-changes.log
+ambari.alerts.file=ambari-alerts.log
+
+log4j.rootLogger=INFO,file
+
+# Direct log messages to a log file
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.File=/var/log/pnda/ambari/ambari-server.log
+log4j.appender.file.MaxFileSize=80MB
+log4j.appender.file.MaxBackupIndex=60
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+
+# Log config changes
+log4j.logger.configchange=INFO,configchange
+log4j.additivity.configchange=false
+log4j.appender.configchange=org.apache.log4j.FileAppender
+log4j.appender.configchange.File=/var/log/pnda/ambari/ambari-server-config-changes.log
+log4j.appender.configchange.layout=org.apache.log4j.PatternLayout
+log4j.appender.configchange.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
+
+# Log alert state changes
+log4j.logger.alerts=INFO,alerts
+log4j.additivity.alerts=false
+log4j.appender.alerts=org.apache.log4j.FileAppender
+log4j.appender.alerts.File=/var/log/pnda/ambari/ambari-alerts.log
+log4j.appender.alerts.layout=org.apache.log4j.PatternLayout
+log4j.appender.alerts.layout.ConversionPattern=%d{ISO8601} %m%n
+
+log4j.logger.org.apache.hadoop.yarn.client=WARN
+log4j.logger.org.apache.slider.common.tools.SliderUtils=WARN
+log4j.logger.org.apache.ambari.server.security.authorization=WARN