带大家一起复习Hadoop CDH 3节点集群安装

主机名
IP
角色
MASTER
192.168.0.64
Hadoop Master,CM Server,Data Node
SLAVE01
192.168.0.62
Data Node,MySQL,CM Agent
SLAVE02
192.168.0.63
Data Node,CM Agent

[root@SLAVE01 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.0.62  SLAVE01
192.168.0.63  SLAVE02
192.168.0.64  MASTER

[root@SLAVE01 ~]# service iptables stop
[root@SLAVE01 ~]# chkconfig iptables off

[root@SLAVE01 ~]# cat /etc/selinux/config

SELINUX=disabled
SELINUXTYPE=targeted

server 192.168.0.64 prefer

[root@MASTER ~]# service ntpd start

[root@SLAVE01 ~]# vim /etc/sysctl.conf 

vm.swappiness = 0

[root@SLAVE01 ~]# sysctl -p

[root@SLAVE01 ~]# echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag 
[root@SLAVE01 ~]# echo echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag >> /etc/rc.local

[root@SLAVE01 ~]# rpm -qa | grep mysql
mysql-libs-5.1.71-1.el6.x86_64
mysql-5.1.71-1.el6.x86_64
mysql-devel-5.1.71-1.el6.x86_64

[root@SLAVE01 ~]# rpm -e mysql-devel-5.1.71-1.el6.x86_64
[root@SLAVE01 ~]# rpm -e mysql
[root@SLAVE01 ~]# rpm -e --nodeps mysql-libs-5.1.71-1.el6.x86_64
[root@SLAVE01 ~]# rpm -qa | grep mysql

[root@SLAVE01 ~]# tar -xvf ./MySQL-5.6.27-1.el6.x86_64.rpm-bundle.tar
[root@SLAVE01 ~]# rpm -ivh MySQL-*.rpm

[root@SLAVE01 ~]# cat /root/.mysql_secret 
# The random password set for the root user at Sun Apr 1 21:22:17 2018 (local time): KYFVIGg8KErbhsCE

[root@SLAVE01 ~]# /usr/bin/mysqld_safe &

[root@SLAVE01 ~]# mysql -uroot -p
Enter password: 
Welcome to the MySQL monitor. Commands end with ; or g.
Your MySQL connection id is 1
Server version: 5.6.27

Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type help; or h for help. Type c to clear the current input statement.

mysql> SET PASSWORD FOR root@localhost = PASSWORD(123456);
Query OK, 0 rows affected (0.00 sec)

mysql> create database scm;
Query OK, 1 row affected (0.00 sec)

mysql> create database hive;
Query OK, 1 row affected (0.00 sec)

mysql> create database rm;
Query OK, 1 row affected (0.00 sec)

mysql> create database oozie;
Query OK, 1 row affected (0.00 sec)

mysql> CREATE USER cdh@% IDENTIFIED BY 123456;
Query OK, 0 rows affected (0.00 sec)

mysql> GRANT ALL ON *.* TO cdh@%;
Query OK, 0 rows affected (0.00 sec)

[root@SLAVE01 ~]# useradd --system --home=/opt/cloudera-manager/cm-5.5.0/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

mkdir /opt/cloudera-manager
mkdir -p /opt/cloudera/parcel-repo
mkdir -p /opt/cloudera/parcels
mkdir -p /var/log/cloudera-scm-headlamp
mkdir -p /var/log/cloudera-scm-firehose
mkdir -p /var/log/cloudera-scm-alertpublisher
mkdir -p /var/log/cloudera-scm-eventserver
mkdir -p /var/lib/cloudera-scm-headlamp
mkdir -p /var/lib/cloudera-scm-firehose
mkdir -p /var/lib/cloudera-scm-alertpublisher
mkdir -p /var/lib/cloudera-scm-eventserver
mkdir -p /var/lib/cloudera-scm-server

chown cloudera-scm:cloudera-scm /opt/cloudera-manager
chown -R cloudera-scm:cloudera-scm /var/log/cloudera-*
chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-*
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
chown cloudera-scm:cloudera-scm /opt/cloudera/parcels

[root@SLAVE01 ~]# tar -xzf cloudera-manager-el6-cm5.5.0_x86_64.tar.gz -C /opt/cloudera-manager
[root@SLAVE01 ~]# chown cloudera-scm:cloudera-scm /opt/cloudera-manager

[root@SLAVE01 ~]# mkdir -p /usr/share/java
[root@SLAVE01 ~]# cp mysql-connector-java-5.1.38.jar /usr/share/java/
[root@SLAVE01 ~]# mkdir -p /usr/lib/hive/lib/
[root@SLAVE01 ~]# cp mysql-connector-java-5.1.38.jar /usr/lib/hive/lib/

[root@SLAVE01 ~]# rpm -ivh jdk-7u80-linux-x64.rpm

[root@SLAVE01 ~]# vim /etc/profile

export JAVA_HOME=/usr/java/default
export CLASS_PATH=./:$JAVA_HOME/lib
export PATH=$JAVA_HOME/bin:$PATH

[root@SLAVE01 ~]# source /etc/profile

[root@SLAVE01 ~]# java -version
java version "1.7.0_80"
Java(TM) SE Runtime Environment (build 1.7.0_80-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.80-b11, mixed mode)

[root@MASTER ~]# cd /opt/cloudera-manager/cm-5.5.0/share/cmf/schema/
[root@MASTER schema]# ./scm_prepare_database.sh -h SLAVE01 -P 3306 mysql scm cdh 123456

[root@SLAVE01 ~]# cd /opt/cloudera-manager/cm-5.5.0/etc/cloudera-scm-agent/
[root@SLAVE01 cloudera-scm-agent]# vim config.ini

server_host=MASTER

[root@MASTER ~]# mv CDH-5.5.0-1.cdh5.5.0.p0.8-el6.parcel /opt/cloudera/parcel-repo/
[root@MASTER ~]# mv manifest.json /opt/cloudera/parcel-repo/
[root@MASTER ~]# mv CDH-5.5.0-1.cdh5.5.0.p0.8-el6.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.5.0-1.cdh5.5.0.p0.8-el6.parcel.sha

[root@MASTER ~]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-server start
Starting cloudera-scm-server: [ OK ]

[root@MASTER ~]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-agent start
Starting cloudera-scm-agent:                               [ OK ]

[root@SLAVE01 cloudera-scm-agent]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-agent start
Starting cloudera-scm-agent:                               [ OK ]

[root@SLAVE02 ~]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-agent start
Starting cloudera-scm-agent:                               [ OK ]

SLAVE01  DB Name:hive  Username:cdh  Password:123456
SLAVE01  DB Name:rm  Username:cdh  Password:123456
SLAVE01  DB Name:oozie  Username:cdh  Password:123456

Change Hosts Network Interfaces Slow Link Modes Thresholds for all hosts.

[root@MASTER ~]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-agent stop
Stopping cloudera-scm-agent:                               [ OK ]
[root@MASTER ~]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-server stop
Stopping cloudera-scm-server: [ OK ]

[root@SLAVE01 ~]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-agent stop
Stopping cloudera-scm-agent:                               [ OK ]

[root@SLAVE02 ~]# /opt/cloudera-manager/cm-5.5.0/etc/init.d/cloudera-scm-agent stop
Stopping cloudera-scm-agent:                               [ OK ]