大数据环境搭建笔记

source /etc/profile

虚拟机环境

名称 内网ip 外网ip 密码
master
slave1
slave2
slave3

变量

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
TZ='Asia/Shanghai';export TZ

export JAVA_HOME=/usr/java/jdk1.8.0_171
export CLASSPATH=$JAVA_HOME/lib/
export PATH=$PATH:$JAVA_HOME/bin
export PATH JAVA_HOME CLASSPATH

export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10
PATH=$PATH:$ZOOKEEPER_HOME/bin

export HIVE_HOME=/usr/hive/apache-hive-2.1.1-bin
export PATH=$PATH:$HIVE_HOME/bin

下载脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
vi wget.sh

#!/bin/bash
darray=(http://172.16.47.240/bigdata/bigdata_tar/jdk-8u171-linux-x64.tar.gz http://172.16.47.240/bigdata/bigdata_tar/zookeeper-3.4.10.tar.gz http://172.16.47.240/bigdata/bigdata_tar/hadoop-2.7.3.tar.gz http://172.16.47.240/bigdata/bigdata_tar/apache-hive-2.1.1-bin.tar.gz)
filearray=(/usr/java /usr/zookeeper /usr/hadoop /usr/hive)
namearray=(/usr/java/jdk-8u171-linux-x64.tar.gz /usr/zookeeper/zookeeper-3.4.10.tar.gz /usr/hadoop/hadoop-2.7.3 /usr/hive/apache-hive-2.1.1-bin)


for i in 0 1 2 3
do
{
wget -P ${filearray[$i]} ${darray[$i]} 

}&
done
wait

echo "jdk,zookeeper,hadoop,hive 下载解压完成"

# 还有问题 暂时别用，上面只是下载的 还需要解压
&& tar -zxvf ${namearray[$i]} -C ${filearray[$i]}
echo "export JAVA_HOME=/usr/java/jdk1.8.0_171" >> /etc/profile
echo "export CLASSPATH=$JAVA_HOME/lib/" >> /etc/profile
echo "export PATH=$PATH:$JAVA_HOME/bin" >> /etc/profile
echo "export PATH JAVA_HOME CLASSPATH" >> /etc/profile
echo "export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3" >> /etc/profile
echo "export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin" >> /etc/profile
echo "PATH=$PATH:$ZOOKEEPER_HOME/bin" >> /etc/profile
echo "export HIVE_HOME=/usr/hive/apache-hive-2.1.1-bin" >> /etc/profile
echo "export PATH=$PATH:$HIVE_HOME/bin" >> /etc/profile
source /etc/profile
echo "环境变量配置成功"



执行:chmod +x wget.sh
执行：./wget.sh

修改本地源

修改主机名

1
2
hostnamectl set-hostname master (slave1 slave2)
bash

配置yum源

1
2
3
4
5
6
7
#!bin/bash
pkill -9 yum
cd /etc/yum.repos.d
rm -rf *
wget http://172.16.47.240/bigdata/repofile/bigdata.repo
yum clean all

时间同步

选择时区

1
2
3
4
选择时区：tzselect
选择地区，填写数字
确定覆盖时间：1
修改环境变量：echo "TZ='Asia/Shanghai';export Tz" >> /etc/profile && /etc/profile

配置ntp

1
yum install -y ntp

master 作为ntp服务器

1
2
3
4
5
6
7
vi /etc/ntp.conf

注释掉 server 0~n
添加:		server 127.127.1.0
		fudge 127.127.1.0 stratum 10
在master上： /bin/systemctl restart ntpd.service
在slave1,slave2上： ntpdate master

配置定时任务

1
2
3
确保crontab服务开启
service crond status
/sbin/service crond start

定时任务

1
2
3
crontab -e
i: */30 10-17 * * * /usr/sbin/ntpdate master
crontab -l

zookeeper

解压

1
2
3
cd ~/usr/zookeeper
tar -zxvf /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper/
rm -rf /usr/zookeeper/zookeeper-3.4.10.tar.gz

配置环境变量

1
2
3
4
5
6
vi /etc/profile

export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10
PATH=$PATH:$ZOOKEEPER_HOME/bin

source /etc/profile

修改配置文件

1
2
1:正常操作
cp /usr/zookeeper/zookeeper-3.4.10/conf/zoo_sample.cfg zoo.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
2:离谱操作
vi /usr/zookeeper/zookeeper-3.4.10/conf/zoo.cfg

插入：
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdata
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1

dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalog

server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888

:wq
1
mkdir /usr/zookeeper/zookeeper-3.4.10/zkdata   zkdatalog

修改myid

1
2
echo 1 >>myid
2 3 依次类推

启动zookeeper

1
2
开启服务：zkServer.sh start
查看状态：zkServer.sh status

Hive

slave2上安装hive

1
2
3
4
5
6
7
yum install -y mysql-community-server

启动服务：
重载修改过的配置文件：systemctl daemon-reload
开启服务：systemctl start mysqld
开机自启：systemctl enable mysqld

登录MySQL

1
2
3
4
获取临时密码：grep "temporary password" /var/log/mysqld.log
k:qmdilTZ5Zz
登录mysql: mysql -uroot -p
手动输入密码，复制大概率出错

密码策略设置

1
2
3
4
设置密码强度为低级：set global validate_password_policy=0;
设置密码长度：set global validate_password_length=4;
修改本地密码：alter user 'root'@'localhost' identified by '123456';
退出：\q

设置远程登录：

1
2
3
4
5
以新密码登录：mysql -uroot -p123456
创建用户：create user 'root'@'%'identified by'123456';
允许远程连接：grant all privileges on *.* to 'root'@'%' with grant option;
刷新权限：flush privileges;
创建数据库：create database xxx;

数据仓库下载安装

1
2
3
4
5
6
在master上 建立文件夹
mkdir -p /usr/hive && cd /usr/hive

下载hive
wget http://47.92.249.178/bigdata/bigdata_tar/apache-hive-2.1.1-bin.tar.gz
解压：tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/
大数据 环境搭建

本博客所有文章除特别声明外，均采用 CC BY-SA 4.0 协议 ，转载请注明出处！