爱宝儿
游戏开发及物联网大数据集群维护及部署开发，数据存储及相关设计，大数据、ETL、Flink、kafka，hadoop框架等多种技术开>发技术。涵盖语言涉及Java、Go、Python、C、C#等。
仅供学习参考
安装脚本：
1
2
|
yum install -y epel-release
yum install -y supervisor
|
程序配置文件
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
[program:XXXXXXXX]
process_name=%(program_name)s
command = 执行命令行
directory = 执行目录
autostart = true
startsecs = 5
autorestart = true
startretries = 3
user = root
redirect_stderr = false
stdout_logfile_maxbytes = 128MB
stdout_logfile_backups = 5
stdout_logfile = 执行目录/logs/XXXXXXXX-deamaon.log
stderr_logfile_maxbytes = 128MB
stderr_logfile_backups = 5
stderr_logfile = 执行目录/logs/XXXXXXXXX-errdeamaon.log
priority=999
|
系统配置
启动大数据相关环境脚本
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
#!/bin/sh
# ServerHosts
ServerHosts='vm0101 vm0102 vm0103'
usage() {
echo "Usage: sh 执行脚本.sh [zookeeper|kafka|flink] [start|stop|restart|status]"
exit 1
}
# ZooKeeper Cluster Start
FunctionZK(){
echo "--------正在 $1 Zookeeper Cluster--------"
for zoo in $ServerHosts
do
ssh $zoo "source /etc/profile;/home/bigdata/zookeeper/bin/zkServer.sh $1"
if [ $1 == 'stop' ]; then
echo 从节点 $zoo 停止zookeeper...[done]
else
echo 从节点 $zoo 启动zookeeper...[done]
fi
sleep 1
done
}
# Kafka Cluster
FunctionKafka(){
echo "--------正在 $1 Kafka Cluster--------"
for kafka in $ServerHosts
do
if [ $1 == 'stop' ]; then
ssh $kafka "source /etc/profile;/home/bigdata/kafka/bin/kafka-server-stop.sh"
else
ssh $kafka "source /etc/profile;/home/bigdata/kafka/bin/kafka-server-start.sh -daemon /home/bigdata/kafka/config/server.properties"
fi
echo 从节点 $kafka $1 kafka ...[done]
sleep 1
done
}
# Flink Cluster
FunctionFlink(){
echo "--------正在 $1 Flink Cluster--------"
if [ $1 == 'stop' ]; then
ssh vm0201 "source /etc/profile;/home/bigdata/flink/bin/stop-cluster.sh"
else
ssh vm0201 "source /etc/profile;/home/bigdata/flink/bin/start-cluster.sh"
fi
}
case $1 in
"zookeeper")
FunctionZK $2
;;
"kafka")
FunctionKafka $2
;;
"flink")
FunctionFlink $2
;;
*)
usage
;;
esac
|
elasticsearch.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
cluster.name: bigdata
node.name: els-node1
path.data: /bigdata/zzwl/elasticsearch7/data
path.logs: /bigdata/zzwl/elasticsearch7/log
network.host: bigdata01
http.port: 9200
transport.tcp.port: 9300
transport.tcp.compress: true
discovery.seed_hosts: ["bigdata01:9300", "bigdata02:9300", "bigdata03"]
luster.initial_master_nodes: ["els-node1", "els-node2", "els-node3"]
增加新的参数，这样head插件可以访问es
http.cors.enabled: true
http.cors.allow-origin: "*"
|
