forked from milinda/KafkaOnEC2
-
Notifications
You must be signed in to change notification settings - Fork 0
/
kafka.yml
115 lines (109 loc) · 3.87 KB
/
kafka.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
---
- name: Create EC2 instances for Zookeeper
hosts: localhost
connection: local
gather_facts: no
tasks:
- name: Launch instances
ec2:
key_name: "{{ ec2.key }}"
group: "{{ ec2.zookeeper_security_group }}"
instance_type: "{{ ec2.zookeeper_instance_type }}"
image: "{{ ec2.image }}"
wait: yes
region: "{{ ec2.region }}"
count: "{{ ec2.zookeeper_instance_count }}"
spot_price: "{{ ec2.zk_spot_price }}"
spot_wait_timeout: 600
register: zookeeperinstances
- name: Add new instances to a host group
add_host: name={{ item.1.public_dns_name }} groups=zookeepernodes zid={{ item.0 + 1 }} private_ip={{ item.1.private_ip }} instance_type={{ item.1.instance_type }}
with_indexed_items: "{{ zookeeperinstances.instances }}"
- name: Wait for SSH to come up
wait_for: host={{ item.public_dns_name }} port=22 delay=60 timeout=360 state=started
with_items: "{{ zookeeperinstances.instances }}"
- name: Make sure the known hosts file exists
file: "path={{ ssh_known_hosts_file }} state=touch"
- name: Check host name availability
shell: "ssh-keygen -f {{ ssh_known_hosts_file }} -F {{ item.public_dns_name }}"
with_items: "{{ zookeeperinstances.instances }}"
register: z_ssh_known_host_results
ignore_errors: yes
- name: Scan the public key
shell: "{{ ssh_known_hosts_command}} {{ item.item.public_dns_name }} >> {{ ssh_known_hosts_file }}"
with_items: "{{ z_ssh_known_host_results.results }}"
when: item.stdout == ""
- name: Install JDK on Zookeeper instances
hosts: zookeepernodes
remote_user: ubuntu
become: true
become_method: sudo
gather_facts: yes
roles:
- jdk
- name: Install and start Zookeeper
hosts: zookeepernodes
remote_user: ubuntu
become: true
become_method: sudo
gather_facts: yes
roles:
- zookeeper
- name: Create EC2 instances for Kafka brokers
hosts: localhost
connection: local
gather_facts: no
tasks:
- name: Launch instances
ec2:
key_name: "{{ ec2.key }}"
group: "{{ ec2.kafka_security_group }}"
instance_type: "{{ ec2.kafka_instance_type }}"
image: "{{ ec2.image }}"
wait: yes
region: "{{ ec2.region }}"
count: "{{ ec2.kafka_instance_count }}"
spot_price: "{{ ec2.kafka_spot_price }}"
spot_wait_timeout: 600
monitoring: yes
volumes:
- device_name: /dev/sdb
volume_type: io1
iops: 1000
volume_size: 100
delete_on_termination: true
register: kafkainstances
- name: Add new instances to a host group
add_host: name={{ item.1.public_dns_name }} groups=kafkanodes broker_id={{ item.0 + 1 }}
with_indexed_items: "{{ kafkainstances.instances }}"
- name: Wait for SSH to come up
wait_for: host={{ item.public_dns_name }} port=22 delay=60 timeout=360 state=started
with_items: "{{ kafkainstances.instances }}"
- name: Make sure the known hosts file exists
file: "path={{ ssh_known_hosts_file }} state=touch"
- name: Check host name availability
shell: "ssh-keygen -f {{ ssh_known_hosts_file }} -F {{ item.public_dns_name }}"
with_items: "{{ kafkainstances.instances }}"
register: k_ssh_known_host_results
ignore_errors: yes
- name: Scan the public key
shell: "{{ ssh_known_hosts_command}} {{ item.item.public_dns_name }} >> {{ ssh_known_hosts_file }}"
with_items: "{{ k_ssh_known_host_results.results }}"
when: item.stdout == ""
- name: Configure file system and install JDK on Kafka instances
hosts: kafkanodes
remote_user: ubuntu
become: true
become_method: sudo
gather_facts: yes
roles:
- makefs
- jdk
- name: Install and start Kafka
hosts: kafkanodes
remote_user: ubuntu
become: true
become_method: sudo
gather_facts: yes
roles:
- kafka