-
Notifications
You must be signed in to change notification settings - Fork 34
/
elk.azcli
287 lines (263 loc) · 23.5 KB
/
elk.azcli
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
##################################################
# Lab for Elastic Stack on Azure
#
# Credits: Getting Started with Elastic Stack 8.0
# Jose Moreno
# May 2022
##################################################
# Variables
rg=elk2
location=westeurope
vnet_name=elk
vnet_prefix=10.99.64.0/22
elk_subnet_name=elk
elk_subnet_prefix=10.99.64.0/27
vm_subnet_name=vm
vm_subnet_prefix=10.99.64.32/27
elk_vm_name=elk
elk_vm_size=Standard_B2ms
elk_cloudinit_file=/tmp/elk_cloudinit.txt
nginx_vm_name=nginx
nginx_vm_size=Standard_B2ms
nginx_cloudinit_file=/tmp/nginx_cloudinit.txt
akv_name=erjositoKeyvault
default_password_secret=defaultPassword
# Get secrets
akv_rg_found=$(az keyvault list -o tsv --query "[?name=='$akv_name'].resourceGroup" 2>/dev/null)
if [[ -n ${akv_rg_found} ]]
then
echo "INFO: AKV ${akv_name} found in resource group $akv_rg_found"
default_password=$(az keyvault secret show --vault-name $akv_name -n $default_password_secret --query 'value' -o tsv 2>/dev/null)
else
echo "ERROR: secrets could not be read because Azure Key Vault ${akv_name} could not be found"
fi
# Create RG and VNets
echo "Creating RG and VNet..."
az group create -n $rg -l $location -o none
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix --subnet-name $vm_subnet_name --subnet-prefix $vm_subnet_prefix -l $location -o none
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $elk_subnet_name --address-prefix $elk_subnet_prefix -o none
# Create NSGs
echo "Creating NSGs..."
az network nsg create -n "${elk_vm_name}-nsg" -g $rg -o none
az network nsg rule create -n SSH --nsg-name "${elk_vm_name}-nsg" -g $rg --priority 1000 --destination-port-ranges 22 --access Allow --protocol Tcp -o none
az network nsg rule create -n Kibana --nsg-name "${elk_vm_name}-nsg" -g $rg --priority 1010 --destination-port-ranges 5601 --access Allow --protocol Tcp -o none
az network nsg rule create -n ElasticSearch --nsg-name "${elk_vm_name}-nsg" -g $rg --priority 1020 --destination-port-ranges 9200 --access Allow --protocol Tcp -o none
az network nsg rule create -n ICMP --nsg-name "${elk_vm_name}-nsg" -g $rg --priority 1030 --source-address-prefixes '*' --destination-address-prefixes '*' --destination-port-ranges '*' --access Allow --protocol Icmp -o none
az network nsg create -n "${nginx_vm_name}-nsg" -g $rg -o none
az network nsg rule create -n SSH --nsg-name "${nginx_vm_name}-nsg" -g $rg --priority 1000 --destination-port-ranges 22 --access Allow --protocol Tcp -o none
az network nsg rule create -n Web80 --nsg-name "${nginx_vm_name}-nsg" -g $rg --priority 1010 --destination-port-ranges 80 --access Allow --protocol Tcp -o none
az network nsg rule create -n Web443 --nsg-name "${nginx_vm_name}-nsg" -g $rg --priority 1020 --destination-port-ranges 443 --access Allow --protocol Tcp -o none
az network nsg rule create -n ICMP --nsg-name "${nginx_vm_name}-nsg" -g $rg --priority 1030 --source-address-prefixes '*' --destination-address-prefixes '*' --destination-port-ranges '*' --access Allow --protocol Icmp -o none
# Create Elastic Search VM
echo "Creating cloudinit file for Elastic Search..."
cat <<EOF > $elk_cloudinit_file
#cloud-config
packages:
- jq
- apt-transport-https
runcmd:
- wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
- echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-8.x.list
- apt update && apt install -y elasticsearch kibana logstash
- apt install -y filebeat metricbeat
- systemctl enable elasticsearch
- systemctl enable kibana
- systemctl start elasticsearch
- systemctl start kibana
EOF
echo "Creating Elastic Stack VM..."
az vm create -n $elk_vm_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys --nsg "${elk_vm_name}-nsg" -o none \
--custom-data $elk_cloudinit_file --public-ip-sku Standard --public-ip-address "${elk_vm_name}-pip" \
--vnet-name $vnet_name --size $elk_vm_size --subnet $elk_subnet_name -l $location --no-wait
# Create test nginx VM
echo "Creating cloudinit file for Nginx VM..."
cat <<EOF > $nginx_cloudinit_file
#cloud-config
packages:
- jq
- nginx
- apt-transport-https
runcmd:
- wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
- echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-8.x.list
- apt update && apt install -y filebeat metricbeat
- cd /tmp
- git clone https://github.com/PacktPublishing/Getting-Started-with-Elastic-Stack-8.0.git
- mkdir -p /var/www/elastic-stack-server
- cp -r Getting-Started-with-Elastic-Stack-8.0/Chapter6/html-webpage/* /var/www/elastic-stack-server
EOF
echo "Creating nginx VM..."
az vm create -n $nginx_vm_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys --nsg "${nginx_vm_name}-nsg" -o none \
--custom-data $nginx_cloudinit_file --public-ip-sku Standard --public-ip-address "${nginx_vm_name}-pip" \
--vnet-name $vnet_name --size $nginx_vm_size --subnet $vm_subnet_name -l $location --no-wait
# Make sure to reset the elastic user's password with elasticsearch-setup-password or elasticsearch-reset-password !!!!
elk_pip=$(az network public-ip show -n "${elk_vm_name}-pip" -g $rg --query ipAddress -o tsv)
ssh $elk_pip "sudo /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic -i"
ssh $elk_pip "sudo /usr/share/elasticsearch/bin/elasticsearch-reset-password -u kibana -i"
# Verify Elastic Search VM
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "systemctl status elasticsearch"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "systemctl status kibana"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "curl -sk -u elastic:${default_password} https://localhost:9200"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "curl -sk -u kibana_system:${default_password} https://localhost:9200/_xpack/security/_authenticate?pretty"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "curl -sk -u elastic:${default_password} https://localhost:9200/_cluster/health"
# Configure ElasticSearch and Kibana
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i '/network.host/c\network.host: 0.0.0.0' /etc/elasticsearch/elasticsearch.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo systemctl restart elasticsearch"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i '/server.host/c\server.host: \"0.0.0.0\"' /etc/kibana/kibana.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i '/server.name/c\server.name: \"elastictest\"' /etc/kibana/kibana.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i '/elasticsearch.username/c\elasticsearch.username: \"kibana\"' /etc/kibana/kibana.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i '/elasticsearch.password/c\elasticsearch.password: \"$default_password\"' /etc/kibana/kibana.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i '/elasticsearch.hosts/c\elasticsearch.hosts: [\"https://localhost:9200\"]' /etc/kibana/kibana.yml"
verification_mode=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo grep verificationMode /etc/kibana/kibana.yml")
if [[ -z "$verification_mode" ]]; then
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/protocol: /a ssl.verification_mode: none' /etc/kibana/kibana.yml"
fi
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i '/elasticsearch.ssl.verificationMode/c\elasticsearch.ssl.verificationMode: none' /etc/kibana/kibana.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo sed -i -e '$ axpack.reporting.kibanaServer.hostname: localhost' /etc/kibana/kibana.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $elk_pip "sudo systemctl restart kibana"
# Configure NGINX server
nginx_pip=$(az network public-ip show -n "${nginx_vm_name}-pip" -g $rg --query ipAddress -o tsv)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/root \/var\/www\/html/c\ root \/var\/www\/elastic-stack-server;' /etc/nginx/sites-available/default"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart nginx"
# Configure filebeat agent
elk_nic_id=$(az vm show -n $elk_vm_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
elk_private_ip=$(az network nic show --ids $elk_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl enable filebeat"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/hosts: \[\"localhost:9200\"\]/c\ hosts: [\"https://$elk_private_ip:9200\"]' /etc/filebeat/filebeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/username: /c\ username: \"elastic\"' /etc/filebeat/filebeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/password: /c\ password: \"$default_password\"' /etc/filebeat/filebeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/protocol: /c\ protocol: \"https\"' /etc/filebeat/filebeat.yml"
verification_mode=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo grep verification_mode /etc/filebeat/filebeat.yml")
if [[ -z "$verification_mode" ]]; then
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/protocol: /a ssl.verification_mode: none' /etc/filebeat/filebeat.yml"
fi
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/verification_mode: /c\ ssl.verification_mode: none' /etc/filebeat/filebeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart filebeat"
# Configure nginx module (to do before enabling?)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/error:/{n;s/.*/ enabled: true/}' /etc/filebeat/modules.d/nginx.yml.disabled"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/access:/{n;s/.*/ enabled: true/}' /etc/filebeat/modules.d/nginx.yml.disabled"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/syslog:/{n;s/.*/ enabled: true/}' /etc/filebeat/modules.d/system.yml.disabled"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart filebeat"
# Load modules
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat modules enable nginx"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\" --modules nginx --dashboards --pipelines"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\" --index-management"
# See https://github.com/elastic/beats/issues/30916
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\" --modules nginx --dashboards --pipelines -M \"nginx.access.enabled=true\""
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\" --modules nginx --dashboards --pipelines -M \"nginx.error.enabled=true\""
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\" --modules system --dashboards --pipelines -M \"system.syslog.enabled=true\""
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart filebeat"
# Install metricbeat if it wasnt installed and configure
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '$ a\server {\n server_name 127.0.0.1;\n location /server_status {\n stub_status;\n allow 127.0.0.1;\n deny all;\n }\n}\n' /etc/nginx/sites-enabled/default"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart nginx"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo apt install -y metricbeat"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl enable metricbeat"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/hosts: \[\"localhost:9200\"\]/c\ hosts: [\"https://$elk_private_ip:9200\"]' /etc/metricbeat/metricbeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/username: /c\ username: \"elastic\"' /etc/metricbeat/metricbeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/password: /c\ password: \"$default_password\"' /etc/metricbeat/metricbeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/protocol: /c\ protocol: \"https\"' /etc/metricbeat/metricbeat.yml"
verification_mode=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo grep verification_mode /etc/metricbeat/metricbeat.yml")
if [[ -z "$verification_mode" ]]; then
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/protocol: /a ssl.verification_mode: none' /etc/metricbeat/metricbeat.yml"
fi
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/verification_mode: /c\ ssl.verification_mode: none' /etc/metricbeat/metricbeat.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo metricbeat modules enable nginx"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/metricsets/c\ metricsets:' /etc/metricbeat/modules.d/nginx.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/stubstatus/c\ - stubstatus' /etc/metricbeat/modules.d/nginx.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo metricbeat modules enable system"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo metricbeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\""
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl start metricbeat"
# Create Azure Service Principal if not existing
keyvault_name=erjositoKeyvault
purpose=metricbeat
keyvault_name=erjositoKeyvault
keyvault_appid_secret_name=$purpose-sp-appid
keyvault_password_secret_name=$purpose-sp-secret
sp_app_id=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_appid_secret_name --query 'value' -o tsv)
sp_app_secret=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_password_secret_name --query 'value' -o tsv)
if [[ -z "$sp_app_id" ]] || [[ -z "$sp_app_secret" ]]
then
echo "Creating new Service Principal..."
sp_name=$purpose
sp_output=$(az ad sp create-for-rbac --name $sp_name --skip-assignment 2>/dev/null)
sp_app_id=$(echo $sp_output | jq -r '.appId')
sp_app_secret=$(echo $sp_output | jq -r '.password')
az keyvault secret set --vault-name $keyvault_name --name $keyvault_appid_secret_name --value $sp_app_id -o none
az keyvault secret set --vault-name $keyvault_name --name $keyvault_password_secret_name --value $sp_app_secret -o none
subscription_id=$(az account show -o tsv --query id)
echo "Assigning Reader role to new Service Principal..."
az role assignment create --scope "/subscriptions/${subscription_id}" --assignee $sp_app_id --role Reader -o none
else
echo "Service Principal $sp_app_id and password located in Azure Key Vault $keyvault_name"
fi
# Configure metricbeat Azure module. We will use the nginx host to collect the metrics
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo metricbeat modules enable azure"
subscription_id=$(az account show -o tsv --query id)
tenant_id=$(az account show -o tsv --query tenantId)
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ tenant_id/c\ tenant_id: \"$tenant_id\"' /etc/metricbeat/modules.d/azure.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ subscription_id/c\ subscription_id: \"$subscription_id\"' /etc/metricbeat/modules.d/azure.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ client_id/c\ client_id: \"$sp_app_id\"' /etc/metricbeat/modules.d/azure.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ client_secret/c\ client_secret: \"$sp_app_secret\"' /etc/metricbeat/modules.d/azure.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo metricbeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\""
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart metricbeat"
# Alternatively, using the keystore (not tested, but recommended!)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "echo -n \"$sp_app_id\" | sudo metricbeat keystore add AZURE_CLIENT_ID --stdin --force"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "echo -n \"$sp_app_secret\" | sudo metricbeat keystore add AZURE_CLIENT_SECRET --stdin"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "echo -n \"$tenant_id\" | sudo metricbeat keystore add AZURE_TENANT_ID --stdin"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "echo -n \"$subscription_id\" | sudo metricbeat keystore add AZURE_SUBSCRIPTION_ID --stdin"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ tenant_id/c\ tenant_id: \x27\${AZURE_TENANT_ID:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ subscription_id/c\ subscription_id: \x27\${AZURE_SUBSCRIPTION_ID:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ client_id/c\ client_id: \x27\${AZURE_CLIENT_ID:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ client_secret/c\ client_secret: \x27\${AZURE_SECRET_ID:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '$ a #\n# Injected from script' /etc/metricbeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '$ a - module: azure\n metricsets:\n - compute_vm_scaleset\n enabled: true\n period: 30s\n refresh_list_interval: 600s' /etc/metricbeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '$ a \ \ client_id: \x27\${AZURE_CLIENT_ID:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '$ a \ \ client_secret: \x27\${AZURE_CLIENT_SECRET:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '$ a \ \ subscription_id: \x27\${AZURE_TENANT_ID:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '$ a \ \ tenant_id: \x27\${AZURE_SUBSCRIPTION_ID:\"\"}\x27' /etc/metricbeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart metricbeat"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo metricbeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\""
# Configure filebeat Azure module. We will use the nginx host to collect the logs
# Create storage account and Events Hub
storage_account_name=$(az storage account list -g $rg -o tsv --query '[0].name')
storage_container_name=filebeat
if [[ -z "$storage_account_name" ]]; then
storage_account_name=elk$RANDOM
echo "No storage account found resource group ${rg}, creating one..."
az storage account create -n $storage_account_name -g $rg --sku Standard_LRS --kind StorageV2 -l $location -o none
az storage container create -n $storage_container_name --account-name $storage_account_name --auth-mode login -o none
else
echo "Storage account $storage_account_name found in RG $rg"
fi
storage_account_key=$(az storage account keys list -n $storage_account_name --query '[0].value' -o tsv)
eh_ns_name=$(az eventhubs namespace list -g $rg --query '[0].name' -o tsv)
eh_ns_name=$eh_name
if [[ -z "$eh_ns_name" ]]; then
eh_name=elk$RANDOM
eh_ns_name=$eh_name
echo "Creating Event Hub ${eh_name}..."
az eventhubs namespace create -n $eh_ns_name -g $rg -l $location --sku Standard -o none
az eventhubs eventhub create -n $eh_name -g $rg --namespace-name $eh_ns_name -o none
else
echo "Events Hub $eh_name found in RG $rg"
fi
eh_ns_cx_string=$(az eventhubs namespace authorization-rule keys list -g $rg --namespace-name $eh_ns_name --name RootManageSharedAccessKey --query primaryConnectionString -o tsv)
# Configure in the portal redirection to this Event Hub (To Do: CLI commands)
# Configure filebeat azure module
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat modules enable azure"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ eventhub:/c\ eventhub: \"$eh_name\"' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ connection_string:/c\ connection_string: \"$eh_ns_cx_string\"' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ consumer_group:/c\ consumer_group: \"\$Default\"' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ storage_account:/c\ storage_account: \"$storage_account_name\"' /etc/filebeat/modules.d/azure.yml"
#ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ storage_account:/c\ storage_account_container: \"$storage_account_container\"' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/ storage_account_key:/c\ storage_account_key: \"$storage_account_key\"' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/activitylogs:/{n;s/.*/ enabled: true/}' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/auditlogs:/{n;s/.*/ enabled: true\n var:/}' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/platformlogs:/{n;s/.*/ enabled: false\n var:/}' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo sed -i '/signinlogs:/{n;s/.*/ enabled: false\n var:/}' /etc/filebeat/modules.d/azure.yml"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo systemctl restart filebeat"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\" --modules azure --dashboards --pipelines -M \"azure.activitylogs.enabled=true\""
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nginx_pip "sudo filebeat setup -E \"setup.kibana.host=${elk_private_ip}:5601\" -E \"setup.kibana.ssl.verification_mode=none\" --modules azure --dashboards --pipelines -M \"azure.auditlogs.enabled=true\""
# Kubernetes:
# https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html
# https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-kubernetes.html