-
Notifications
You must be signed in to change notification settings - Fork 32
/
routeserver.azcli
1036 lines (970 loc) · 56.3 KB
/
routeserver.azcli
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#############################################################################
# Creates a basic 1-vnet setup with ARS, CSR as NVA and a VM
# Optionally creates additional components such as Linux-based appliances,
# peered vnets, ER/VPN VNGs and simulated onprem environments.
# This script has multiple sections, it is not intended to use all sections
# at once, but to pick and choose what you need for your particular test.
#
# July 2021, Jose Moreno
#############################################################################
# Control
nva_type=linux # can be csr or linux
redundant_nva=yes # this script only supports the CSR for redundant CSRs
create_spokes=no # can be yes or no
create_remote_spoke=no # create a spoke in a remote region
vpn_branch=no # whether simulating a VPN branch
onprem_nva_type=linux # for the onprem NVA, whether Linux or CSR
er_branch=no # whether simulating an ER branch
# Variables
rg=routeserver
location=northeurope
remote_location=westeurope
vnet_name=hub
vnet_prefix=172.16.0.0/16
vnet_prefix_long='172.16.0.0 255.255.0.0'
rs_subnet_name=RouteServerSubnet
rs_subnet_prefix=172.16.1.0/24
rs_name=hubrs
hub_vm_subnet_name=vm
hub_vm_subnet_prefix=172.16.3.0/24
# GatewaySubnet
gw_subnet_prefix=172.16.254.0/24
# CSR
publisher=cisco
offer=cisco-csr-1000v
sku=16_12-byol
csr_username=$(whoami)
csr_password=Microsoft123!
csr_size=Standard_B2ms
hub_csrext_subnet_name=csrext
hub_csrext_subnet_prefix=172.16.11.0/24
hub_csrext_subnet_prefix_long_reversemask='172.16.11.0 0.0.0.255'
hub_csrext_nsg_name=hubcsrext-nsg
hub_csrint_subnet_name=csrint
hub_csrint_subnet_prefix=172.16.12.0/24
hub_csrint_nsg_name=hubcsrint-nsg
hub_csr_asn=65001
hub_csr1_name=csr1
hub_csr1_bgp_ip=172.16.12.11
hub_csr2_name=csr2
hub_csr2_bgp_ip=172.16.12.12
# Linux NVA
hub_linuxnva_asn=65002
hub_linuxnva_subnet_name=linuxnva
hub_linuxnva_subnet_prefix=172.16.10.0/24
hub_linuxnva_name=ubuntufw
hub_linuxnva_pip_name="${hub_linuxnva_name}-pip"
hub_linuxnva_cloudinit_file=/tmp/linuxnva_cloudinit.txt
# Spoke 1
spoke1_name=spoke1
spoke1_prefix=172.17.0.0/16
spoke1_vm_subnet_name=vm
spoke1_vm_subnet_prefix=172.17.0.0/24
spoke1_csr_subnet_name=csr
spoke1_csr_subnet_prefix=172.17.1.0/24
spoke1_csr_name=spoke1csr
spoke1_csr_asn=65001
# Spoke 2
spoke2_name=spoke2
spoke2_prefix=172.18.0.0/16
spoke2_vm_subnet_name=vm
spoke2_vm_subnet_prefix=172.18.0.0/24
# Spoke 3 (remote)
spoke3_name=spoke3
spoke3_prefix=172.19.0.0/16
spoke3_vm_subnet_name=vm
spoke3_vm_subnet_prefix=172.19.0.0/24
# Azure VPN GW
vpngw_name=vpngw
vpngw_asn=65515
vpngw_pip="${vpngw_name}-pip"
# Azure ExpressRoute GW
ergw_name=ergw
ergw_pip="${ergw_name}-pip"
er_location=germanywestcentral
er_pop=Frankfurt
er_provider=Megaport
er_circuit_name=tester
mcr_asn=65001
megaport_script_path="/home/jose/repos/azcli/megaport.sh"
# VPN onprem (simulated with Linux NVA)
onprem_vnet_name=onprem
onprem_vnet_prefix=192.168.0.0/16
onprem_nva_subnet_name=onprem
onprem_nva_subnet_prefix=192.168.0.0/24
onprem_gw_subnet_name=GatewaySubnet
onprem_gw_subnet_prefix=192.168.1.0/24
onprem_linuxnva_asn=65102
onprem_linuxnva_name=onpremnva
onprem_linuxnva_pip=${onprem_linuxnva_name}-pip
onprem_linuxnva_ip=192.168.0.20
# gcloud (to simulate onprem ER)
project_name=onprem
project_id="${project_name}${RANDOM}"
machine_type=e2-micro
gcp_asn=16550
region=europe-west3 # other values: europe-west3, australia-southeast1
zone=europe-west3-b # other values: europe-west3-b, australia-southeast1-b
gcp_vm_name=vm
gcp_vpc_name=vpc
gcp_subnet_name=vm
gcp_subnet_prefix='10.4.2.0/24'
attachment_name=attachment
router_name=router
####################
# Helper functions #
####################
# Auxiliary function to get the first IP of a subnet (default gateway)
function first_ip(){
subnet=$1
IP=$(echo $subnet | cut -d/ -f 1)
IP_HEX=$(printf '%.2X%.2X%.2X%.2X\n' `echo $IP | sed -e 's/\./ /g'`)
NEXT_IP_HEX=$(printf %.8X `echo $(( 0x$IP_HEX + 1 ))`)
NEXT_IP=$(printf '%d.%d.%d.%d\n' `echo $NEXT_IP_HEX | sed -r 's/(..)/0x\1 /g'`)
echo "$NEXT_IP"
}
# Update home PC IP
function update_myip() {
myip=$(curl -s4 ifconfig.co) && echo $myip
az network nsg rule update -n SSHInbound --nsg-name $hub_csrext_nsg_name -g $rg \
--protocol 'TCP' --access Allow --priority 1020 --direction Inbound \
--source-address-prefixes "${myip}/32" --source-port-ranges '*' \
--destination-address-prefixes VirtualNetwork --destination-port-ranges '22'
az network route-table route update --route-table-name hubvmrt -g $rg --address-prefix "${myip}/32" --name mypc
az network route-table route update --route-table-name spokevmrt -g $rg --address-prefix "${myip}/32" --name mypc
}
#####################
# Hub VNet with ARS #
#####################
# Create Vnet
echo "Creating RG and VNet..."
az group create -n $rg -l $location -o none
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix --subnet-name $rs_subnet_name --subnet-prefix $rs_subnet_prefix -o none
# Create additional subnets (no subnet can be created while the route server is being provisioned, same as VNGs)
az network vnet subnet create -n $hub_csrext_subnet_name --address-prefix $hub_csrext_subnet_prefix --vnet-name $vnet_name -g $rg -o none
az network vnet subnet create -n $hub_csrint_subnet_name --address-prefix $hub_csrint_subnet_prefix --vnet-name $vnet_name -g $rg -o none
az network vnet subnet create -n $hub_vm_subnet_name --address-prefix $hub_vm_subnet_prefix --vnet-name $vnet_name -g $rg -o none
az network vnet subnet create -n GatewaySubnet --address-prefix $gw_subnet_prefix --vnet-name $vnet_name -g $rg -o none
# Create Route Server
echo "Creating Route Server..."
rs_subnet_id=$(az network vnet subnet show -n $rs_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
rs_pip_name="${rs_name}-pip"
az network public-ip create -g $rg -n $rs_pip_name --allocation-method Static --sku Standard -o none
az network routeserver create -n $rs_name -g $rg --hosted-subnet $rs_subnet_id -l $location --public-ip-address $rs_pip_name -o none
az network routeserver update -n $rs_name -g $rg --allow-b2b-traffic true -o none # Optional
# az network routeserver delete -n $rs_name -g $rg -y # Danger Zone!
# Get info (once created)
rs_ip1=$(az network routeserver show -n $rs_name -g $rg --query 'virtualRouterIps[0]' -o tsv) && echo $rs_ip1
rs_ip2=$(az network routeserver show -n $rs_name -g $rg --query 'virtualRouterIps[1]' -o tsv) && echo $rs_ip2
rs_asn=$(az network routeserver show -n $rs_name -g $rg --query 'virtualRouterAsn' -o tsv) && echo $rs_asn
# Create test VM in hub
az vm create -n hubvm -g $rg -l $location --image Ubuntu2204 --generate-ssh-keys \
--public-ip-address hubvm-pip --vnet-name $vnet_name --size Standard_B1s --subnet $hub_vm_subnet_name -o none
hub_vm_ip=$(az network public-ip show -n hubvm-pip --query ipAddress -o tsv -g $rg) && echo $hub_vm_ip
hub_vm_nic_id=$(az vm show -n hubvm -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv) && echo $hub_vm_nic_id
hub_vm_private_ip=$(az network nic show --ids $hub_vm_nic_id --query 'ipConfigurations[0].privateIPAddress' -o tsv) && echo $hub_vm_private_ip
####################
# CSR #
####################
# Default gateways
if [[ "$nva_type" == "csr" ]]; then
hub_csrext_default_gw=$(first_ip $hub_csrext_subnet_prefix) && echo $hub_csrext_default_gw
hub_csrint_default_gw=$(first_ip $hub_csrint_subnet_prefix) && echo $hub_csrint_default_gw
# Create hub CSR1 as NVA with 2 NICs (easier for NAT)
version=$(az vm image list -p $publisher -f $offer -s $sku --all --query '[0].version' -o tsv)
az vm image terms accept --urn ${publisher}:${offer}:${sku}:${version} -o none
# NSGs (104.21.25.86 and 172.67.133.228 are the addresses of ifconfig.co)
az network nsg create -n $hub_csrext_nsg_name -g $rg -l $location -o none
myip=$(curl -s4 ifconfig.co) && echo $myip
az network nsg rule create -n Internet2VnetInbound --nsg-name $hub_csrext_nsg_name -g $rg \
--protocol '*' --access Allow --priority 1010 --direction Inbound \
--source-address-prefixes 8.8.8.8/32 104.21.25.86/32 172.67.133.228/32 --source-port-ranges '*' \
--destination-address-prefixes VirtualNetwork --destination-port-ranges '*' -o none
az network nsg rule create -n SSHInbound --nsg-name $hub_csrext_nsg_name -g $rg \
--protocol 'TCP' --access Allow --priority 1020 --direction Inbound \
--source-address-prefixes "${myip}/32" --source-port-ranges '*' \
--destination-address-prefixes VirtualNetwork --destination-port-ranges '22' -o none
az network nsg create -n $hub_csrint_nsg_name -g $rg -l $location -o none
az network nsg rule create -n Vnet2InternetInbound --nsg-name $hub_csrint_nsg_name -g $rg \
--protocol '*' --access Allow --priority 1010 --direction Inbound \
--source-address-prefixes VirtualNetwork --source-port-ranges '*' \
--destination-address-prefixes Internet --destination-port-ranges '*' -o none
az network nsg rule create -n Internet2VnetOutbound --nsg-name $hub_csrint_nsg_name -g $rg \
--protocol '*' --access Allow --priority 1010 --direction Outbound \
--source-address-prefixes 8.8.8.8/32 104.21.25.86/32 172.67.133.228/32 --source-port-ranges '*' \
--destination-address-prefixes VirtualNetwork --destination-port-ranges '*' -o none
# PIP
az network public-ip create -g $rg -n "${hub_csr1_name}-pip" --sku basic --allocation-method Static -o none
# NICs
az network nic create -n "${hub_csr1_name}-nic0" -g $rg --vnet-name $vnet_name --subnet $hub_csrext_subnet_name --network-security-group "$hub_csrext_nsg_name" --public-ip-address "${hub_csr1_name}-pip" --ip-forwarding -o none
az network nic create -n "${hub_csr1_name}-nic1" -g $rg --vnet-name $vnet_name --subnet $hub_csrint_subnet_name --network-security-group "$hub_csrint_nsg_name" --private-ip-address $hub_csr1_bgp_ip --ip-forwarding -o none
# VM
az vm create -n $hub_csr1_name -g $rg -l $location \
--image ${publisher}:${offer}:${sku}:${version} --size $csr_size \
--admin-username "$csr_username" --admin-password $csr_password --authentication-type all --generate-ssh-keys \
--nics "${hub_csr1_name}-nic0" "${hub_csr1_name}-nic1" -o none
#hub_csr1_nic_id=$(az vm show -n $hub_csr1_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
#az network nic update --ids $hub_csr1_nic_id --ip-forwarding
# Route table without gateway route propagation on the external nic
az network route-table create -n csrext -g $rg -l $location --disable-bgp-route-propagation -o none
az network vnet subnet update --name $hub_csrext_subnet_name --route-table csrext --vnet-name $vnet_name --resource-group $rg -o none
# Test access over SSH
hub_csr1_ip=$(az network public-ip show -n "${hub_csr1_name}-pip" --query ipAddress -o tsv -g $rg) && echo $hub_csr1_ip
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" "sh ip int b"
# Configure hub CSR1 with BGP
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
no ip domain lookup
interface Loopback 0
ip address 1.1.1.1 255.255.255.255
router bgp ${hub_csr_asn}
network 1.1.1.1 mask 255.255.255.255
neighbor $rs_ip1 remote-as $rs_asn
neighbor $rs_ip1 ebgp-multihop 2
neighbor $rs_ip1 update-source GigabitEthernet2
neighbor $rs_ip1 timers 5 15
neighbor $rs_ip2 remote-as $rs_asn
neighbor $rs_ip2 ebgp-multihop 2
neighbor $rs_ip2 update-source GigabitEthernet2
neighbor $rs_ip2 timers 5 15
ip route 10.0.0.0 255.0.0.0 $hub_csrint_default_gw
ip route $vnet_prefix_long $hub_csrint_default_gw
ip route 172.16.0.0 255.240.0.0 $hub_csrint_default_gw
ip route 192.168.0.0 255.255.0.0 $hub_csrint_default_gw
ip route $rs_ip1 255.255.255.255 $hub_csrint_default_gw
ip route $rs_ip2 255.255.255.255 $hub_csrint_default_gw
end
wr mem
EOF
# Route Server is reachable over ICMP :)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr1_ip "ping $rs_ip1"
# Create peering
az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $hub_csr1_bgp_ip --peer-asn $hub_csr_asn -n $hub_csr1_name -o none
# Show peerings and effective routes on the VM nic (at this point only the 1.1.1.1 should be there)
az network routeserver peering list --routeserver $rs_name -g $rg -o table
az network nic show-effective-route-table --ids $hub_vm_nic_id -o table
# OPTIONAL: Inject default route from CSR1, and configure NAT
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
interface GigabitEthernet1
ip nat outside
interface GigabitEthernet2
ip nat inside
access-list 100 remark SOURCES TO BE NATTED
access-list 100 deny ip $hub_csrext_subnet_prefix_long_reversemask any
access-list 100 permit ip 10.0.0.0 0.255.255.255 any
ip nat inside source list 100 interface GigabitEthernet1 overload
ip route 0.0.0.0 0.0.0.0 $hub_csrext_default_gw
ip route 0.0.0.0 128.0.0.0 $hub_csrext_default_gw
ip route 128.0.0.0 128.0.0.0 $hub_csrext_default_gw
router bgp ${hub_csr_asn}
network 0.0.0.0 mask 0.0.0.0
network 0.0.0.0 mask 128.0.0.0
network 128.0.0.0 mask 128.0.0.0
default-information originate
end
wr mem
EOF
# Bypass the NVA for SSH access
az network route-table create -n hubvmrt -g $rg -l $location
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $hub_vm_subnet_name --route-table hubvmrt
myip=$(curl -s4 ifconfig.co) && echo $myip
az network route-table route create --route-table-name hubvmrt -g $rg --address-prefix "${myip}/32" --name mypc --next-hop-type Internet
# Test egress access from the VM
az network routeserver peering list-learned-routes -n $hub_csr1_name --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table
az network nic show-effective-route-table --ids $hub_vm_nic_id -o table
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_vm_ip" "curl -s4 ifconfig.co"
# Measure route injection time
# Create route in CSR
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
interface Loopback 10
ip address 10.10.10.10 255.255.255.255
router bgp ${hub_csr_asn}
network 10.10.10.10 mask 255.255.255.255
end
EOF
start_time=$(date +%s)
echo "Interface created in CSR, now waiting for route to appear..."
route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
until [[ -n "$route" ]]
do
route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
done
finish_time=$(date +%s)
run_time=$(($finish_time - $start_time))
((minutes=run_time/60))
((seconds=run_time%60))
echo "Route appeared in the effective route table of the hub VM after $minutes minutes and $seconds seconds"
# Result: around 20-45s
# Measure route removal time
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
no interface Loopback10
router bgp ${hub_csr_asn}
no network 10.10.10.10 mask 255.255.255.255
end
EOF
start_time=$(date +%s)
echo "Interface deleted in CSR, now waiting for route to disappear..."
route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
until [[ -z "$route" ]]
do
route=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("10.10.10.10/32"))) | .[]' 2>/dev/null)
done
finish_time=$(date +%s)
run_time=$(($finish_time - $start_time))
((minutes=run_time/60))
((seconds=run_time%60))
echo "Route disappeared from the effective route table of the hub VM after $minutes minutes and $seconds seconds"
# Result: around 20-30s
# CSR2
if [[ "$redundant_nva" == "yes" ]]; then
# PIP
az network public-ip create -g $rg -n "${hub_csr2_name}-pip" --sku basic --allocation-method Static
# NICs
az network nic create -n "${hub_csr2_name}-nic0" -g $rg --vnet-name $vnet_name --subnet $hub_csrext_subnet_name --network-security-group "$hub_csrext_nsg_name" --public-ip-address "${hub_csr2_name}-pip" --ip-forwarding
az network nic create -n "${hub_csr2_name}-nic1" -g $rg --vnet-name $vnet_name --subnet $hub_csrint_subnet_name --network-security-group "$hub_csrint_nsg_name" --private-ip-address $hub_csr2_bgp_ip --ip-forwarding
# VM
az vm create -n $hub_csr2_name -g $rg -l $location \
--image ${publisher}:${offer}:${sku}:${version} \
--admin-username "$csr_username" --admin-password $csr_password --authentication-type all --generate-ssh-keys \
--nics "${hub_csr2_name}-nic0" "${hub_csr2_name}-nic1"
# Configure hub CSR2 with BGP
hub_csr2_ip=$(az network public-ip show -n "${hub_csr2_name}-pip" --query ipAddress -o tsv -g $rg) && echo $hub_csr2_ip
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr2_ip" "sh ip int b"
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr2_ip" <<EOF
config t
no ip domain lookup
interface Loopback0
ip address 2.2.2.2 255.255.255.255
interface GigabitEthernet1
ip nat outside
interface GigabitEthernet2
ip nat inside
access-list 100 remark SOURCES TO BE NATTED
access-list 100 deny ip $hub_csrext_subnet_prefix_long_reversemask any
access-list 100 permit ip 10.0.0.0 0.255.255.255 any
ip nat inside source list 100 interface GigabitEthernet1 overload
router bgp ${hub_csr_asn}
default-information originate
network 0.0.0.0 mask 0.0.0.0
network 0.0.0.0 mask 128.0.0.0
network 128.0.0.0 mask 128.0.0.0
network 2.2.2.2 mask 255.255.255.255
neighbor $rs_ip1 remote-as $rs_asn
neighbor $rs_ip1 ebgp-multihop 2
neighbor $rs_ip1 update-source GigabitEthernet2
neighbor $rs_ip1 route-map ToRS out
neighbor $rs_ip1 timers 5 15
neighbor $rs_ip2 remote-as $rs_asn
neighbor $rs_ip2 ebgp-multihop 2
neighbor $rs_ip2 update-source GigabitEthernet2
neighbor $rs_ip2 route-map ToRS out
neighbor $rs_ip2 timers 5 15
route-map ToRS
set as-path prepend ${hub_csr_asn} ${hub_csr_asn}
ip route 0.0.0.0 0.0.0.0 $hub_csrext_default_gw
ip route 0.0.0.0 128.0.0.0 $hub_csrext_default_gw
ip route 128.0.0.0 128.0.0.0 $hub_csrext_default_gw
ip route 10.0.0.0 255.0.0.0 $hub_csrint_default_gw
ip route 172.16.0.0 255.240.0.0 $hub_csrint_default_gw
ip route 192.168.0.0 255.255.0.0 $hub_csrint_default_gw
ip route $rs_ip1 255.255.255.255 $hub_csrint_default_gw
ip route $rs_ip2 255.255.255.255 $hub_csrint_default_gw
end
wr mem
EOF
# Route Server is reachable over ICMP
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr2_ip "ping $rs_ip1"
# Create peering
az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $hub_csr2_bgp_ip --peer-asn $hub_csr_asn -n $hub_csr2_name
# az network routeserver peering delete --routeserver $rs_name -g $rg -n $hub_csr2_name -y -o none # In case you need to recreate the BGP peering
# Verify that the RS is learning the routes from both CSRs
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr1_ip "sh ip bgp summ"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr2_ip "sh ip bgp summ"
az network routeserver peering list --routeserver $rs_name -g $rg -o table
az network routeserver peering list-learned-routes -n $hub_csr1_name --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table
az network routeserver peering list-learned-routes -n $hub_csr2_name --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table
az network nic show-effective-route-table --ids $hub_vm_nic_id -o table
# Simulate outage of CSR1 and measure time for CSR2 to take over (looking at the effective routes)
function bring_down_csr1() {
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
int GigabitEthernet2
shutdown
EOF
start_time=$(date +%s)
echo "Internal interface shutdown in CSR1, now waiting for the effective route to switch to CSR2 ($hub_csr2_bgp_ip)..."
nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]') && echo $nexthop
until [[ "$nexthop" == "$hub_csr2_bgp_ip" ]]
do
nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]' 2>/dev/null)
echo "Next hop for 0.0.0.0/0 is $nexthop"
done
finish_time=$(date +%s)
run_time=$(($finish_time - $start_time))
((minutes=run_time/60))
((seconds=run_time%60))
echo "Effective route pointing now to $nexthop after $minutes minutes and $seconds seconds"
}
# Restore CSR1 connectivity
function bring_up_csr1() {
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$hub_csr1_ip" <<EOF
config t
int GigabitEthernet2
no shutdown
EOF
start_time=$(date +%s)
echo "Internal interface restored in CSR1, now waiting for the effective route to switch back to CSR1 ($hub_csr1_bgp_ip)..."
nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]' 2>/dev/null)
until [[ "$nexthop" == "$hub_csr1_bgp_ip" ]]
do
nexthop=$(az network nic show-effective-route-table --ids $hub_vm_nic_id -o json | jq -r '.value | map(select(.addressPrefix[0] | contains("0.0.0.0/0"))) | .[].nextHopIpAddress[]' 2>/dev/null)
echo "Next hop for 0.0.0.0/0 is $nexthop"
done
finish_time=$(date +%s)
run_time=$(($finish_time - $start_time))
((minutes=run_time/60))
((seconds=run_time%60))
echo "Effective route pointing now to $nexthop after $minutes minutes and $seconds seconds"
}
bring_down_csr1
bring_up_csr1
fi
fi
################
# Spokes #
################
if [[ "$create_spokes" == "yes" ]]; then
# Create RT to bypass the NVA for SSH connections from the local PC
echo "Creating route table..."
az network route-table create -n spokevmrt -g $rg -l $location -o none
myip=$(curl -s4 ifconfig.co) && echo $myip
az network route-table route create --route-table-name spokevmrt -g $rg --address-prefix "${myip}/32" --name mypc --next-hop-type Internet -o none
# Spoke1 with Ubuntu VM
echo "Creating spoke 1..."
az network vnet create -g $rg -n $spoke1_name --address-prefix $spoke1_prefix --subnet-name $spoke1_vm_subnet_name --subnet-prefix $spoke1_vm_subnet_prefix -l $location -o none
az network vnet subnet create -n $spoke1_csr_subnet_name --address-prefix $spoke1_csr_subnet_prefix --vnet-name $spoke1_name -g $rg -o none
az network vnet subnet update -g $rg --vnet-name $spoke1_name -n $spoke1_vm_subnet_name --route-table spokevmrt -o none
# az network vnet peering delete -n hubtospoke1 -g $rg --vnet-name $vnet_name
# az network vnet peering delete -n spoke1tohub -g $rg --vnet-name $spoke1_name
az network vnet peering create -n hubtospoke1 -g $rg --vnet-name $vnet_name --remote-vnet $spoke1_name --allow-vnet-access --allow-forwarded-traffic --allow-gateway-transit -o none
az network vnet peering create -n spoke1tohub -g $rg --vnet-name $spoke1_name --remote-vnet $vnet_name --allow-vnet-access --allow-forwarded-traffic --use-remote-gateways -o none
az vm create -n spoke1-vm -g $rg -l $location --image Ubuntu2204 --generate-ssh-keys \
--public-ip-address spoke1-vm-pip --vnet-name $spoke1_name --size Standard_B1s --subnet $spoke1_vm_subnet_name -o none
spoke1_ip=$(az network public-ip show -n spoke1-vm-pip --query ipAddress -o tsv -g $rg) && echo $spoke1_ip
spoke1_nic_id=$(az vm show -n spoke1-vm -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
spoke1_private_ip=$(az network nic show --ids $spoke1_nic_id --query 'ipConfigurations[0].privateIPAddress' -o tsv) && echo $spoke1_private_ip
# Spoke 2 with Ubuntu VM
echo "Creating spoke 2..."
az network vnet create -g $rg -n $spoke2_name --address-prefix $spoke2_prefix --subnet-name $spoke2_vm_subnet_name --subnet-prefix $spoke2_vm_subnet_prefix -l $location -o none
az network vnet subnet update -g $rg --vnet-name $spoke2_name -n $spoke2_vm_subnet_name --route-table spokevmrt -o none
az network vnet peering create -n hubtospoke2 -g $rg --vnet-name $vnet_name --remote-vnet $spoke2_name --allow-vnet-access --allow-forwarded-traffic --allow-gateway-transit -o none
az network vnet peering create -n spoke2tohub -g $rg --vnet-name $spoke2_name --remote-vnet $vnet_name --allow-vnet-access --allow-forwarded-traffic --use-remote-gateways -o none
az vm create -n spoke2-vm -g $rg -l $location --image Ubuntu2204 --generate-ssh-keys \
--public-ip-address spoke2-vm-pip --vnet-name $spoke2_name --size Standard_B1s --subnet $spoke2_vm_subnet_name -o none
spoke2_ip=$(az network public-ip show -n spoke2-vm-pip --query ipAddress -o tsv -g $rg) && echo $spoke2_ip
spoke2_nic_id=$(az vm show -n spoke2-vm -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
spoke2_private_ip=$(az network nic show --ids $spoke2_nic_id --query 'ipConfigurations[0].privateIPAddress' -o tsv) && echo $spoke2_private_ip
# Effective routes
az network nic show-effective-route-table --ids $spoke1_nic_id -o table
az network nic show-effective-route-table --ids $spoke2_nic_id -o table
if [["$create_remote_spoke" == "yes" ]]; then
# Spoke 3 with Ubuntu VM (remote)
echo "Creating route table in $remote_location..."
az network route-table create -n remotespokevmrt -g $rg -l $remote_location -o none
myip=$(curl -s4 ifconfig.co) && echo $myip
az network route-table route create --route-table-name remotespokevmrt -g $rg --address-prefix "${myip}/32" --name mypc --next-hop-type Internet -o none
echo "Creating spoke 3 in $remote_location..."
az network vnet create -g $rg -n $spoke3_name --address-prefix $spoke3_prefix --subnet-name $spoke3_vm_subnet_name --subnet-prefix $spoke3_vm_subnet_prefix -l $remote_location -o none
az network vnet subnet update -g $rg --vnet-name $spoke3_name -n $spoke3_vm_subnet_name --route-table remotespokevmrt -o none
az network vnet peering create -n hubtospoke3 -g $rg --vnet-name $vnet_name --remote-vnet $spoke3_name --allow-vnet-access --allow-forwarded-traffic --allow-gateway-transit -o none
az network vnet peering create -n spoke3tohub -g $rg --vnet-name $spoke3_name --remote-vnet $vnet_name --allow-vnet-access --allow-forwarded-traffic --use-remote-gateways -o none
az vm create -n spoke3-vm -g $rg -l $remote_location --image Ubuntu2204 --generate-ssh-keys \
--public-ip-address spoke3-vm-pip --vnet-name $spoke3_name --size Standard_B1s --subnet $spoke3_vm_subnet_name -o none
fi
# Check routing on NVA
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_csr1_ip "show ip route static"
# Connectivity from spoke to NVA (1.1.1.1 is a loopback on CSR1, advertised to ARS via BGP)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke1_ip "ping 1.1.1.1 -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke2_ip "ping 1.1.1.1 -c 5"
# Verify spoke-to-spoke connectivity
# ICMP
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke1_ip "ping $spoke2_private_ip -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke2_ip "ping $spoke1_private_ip -c 5"
# SSH from spoke to spoke (using one of them as jump host)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no -J $spoke1_ip $spoke2_private_ip "ip a"
# CSR in spoke1 (using the hub NSG, I am lazy...)
az network route-table create -n spoke1csr -g $rg -l $location --disable-bgp-route-propagation -o none
az network vnet subnet update -g $rg --vnet-name $spoke1_name -n $spoke1_csr_subnet_name --route-table spoke1csr -o none # To prevent ARS routes to be injected here
az network public-ip create -g $rg -n "${spoke1_csr_name}-pip" --sku basic --allocation-method Static -o none
az network nic create -n "${spoke1_csr_name}-nic0" -g $rg --vnet-name $spoke1_name --subnet $spoke1_csr_subnet_name --public-ip-address "${spoke1_csr_name}-pip" --network-security-group "$hub_csrext_nsg_name" --ip-forwarding -o none
az vm create -n $spoke1_csr_name -g $rg -l $location \
--image ${publisher}:${offer}:${sku}:${version} \
--admin-username "$csr_username" --admin-password $csr_password --authentication-type all --generate-ssh-keys \
--nics "${spoke1_csr_name}-nic0" -o none
# Configure spoke1 CSR with BGP
spoke1_csr_ip=$(az network public-ip show -n "${spoke1_csr_name}-pip" --query ipAddress -o tsv -g $rg) && echo $spoke1_csr_ip
spoke1_csr_default_gw=$(first_ip $spoke1_csr_subnet_prefix) && echo $spoke1_csr_default_gw
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$spoke1_csr_ip" "sh ip int b"
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "$spoke1_csr_ip" <<EOF
config t
no ip domain lookup
interface Loopback0
ip address 22.22.22.22 255.255.255.255
router bgp ${spoke1_csr_asn}
network 22.22.22.22 mask 255.255.255.255
neighbor $rs_ip1 remote-as $rs_asn
neighbor $rs_ip1 ebgp-multihop 2
neighbor $rs_ip1 update-source GigabitEthernet1
neighbor $rs_ip1 timers 5 15
neighbor $rs_ip2 remote-as $rs_asn
neighbor $rs_ip2 ebgp-multihop 2
neighbor $rs_ip2 update-source GigabitEthernet1
neighbor $rs_ip2 timers 5 15
ip route $rs_ip1 255.255.255.255 $hub_csrint_default_gw
ip route $rs_ip2 255.255.255.255 $hub_csrint_default_gw
end
wr mem
EOF
# Add BGP peering to ARS
spoke1_csr_private_ip=$(az network nic show -n "${spoke1_csr_name}-nic0" -g $rg --query 'ipConfigurations[0].privateIPAddress' -o tsv) && echo $spoke1_csr_private_ip
az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $spoke1_csr_private_ip --peer-asn $spoke1_csr_asn -n $spoke1_csr_name -o none
# Since the NVA is in spoke1, you want to vnet-peer spoke1 and spoke2, otherwise the effective route in spoke2 would show up as none
az network vnet peering create -n spoke2tospoke1 -g $rg --vnet-name $spoke2_name --remote-vnet $spoke1_name --allow-vnet-access --allow-forwarded-traffic -o none
az network vnet peering create -n spoke1tospoke1 -g $rg --vnet-name $spoke1_name --remote-vnet $spoke2_name --allow-vnet-access --allow-forwarded-traffic -o none
fi
####################
# Linux NVA in hub #
####################
if [[ "$redundant_nva" == "yes" ]]; then
nva_no=2
else
nva_no=1
fi
for nva_id in $(seq 1 $nva_no); do
if [[ "$nva_type" == "linux" ]]; then
# Cloud init file to install StrongSwan (not used in this lab) and BIRD 1.6
cat <<EOF > $hub_linuxnva_cloudinit_file
#cloud-config
runcmd:
- apt update && apt install -y bird strongswan libcharon-extra-plugins strongswan-pki net-tools
- sysctl -w net.ipv4.ip_forward=1
- sudo iptables -t nat -A POSTROUTING ! -d '10.0.0.0/8' -o eth0 -j MASQUERADE
EOF
echo "Creating Linux NVA ${hub_linuxnva_subnet_name}-${nva_id}..."
az network vnet subnet create --vnet-name $vnet_name --name $hub_linuxnva_subnet_name -g $rg --address-prefixes $hub_linuxnva_subnet_prefix -o none
az network route-table create -n hublinuxnva -g $rg -l $location --disable-bgp-route-propagation -o none
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $hub_linuxnva_subnet_name --route-table hublinuxnva -o none
az vm create -n "${hub_linuxnva_subnet_name}-${nva_id}" -g $rg -l $location --image Ubuntu2204 --generate-ssh-keys \
--public-ip-address "${hub_linuxnva_subnet_name}-${nva_id}-pip" --vnet-name $vnet_name --size Standard_B1s --subnet $hub_linuxnva_subnet_name \
--custom-data $hub_linuxnva_cloudinit_file -o none
hub_linuxnva_nic_id=$(az vm show -n "${hub_linuxnva_subnet_name}-${nva_id}" -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic update --ids $hub_linuxnva_nic_id --ip-forwarding -o none
hub_linuxnva_pip_ip=$(az network public-ip show -n "${hub_linuxnva_subnet_name}-${nva_id}-pip" -g $rg --query ipAddress -o tsv) && echo $hub_linuxnva_pip_ip
hub_linuxnva_private_ip=$(az network nic show --ids $hub_linuxnva_nic_id --query 'ipConfigurations[0].privateIPAddress' -o tsv) && echo $hub_linuxnva_private_ip
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "ip a"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show route"
hub_linuxnva_default_gw=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip netstat -rnv | awk '$1 == "0.0.0.0" {print $2}') && echo $hub_linuxnva_default_gw
# RS adjacency
echo "Connecting Linux NVA "${hub_linuxnva_subnet_name}-${nva_id}" on $hub_linuxnva_private_ip to ARS..."
az network routeserver peering create --routeserver $rs_name -g $rg --peer-ip $hub_linuxnva_private_ip --peer-asn $hub_linuxnva_asn -n "${hub_linuxnva_subnet_name}-${nva_id}" -o none
# Bird config file
bird_config_file=/tmp/bird.conf
cat <<EOF > $bird_config_file
log syslog all;
router id $hub_linuxnva_private_ip;
protocol device {
scan time 10; # Scan interfaces every 10 seconds
}
# Disable automatically generating direct routes to all network interfaces.
protocol direct {
#disabled; # Enabled by default
}
# Forbid synchronizing BIRD routing tables with the OS kernel.
protocol kernel {
import all; # Import to table, default is import all
export none; # Export to protocol. default is export none
}
# Static IPv4 routes.
protocol static {
route 5.5.5.5/32 via $hub_linuxnva_default_gw;
route 0.0.0.0/1 via $hub_linuxnva_default_gw;
route 128.0.0.0/1 via $hub_linuxnva_default_gw;
# route $vnet_prefix via $hub_linuxnva_default_gw;
}
# BGP peers
protocol bgp rs0 {
description "RouteServer instance 0";
multihop;
local $hub_linuxnva_private_ip as $hub_linuxnva_asn;
neighbor $rs_ip1 as $rs_asn;
import filter {accept;};
export filter {accept;};
}
protocol bgp rs1 {
description "RouteServer instance 1";
multihop;
local $hub_linuxnva_private_ip as $hub_linuxnva_asn;
neighbor $rs_ip2 as $rs_asn;
import filter {accept;};
export filter {accept;};
}
EOF
echo "Configuring BIRD in Linux NVA "${hub_linuxnva_subnet_name}-${nva_id}" on public IP $hub_linuxnva_pip_ip..."
hub_linuxnva_user=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "whoami") && echo "User: $hub_linuxnva_user"
scp $bird_config_file "${hub_linuxnva_pip_ip}:/home/${hub_linuxnva_user}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo mv /home/${hub_linuxnva_user}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo systemctl restart bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show route"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols all"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols rs0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols rs1"
# Routes advertised to ARS
az network routeserver peering list-learned-routes -n "${hub_linuxnva_subnet_name}-${nva_id}" --routeserver $rs_name -g $rg --query 'RouteServiceRole_IN_0' -o table
fi
done
#################################
# Onprem simulation with VPN/ER #
#################################
if [[ "$er_branch" == "yes" ]]; then
# Create ER and ER Gateway to simulate onprem
az network public-ip create -g $rg -n $ergw_pip --allocation-method Dynamic --sku Basic -o none
az network vnet-gateway create -g $rg -n $ergw_name --gateway-type ExpressRoute --sku Standard -l $location \
--vnet $vnet_name --public-ip-addresses $ergw_pip -o none
az network express-route create -n $er_circuit_name --peering-location $er_pop -g $rg \
--bandwidth 50 Mbps --provider $er_provider -l $er_location --sku-family MeteredData --sku-tier Standard -o none
# After provisioning circuit in Megaport portal continue
echo "Creating Megaport Cloud Router and configuring circuit..."
service_key=$(az network express-route show -n $er_circuit_name -g $rg --query serviceKey -o tsv)
$megaport_script_path -s=jomore-${er_pop} -a=create_mcr -k=$service_key --asn=$mcr_asn
sleep 60 # Wait 1 minute before creating the connections. This could be replaced with a loop checking ./megaport.sh -a=list_live
$megaport_script_path -s=jomore-${er_pop} -a=create_vxc -k=$service_key
# Connect circuit to VNet
circuit_id=$(az network express-route show -n $er_circuit_name -g $rg -o tsv --query id) && echo $circuit_id
az network vpn-connection create -n erconnection -g $rg --vnet-gateway1 $ergw_name --express-route-circuit2 $circuit_id
# Create GCP environment to simulate onprem
# Get environment info
account=$(gcloud info --format json | jq -r '.config.account')
billing_account=$(gcloud beta billing accounts list --format json | jq -r '.[0].name')
billing_account_short=$(echo "$billing_account" | cut -f 2 -d/)
# Create project
gcloud projects create $project_id --name $project_name
gcloud config set project $project_id
gcloud beta billing projects link "$project_id" --billing-account "$billing_account_short"
gcloud services enable compute.googleapis.com
# VPC and instance
gcloud compute networks create "$gcp_vpc_name" --bgp-routing-mode=regional --mtu=1500 --subnet-mode=custom
gcloud compute networks subnets create "$gcp_subnet_name" --network "$gcp_vpc_name" --range "$gcp_subnet_prefix" --region=$region1
gcloud compute instances create "$gcp_vm_name" --image-family=ubuntu-2004-lts --image-project=ubuntu-os-cloud --machine-type "$machine_type" --network "$gcp_vpc_name" --subnet "$gcp_subnet_name" --zone "$zone"
gcloud compute firewall-rules create "${gcp_vpc_name}-allow-icmp" --network "$gcp_vpc_name" --priority=1000 --direction=INGRESS --rules=icmp --source-ranges=0.0.0.0/0 --action=ALLOW
gcloud compute firewall-rules create "${gcp_vpc_name}-allow-ssh" --network "$gcp_vpc_name" --priority=1010 --direction=INGRESS --rules=tcp:22 --source-ranges=0.0.0.0/0 --action=ALLOW
gcloud compute firewall-rules create "${gcp_vpc_name}-allow-web" --network "$gcp_vpc_name" --priority=1020 --direction=INGRESS --rules=tcp:80 --source-ranges=192.168.0.0/16 --action=ALLOW
# gcloud compute ssh $gcp_vm_name --zone=$zone --command="ip a" # This command will pause the script if the key file is password-protected
# Create interconnect
gcloud compute routers create $router_name --project=$project_id --network=$gcp_vpc_name --asn=$gcp_asn --region=$region
gcloud compute interconnects attachments partner create $attachment_name --region $region --router $router_name --edge-availability-domain availability-domain-1
pairing_key=$(gcloud compute interconnects attachments describe $attachment_name --region $region --format json | jq -r '.pairingKey')
# Create VXC in Megaport
$megaport_script_path -g -s=jomore-${er_pop} -a=create_vxc -k=$pairing_key
# Activate attachment
# wait_for_gcp_attachment_ready $attachment_name $region1
sleep 120
gcloud compute interconnects attachments partner update $attachment_name --region $region --admin-enabled
fi
if [[ "$vpn_branch" == "yes" ]] && [[ "$onprem_nva_type" == "linux" ]]; then
# Initialization
az network vnet create -n $onprem_vnet_name -g $rg --address-prefixes $onprem_vnet_prefix --subnet-name $onprem_nva_subnet_name --subnet-prefixes $onprem_nva_subnet_prefix
az network vnet subnet create -g $rg --vnet-name $onprem_vnet_name -n GatewaySubnet --address-prefix $onprem_gw_subnet_prefix
az network route-table create -n onpremnva -g $rg -l $location --disable-bgp-route-propagation
az network vnet subnet update -g $rg --vnet-name $onprem_vnet_name -n $onprem_nva_subnet_name --route-table onpremnva
# Create onprem Linux NVA
onprem_linuxnva_cloudinit_file=/tmp/linuxnva_cloudinit.txt
cat <<EOF > $onprem_linuxnva_cloudinit_file
#cloud-config
runcmd:
- apt update && apt install -y bird strongswan
- sysctl -w net.ipv4.ip_forward=1
- sysctl -w net.ipv4.conf.all.accept_redirects = 0
- sysctl -w net.ipv4.conf.all.send_redirects = 0
EOF
az vm create -n $onprem_linuxnva_name -g $rg -l $location --image Ubuntu2204 --generate-ssh-keys \
--public-ip-address $onprem_linuxnva_pip --vnet-name $onprem_vnet_name --size Standard_B1s --subnet $onprem_nva_subnet_name \
--custom-data $onprem_linuxnva_cloudinit_file --private-ip-address "$onprem_linuxnva_ip"
onprem_linuxnva_nic_id=$(az vm show -n $onprem_linuxnva_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic update --ids $onprem_linuxnva_nic_id --ip-forwarding
onprem_linuxnva_pip_ip=$(az network public-ip show -n $onprem_linuxnva_pip -g $rg --query ipAddress -o tsv) && echo $onprem_linuxnva_pip_ip
onprem_linuxnva_private_ip=$(az network nic show --ids $onprem_linuxnva_nic_id --query 'ipConfigurations[0].privateIPAddress' -o tsv) && echo $onprem_linuxnva_private_ip
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ip a"
onprem_linuxnva_default_gw=$(ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip netstat -rnv | awk '$1 == "0.0.0.0" {print $2}') && echo $onprem_linuxnva_default_gw
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show route"
# Configure StrongSwan VPN to the hub Linux NVA
# See https://blog.sys4.de/routing-based-vpn-with-strongswan-de.html
# See https://wiki.strongswan.org/projects/strongswan/wiki/RouteBasedVPN
echo "Configuring VPN between A:${linuxnva_pip_ip}/${linuxnva_private_ip} and B:${onprem_linuxnva_pip_ip}/${onprem_linuxnva_private_ip}"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ip tunnel add vti0 local $hub_linuxnva_private_ip remote $onprem_linuxnva_pip_ip mode vti key 12"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo sysctl -w net.ipv4.conf.vti0.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ip link set up dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ip route add $onprem_linuxnva_private_ip/32 dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip tunnel add vti0 local $onprem_linuxnva_private_ip remote $hub_linuxnva_pip_ip mode vti key 12"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo sysctl -w net.ipv4.conf.vti0.disable_policy=1"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip link set up dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ip route add $hub_linuxnva_private_ip/32 dev vti0"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf"
vpn_psk=$(openssl rand -base64 64)
vpn_psk=${vpn_psk//$'\n'/} # Remove line breaks
psk_file_a=/tmp/ipsec.secrets.a
psk_file_b=/tmp/ipsec.secrets.b
cat <<EOF > $psk_file_a
$hub_linuxnva_pip_ip $onprem_linuxnva_pip_ip : PSK "$vpn_psk"
EOF
cat <<EOF > $psk_file_b
$onprem_linuxnva_pip_ip $hub_linuxnva_pip_ip : PSK "$vpn_psk"
EOF
ipsec_file_a=/tmp/ipsec.conf.a
ipsec_file_b=/tmp/ipsec.conf.b
cat <<EOF > $ipsec_file_a
config setup
charondebug="all"
uniqueids=yes
strictcrlpolicy=no
conn to-onprem
authby=secret
leftid=$hub_linuxnva_pip_ip
leftsubnet=0.0.0.0/0
right=$onprem_linuxnva_pip_ip
rightsubnet=0.0.0.0/0
ike=aes256-sha2_256-modp1024!
esp=aes256-sha2_256!
keyingtries=0
ikelifetime=1h
lifetime=8h
dpddelay=30
dpdtimeout=120
dpdaction=restart
auto=start
mark=12
EOF
cat <<EOF > $ipsec_file_b
config setup
charondebug="all"
uniqueids=yes
strictcrlpolicy=no
conn to-azure
authby=secret
leftid=$onprem_linuxnva_pip_ip
leftsubnet=0.0.0.0/0
right=$hub_linuxnva_pip_ip
rightsubnet=0.0.0.0/0
ike=aes256-sha2_256-modp1024!
esp=aes256-sha2_256!
keyingtries=0
ikelifetime=1h
lifetime=8h
dpddelay=30
dpdtimeout=120
dpdaction=restart
auto=start
mark=12
EOF
username=$(whoami)
scp $psk_file_a $hub_linuxnva_pip_ip:/home/$username/ipsec.secrets
scp $psk_file_b $onprem_linuxnva_pip_ip:/home/$username/ipsec.secrets
scp $ipsec_file_a $hub_linuxnva_pip_ip:/home/$username/ipsec.conf
scp $ipsec_file_b $onprem_linuxnva_pip_ip:/home/$username/ipsec.conf
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo mv ./ipsec.* /etc/"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo systemctl restart ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo mv ./ipsec.* /etc/"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo systemctl restart ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status ipsec"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo ipsec status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo ipsec status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "ping $onprem_linuxnva_private_ip -c 5"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "ping $hub_linuxnva_private_ip -c 5"
# Configure BGP onprem with Bird
bird_config_file=/tmp/bird.conf
cat <<EOF > $bird_config_file
log syslog all;
router id $onprem_linuxnva_private_ip;
protocol device {
scan time 10;
}
protocol direct {
#disabled;
}
protocol kernel {
import all;
export none;
}
protocol static {
route 6.6.6.6/32 via $onprem_linuxnva_default_gw;
route $onprem_vnet_prefix via $onprem_linuxnva_default_gw;
}
protocol bgp azurenva {
description "BGP to Azure";
multihop;
local $onprem_linuxnva_private_ip as $onprem_linuxnva_asn;
neighbor $hub_linuxnva_private_ip as $hub_linuxnva_asn;
import filter {accept;};
export filter {accept;};
}
EOF
scp $bird_config_file "${onprem_linuxnva_pip_ip}:/home/${hub_linuxnva_user}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo mv /home/${hub_linuxnva_user}/bird.conf /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo systemctl restart bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $onprem_linuxnva_pip_ip "sudo birdc show route"
# Configure BGP on Azure
bird_config_file=/tmp/bird.conf
cat <<EOF > $bird_config_file
protocol bgp onprem {
description "BGP to Onprem";
multihop;
local $hub_linuxnva_private_ip as $hub_linuxnva_asn;
neighbor $onprem_linuxnva_private_ip as $onprem_linuxnva_asn;
import filter {accept;};
export filter {accept;};
}
EOF
scp $bird_config_file "${hub_linuxnva_pip_ip}:/home/${username}/bird.conf.new"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "cat /home/${hub_linuxnva_user}/bird.conf.new"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo cat /etc/bird/bird.conf /home/${hub_linuxnva_user}/bird.conf.new >/home/${hub_linuxnva_user}/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo mv /home/${hub_linuxnva_user}/bird.conf /etc/bird/bird.conf"
# ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo more /etc/bird/bird.conf"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo systemctl restart bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "systemctl status bird"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show status"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show protocols"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $hub_linuxnva_pip_ip "sudo birdc show route"
# Check effective routes (6.6.6.6 should be there)
az network nic show-effective-route-table --ids $hub_vm_nic_id -o table
az network nic show-effective-route-table --ids $spoke1_vm_nic_id -o table
$spoke1_
# Ping onprem's IP
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $spoke1_ip "ping $onprem_linuxnva_private_ip -c 5"
fi
# Create onprem CSR (WORK IN PROGRESS)
if [[ "$vpn_branch" == "yes" ]] && [[ "$onprem_nva_type" == "csr" ]]; then
nva_size=Standard_B2ms
onprem_csr_ip=192.168.0.10
publisher=cisco
offer=cisco-csr-1000v
sku=16_12-byol
version=$(az vm image list -p $publisher -f $offer -s $sku --all --query '[0].version' -o tsv 2>/dev/null)
az vm create -n onprem-csr -g "$rg" -l "$location" --image "${publisher}:${offer}:${sku}:${version}" --size "$nva_size" \
--generate-ssh-keys --public-ip-address "onprem-csr-pip" --public-ip-address-allocation static --public-ip-sku Standard \
--vnet-name "$onprem_vnet_name" --subnet "$onprem_nva_subnet_name" \
--private-ip-address "$onprem_csr_ip"
# Configure onprem CSR for VPN
onprem_default_gw=192.168.0.1
vpngw_pip_ip=$(az network public-ip show -n $vpngw_pip -g $rg --query ipAddress -o tsv) && echo $vpngw_pip_ip
vpngw_bgp_ip=$(az network vnet-gateway show -n $vpngw_name -g $rg --query bgpSettings.bgpPeeringAddresses[0].tunnelIpAddresses[0]) && echo $vpngw_bgp_ip
vpngw_bgp_asn=$(az network vnet-gateway show -n $vpngw_name -g $rg --query bgpSettings.asn) && echo $vpngw_bgp_asn
onprem_asn=65200
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o KexAlgorithms=+diffie-hellman-group14-sha1 "$onprem_csr_ip"
config t
crypto ikev2 proposal azure-proposal
encryption aes-cbc-256 aes-cbc-128 3des
integrity sha1
group 2
crypto ikev2 policy azure-policy
proposal azure-proposal
crypto ikev2 profile azure-profile
match address local interface GigabitEthernet1
authentication remote pre-share
authentication local pre-share
keyring local azure-keyring
crypto ipsec transform-set azure-ipsec-proposal-set esp-aes 256 esp-sha-hmac
mode tunnel
crypto ipsec profile azure-vti
set security-association lifetime kilobytes 102400000
set transform-set azure-ipsec-proposal-set
set ikev2-profile azure-profile
crypto isakmp policy 1
encr aes
authentication pre-share
group 14
crypto ipsec transform-set csr-ts esp-aes esp-sha-hmac
mode tunnel
crypto ikev2 keyring azure-keyring
peer ${vpngw_pip_ip}
address ${vpngw_pip_ip}
pre-shared-key ${ipsec_psk}
crypto ikev2 profile azure-profile
match identity remote address ${vpngw_pip_ip} 255.255.255.255
crypto isakmp key ${psk} address ${vpngw_pip_ip}
interface Tunnel0
ip unnumbered GigabitEthernet1
ip tcp adjust-mss 1350
tunnel source GigabitEthernet1
tunnel mode ipsec ipv4
tunnel destination ${vpngw_pip_ip}
tunnel protection ipsec profile azure-vti
router bgp ${onprem_asn}
bgp router-id interface GigabitEthernet1
bgp log-neighbor-changes
redistribute ospf 100 route-map O2B
redistribute static route-map S2B
maximum-paths eibgp 4
neighbor ${vpngw_bgp_ip} remote-as ${vpngw_bgp_asn}
neighbor ${vpngw_bgp_ip} update-source GigabitEthernet1
neighbor ${vpngw_bgp_ip} ebgp-multihop 5
ip route ${vpngw_bgp_ip} 255.255.255.255 Tunnel0
ip route ${vpngw_pip_ip} 255.255.255.255 ${onprem_default_gw}
end
wr mem
EOF
# Create LNG, connection
az network local-gateway create -g $rg -n onprem --gateway-ip-address $onprem_csr_ip --local-address-prefixes "$onprem_csr_ip\32" \
--bgp-peering-address $onprem_csr_ip --asn ${onprem_asn}
az network vpn-connection create -g $rg --shared-key $psk -n onprem --vnet-gateway1 $vpngw_name --local-gateway2 onprem
fi
#######
# VPN #
#######
# NOTE: Azure VPN Gateway gets created in Failed state if deployed AFTER the ARS (this might be fixed now)
# Create VPN Gateway
if [[ "$vpn_branch" == "yes" ]]; then
az network vnet subnet create -g $rg --vnet-name $vnet_name -n GatewaySubnet --address-prefix $gw_subnet_prefix
az network public-ip create -g $rg -n "${vpngw_pip}-a" --allocation-method Dynamic --sku Basic
az network public-ip create -g $rg -n "${vpngw_pip}-b" --allocation-method Dynamic --sku Basic
az network vnet-gateway create -g "$rg" --sku VpnGw1 --gateway-type Vpn --vpn-type RouteBased \
--vnet "$vnet_name" -n $vpngw_name --asn "$vpngw_asn" --public-ip-addresses "${vpngw_pip}-a" "${vpngw_pip}-b" --no-wait