os-aio-sahara.txt

###########################################################################################
##### Lab 1 - Deploy OpenStack Keystone, Glance, Neutron, Nova, Horizon, Heat, Sahara #####
###########################################################################################


########## Eksekusi di Node openstackX ##########

##### IP, Gateway, DNS Resolver, Hostname #####
nmtui

### Node Controller ###
Interface: eth0
IP Address: 10.X0.X0.10/24
Gateway: 10.X0.X0.1
DNS Resolver: 10.X0.X0.1

Interface: eth1
IP Address: 10.X1.X1.10/24

Hostname: openstackX


-----Verifikasi Konektifitas-----
ping -c 3 10.X0.X0.1
ping -c 3 10.X0.X0.10
ping -c 3 10.X1.X1.1
ping -c 3 10.X1.X1.10
ping -c 3 8.8.8.8


##### Name Resolution #####
echo "10.X0.X0.10 openstackX" >> /etc/hosts


##### Repositori #####
yum -y install centos-release-openstack-newton.noarch epel-release.noarch
[ ! -d /etc/yum.repos.d.orig ] && cp -vR /etc/yum.repos.d /etc/yum.repos.d.orig
yum repolist
yum -y update


##### NTP #####
yum -y install chrony
systemctl enable chronyd.service
systemctl restart chronyd.service
systemctl status chronyd.service
chronyc sources


##### Firewall #####
systemctl stop firewalld.service
systemctl disable firewalld.service
systemctl status firewalld.service


##### Networking #####
systemctl disable NetworkManager.service
systemctl stop NetworkManager.service
systemctl status NetworkManager.service
systemctl enable network.service
systemctl restart network.service
systemctl status network.service


##### Paket Utilities #####
yum -y install vim wget screen crudini


### Create Storage Partition ###
fdisk /dev/vda

vda3 10G 83 (Linux)
vda4 10G 83 (Linux)

partprobe

### XFS for Swift ###
mkfs.xfs /dev/vda3
mkfs.xfs /dev/vda4


##### Paket Packstack #####
yum -y install openstack-packstack.noarch


##### Generating Packstack Answer File #####
packstack --gen-answer-file=os-aio.paf
vim paf-os-admin-lab1.txt

CONFIG_CINDER_INSTALL=n
CONFIG_SWIFT_INSTALL=y
CONFIG_SAHARA_INSTALL=y
CONFIG_HEAT_INSTALL=y
CONFIG_NAGIOS_INSTALL=n
CONFIG_USE_EPEL=y
#CONFIG_KEYSTONE_ADMIN_PW=9288844cb55f4c64
CONFIG_KEYSTONE_ADMIN_PW=rahasia
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan,vlan,flat
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=external
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=external:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=br-ex:eth1
CONFIG_NEUTRON_OVS_BRIDGES_COMPUTE=br-ex
CONFIG_PROVISION_DEMO=n
CONFIG_SWIFT_STORAGES=/dev/vda3,/dev/vda4
CONFIG_SWIFT_STORAGE_FSTYPE=xfs


screen -R packstack

packstack --answer-file=os-aio.paf

### Keluar screen tanpa mematikan tekan Ctrl+A kemudian tekan D


##################################
##### Troubleshoot Packstack #####
##################################


#1 Metadata DHCP Agent
crudini --set /etc/neutron/dhcp_agent.ini DEFAULT enable_isolated_metadata True
systemctl restart neutron-dhcp-agent
systemctl status neutron-dhcp-agent


#2. Error: Failed to connect socket to '/var/run/libvirt/virtlogd-sock'
#Solusi: Aktifkan dan jalankan service virtlogd
systemctl status virtlogd
systemctl enable virtlogd
systemctl restart virtlogd
systemctl status virtlogd


#3. Set Hypervisor KVM
crudini --set /etc/nova/nova.conf libvirt virt_type kvm
systemctl restart openstack-nova-compute
systemctl status openstack-nova-compute


#4. Instal paket openstack-sahara-ui
yum -y install openstack-sahara-ui
systemctl restart httpd
systemctl status httpd


##### Launch Instance BUI #####
#Jalankan web browser dan buka alamat http://10.X0.X0.10/dashboard

#0 Login as admin with password PASSWORD
cat /root/keystonerc_admin

#1 Create images
Download cirros image
https://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img

Admin > System > Images
Click Create Image
Image Name: cirros0
Image Source: Image Location
File: Browse cirros-0.3.5-x86_64-disk.img
Format: QCOW2

#2 Create external network
Admin > System > Networks
Click Create Network
Name: net-ext
Project: admin
Provider Network Type: flat
Physical Network: extnet
Admin State: Up
Shared: Checked
External Network: Checked

#3 Create external subnet
Admin > System > Networks
Click net-ext
Click Create Subnet
Subnet Name: subnet-ext
Network Address: 10.X1.X1.0/24
IP Version: IPv4
Gateway IP: 10.X1.X1.1
Enable DHCP: Checked
Allocation Pools: 10.X1.X1.100,10.X1.X1.199
DNS Name Servers: 10.X1.X1.1

#4 Add SSH key
Project > Compute > Access & Security > Key Pairs
Click Import Key Pair
Key Pair Name: key0
Public Key: [copy and paste SSH public key]

#5 Add security group rules
Project > Compute > Access & Security > Security Groups
Click Create Security Group
Name: sg0
Description: My security group 0
Click Manage Rules on sg0

Click Add Rule
Rule: All ICMP
Direction: Ingress
Remote: CIDR
CIDR: 0.0.0.0/0

Click Add Rule
Rule: All TCP
Remote: CIDR
CIDR: 0.0.0.0/0

Click Add Rule
Rule: All UDP
Remote: CIDR
CIDR: 0.0.0.0/0


#6 Launch instance
Project > Compute > Instance
Click Launch Instance
Instance Name: instance0
Flavor: m1.tiny
Instance Boot Source: Boot from image
Image Name: cirros0
Key Pair: key0
Security Group: sg0
Selected networks: net-ext


############################################
##### Lab 2a - Spark Cluster Provision #####
############################################

#1 Create images
Download park image:
http://sahara-files.mirantis.com/images/upstream/newton/sahara-newton-spark-1.6.0-ubuntu.qcow2

Admin > System > Images
Click Create Image
Image Name: sahara-newton-spark-1.6.0-ubuntu
Image Source: Image Location
File: Browse sahara-newton-spark-1.6.0-ubuntu.qcow2
Format: QCOW2


#2 Image Registry
Project > Data Processing > Clusters > Image Registry
Click Register Image
Image: sahara-newton-spark-1.6.0-ubuntu
User Name: ubuntu
Plugin & Version Tag: spark & 1.6.0 Click +


#3 Node Group Templates
Project > Data Processing > Clusters > Node Group Templates
Click Create Template
Plugin Name: Apache Spark
Version: 1.6.0

Template Name: template-master
OpenStack Flavor: m1.medium
Availability Zone: nova
Storage location: Ephemeral Drive
Base Image: sahara-newton-spark-1.6.0-ubuntu
Floating IP Pool: do not assign floating IPs
Auto configure: checked
HDFS processes: namenode
Spark processes: master
Auto security group: Unchecked
Security Groups: sg0

Click Create Template
Plugin Name: Apache Spark
Version: 1.6.0

Template Name: template-worker
OpenStack Flavor: m1.medium
Availability Zone: nova
Storage location: Ephemeral Drive
Base Image: sahara-newton-spark-1.6.0-ubuntu
Floating IP Pool: do not assign floating IPs
Auto configure: checked
HDFS processes: datanode
Spark processes: slave
Auto security group: Unchecked
Security Groups: sg0


#3 Cluster Templates
Project > Data Processing > Clusters > Cluster Templates
Click Create Template
Plugin Name: Apache Spark
Version: 1.6.0

Template Name: template-cluster-spark
Auto-configure: checked

Node Groups
Select: template-master Click +
Select: template-slave Click +


#4 Launch Cluster
Project > Data Processing > Clusters > Clusters
Click Launch Cluster
Plugin Name: Apache Spark
Version: 1.6.0

Cluster Name: cluster-spark
Cluster Template: template-cluster-spark
Cluster Count: 1
Base Image: sahara-newton-spark-1.6.0-ubuntu
Keypair: key0
Neutron Management Network: net-ext
Click Launch


###########################################
##### Lab 2b - Analytics as a Service #####
###############s############################

#1 Store spark binary and input file
Project > Object Store > Containers > +Container
Container Name: acci
Download sample_input.txt and spark-wordcount.jar files from this repo
https://github.com/openstack/sahara-tests/tree/master/sahara_tests/scenario/defaults/edp-examples/edp-spark
Upload the files to acci container

#2 Job Binaries
Project > Data Processing > Jobs > Job Binaries
Click Create Job Binary
Name: spark-wordcount
Storage type: Swift
URL swift: acci/spark-wordcount.jar
Username: admin
Password: rahasia

#3 Job Templates
Project > Data Processing > Jobs > Job Templates
Click Create Job Template
Name: job-spark-wordcount
Job Type: spark
Choose a main binary: spark-wordcount
Click Create

4# Launch Job
Project > Data Processing > Jobs > Job Templates
Click Launch on Existings Cluster
Cluster: cluster-spark

Configure:
Main class: sahara.edp.spark.SparkWordCount
Configurations:
edp.spark.adapt_for_swift True
fs.swift.service.sahara.username admin
fs.swift.service.sahara.password rahasia
Arguments:
swift://acci/sample_input.txt
swift://acci/output
Click Launch

Check the job output on container acci