From 934bc14fc79521488710c3fd93d4f59e12b2516a Mon Sep 17 00:00:00 2001 From: Michael Kamprath Date: Sat, 16 May 2020 21:14:56 -0700 Subject: [PATCH] added upcloud deploy script (#20) --- CHANGELOG.md | 3 +- deployment-scripts/.gitignore | 2 + deployment-scripts/README.md | 47 +++++- deployment-scripts/upcloud-deploy.py | 232 +++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 deployment-scripts/.gitignore create mode 100644 deployment-scripts/upcloud-deploy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b66a630..71a562d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added * Created a deployment script for running the `multistreaming-server` locally. +* Created a deployment script for the [UpCloud cloud hosting service](https://upcloud.com/signup/?promo=A2CVWA). ### Changed * Improved error handling in the RTMP configuration generation script. @@ -22,7 +23,7 @@ All notable changes to this project will be documented in this file. * Migrated ### Added -* Created a deployment script for the Linode cloud hosting service. +* Created a deployment script for the [Linode cloud hosting service](https://www.linode.com/?r=37246e0d6a6198293308e698647804fbfe02845e). ### Fixed * Corrected how transcoding is implemented: larger buffer, better audio diff --git a/deployment-scripts/.gitignore b/deployment-scripts/.gitignore new file mode 100644 index 0000000..422d05b --- /dev/null +++ b/deployment-scripts/.gitignore @@ -0,0 +1,2 @@ +# ignore pythyon environments set up for development +upcloud/ diff --git a/deployment-scripts/README.md b/deployment-scripts/README.md index 416e40e..14bc2d2 100644 --- a/deployment-scripts/README.md +++ b/deployment-scripts/README.md @@ -3,7 +3,17 @@ Here you will find deployment scripts for various cloud services. ## Linode [Sign up for Linode cloud hosting here](https://www.linode.com/?r=37246e0d6a6198293308e698647804fbfe02845e). -This script will create a Linode server, deploy the `multistreaming-server` docker image pulled from Docker Hub, and then launch the image using the JSON configuration file you specify. The `linode-cli` software should be installed on your local computer ([click here for instructions](https://www.linode.com/docs/platform/api/linode-cli/)). +The `linode-deploy.sh` shell script will create a Linode server, deploy the `multistreaming-server` docker image pulled from Docker Hub, and then launch the image using the JSON configuration file you specify. The `linode-cli` software should be installed on your local computer ([click here for instructions](https://www.linode.com/docs/platform/api/linode-cli/)). + +The script is run in the following manner: +``` +RTMP_SERVER_LINODE_NODE_TYPE="g6-standard-6" \ + ./linode-deploy.sh \ + -c /path/to/rtmp-config.json \ + -p secure-r00t-password \ + -k ~/.ssh/id_rsa.pub \ + -s stream_password +``` When running this script, it takes the following arguments: * `-c /path/to/config.json` - The file path the the JSON file containing the multistreaming server configuration. _REQUIRED_ @@ -14,12 +24,45 @@ When running this script, it takes the following arguments: If the script successfully completes, you will have a Linode server running with the Multi-Service RTMP Broadcaster software running. The script also prints some useful information and commands to use at the end of its run. Most notable is the server's IP address. This should be used in your streaming software configuration as described in [this project's README](https://github.com/michaelkamprath/multi-service-rtmp-broadcaster/blob/master/README.md). -Two environment variables you might consider setting is the `RTMP_SERVER_LINODE_NODE_TYPE` variable to set the Linode type your server should use and `RTMP_SERVER_LINODE_REGION` variable to set the Linode region your server should reside. The Linode server type should have sufficient cores to handle the transcoding load required for your configuration. The Linode region should be the one closest to where you are streaming from. +There are several environment variables that control details of the type of server that will get deployed. Use the `-h` option to learn more. Two environment variables you might consider setting is the `RTMP_SERVER_LINODE_NODE_TYPE` variable to set the Linode type your server should use and `RTMP_SERVER_LINODE_REGION` variable to set the Linode region your server should reside. The Linode server type should have sufficient cores to handle the transcoding load required for your configuration. The Linode region should be the one closest to where you are streaming from. + +## UpCloud +[Sign up for UpCloud cloud hosting here](https://upcloud.com/signup/?promo=A2CVWA) + +This `upcloud-deploy.py` python script will create a server using the UpCloud service, deploy the `multistreaming-server` docker image pulled from Docker Hub, and then launch the image using the JSON configuration file you specify. The `upcloud-api` Python library must be installed on your local computer where you will run this script ([click here for instructions](https://github.com/UpCloudLtd/upcloud-python-api)). + +The script is run in the following manner: +``` +RTMP_SERVER_UPCLOUD_NODE_CORES="2" RTMP_SERVER_UPCLOUD_NODE_RAM="2048" RTMP_SERVER_UPCLOUD_NODE_DISK="32" \ + python3 upcloud-deploy.py \ + -u upcloud_username -p upcloud_password \ + -c /path/to/rtmp-config.json \ + -k ~/.ssh/id_rsa.pub \ + -s stream_password +``` +When running this script, it takes the following arguments: +* `-u ` - The user name for the UpCloud account that will be hosting the server. +* `-p ` - The user password for the UpCloud account that will be hosting the server. +* `-c /path/to/config.json` - The file path the the JSON file containing the multistreaming server configuration. _REQUIRED_ +* `-k /path/to/ssh_key.pub` - The file path to the public key file that should be used for keyless SSH connections to the server. If not specified, `~/.ssh/id_rsa.pub` will be used. Having a SSH key file defined on your system is _REQUIRED_. +* `-s stream_password` - The password that someone needs to use to push a stream to the rebroadcasting server. _REQUIRED_ +* `-h` - This will display more detailed information on how to use the script, including environment variables that are supported. + +If the script successfully completes, you will have a UpCloud server running with the Multi-Service RTMP Broadcaster software running. The script also prints some useful information and commands to use at the end of its run. Most notable is the server's IP address. This should be used in your streaming software configuration as described in [this project's README](https://github.com/michaelkamprath/multi-service-rtmp-broadcaster/blob/master/README.md). + +There are several environment variables that control details of the type of server that will get deployed. Use the `-h` option to learn more. The environment variables you might consider setting are `RTMP_SERVER_UPCLOUD_NODE_CORES` to indicate the number of cores your server will need, `RTMP_SERVER_UPCLOUD_NODE_RAM` to indicate how much RAM (in MB) your server will need, `RTMP_SERVER_UPCLOUD_NODE_DISK` to indicate the size of the disk (in GB) your server will need (note that all live streams get recorded on the server), and `RTMP_SERVER_UPCLOUD_REGION` to indicate what region your server should be deployed in ([possible values listed here](https://github.com/UpCloudLtd/upcloud-python-api/blob/master/upcloud_api/constants.py#L7)). ## Local Host Running the `multistreaming-server` locally is a great option if your local internet connection can support bandwidth required to push all of the rebroadcasted streams you intend to push. You should also consider if your local CPU should have sufficient cores to perform any transcoding you desire for individual streams. This script will simply launch a Docker container locally for the `multistreaming-server`. +The script is run in the following manner: +``` +./local-deploy.sh \ + -c /path/to/rtmp-config.json \ + -s stream_password +``` + When running this script, it takes the following arguments: * `-c /path/to/config.json` - The file path the the JSON file containing the multistreaming server configuration. _REQUIRED_ * `-s stream_password` - The password that someone needs to use to push a stream to the rebroadcasting server. _REQUIRED_ diff --git a/deployment-scripts/upcloud-deploy.py b/deployment-scripts/upcloud-deploy.py new file mode 100644 index 0000000..a1ba460 --- /dev/null +++ b/deployment-scripts/upcloud-deploy.py @@ -0,0 +1,232 @@ +# upcloud-deploy.py +# +# This python script deploys and instance of the Multiservice RTMP broadcaster +# to a newly created server in the UpCloud hosting service. +# +# Assumes that the upcloud-api python library is installed. +# Run with python3 +# +import getopt +import os +import subprocess +import sys +import time + +import upcloud_api as up + +# This shell script is run on the server once the server is running. +# It is used to install and configur Docker. +INIT_SCRIPT = """ +# Update the system and install useful packages +apt-get update +apt-get upgrade -y +apt-get install htop + +# Install docker +curl -fsSL https://get.docker.com -o get-docker.sh +sh get-docker.sh + +# Enable firewall +ufw enable +ufw allow 22 +ufw allow http +ufw allow 1935 + +# configure the rtmpserver user +usermod -a -G docker rtmpserver + +""" + +def print_help(): + print( + 'Example usage:\n ' + 'python3 upcloud-deploy.py ' + '-u ' + '-p ' + '-c /path/to/config.json ' + '-k /path/to/ssh_key.pub ' + '-s ' + '\n' + ) + print('Environment Variables (can be used instead of command arguments):') + print(' RTMP_SERVER_UPCLOUD_USER : UpCloud user name to authenticate with. Can replace \'-u\' option.') + print(' RTMP_SERVER_UPCLOUD_PASSWORD : UpCloud user password to authenticate with. Can replace \'-p\' option.') + print(' RTMP_SERVER_UPCLOUD_REGION : UpCloud region to create server in. Defaults to \'us-sjo1\'.') + print(' RTMP_SERVER_UPCLOUD_NODE_CORES : The number cores for the UpCloud server. Defaults to \'4\'') + print(' RTMP_SERVER_UPCLOUD_NODE_RAM : The amount of RAM in MB for the UpCloud server. Defaults to \'4096\'') + print(' RTMP_SERVER_UPCLOUD_NODE_DISK : The amount of disk space in GB for the UpCloud server. Defaults to \'64\'') + +# Default values for config +upcloud_user = None \ + if 'RTMP_SERVER_UPCLOUD_USER' not in os.environ \ + else os.environ['RTMP_SERVER_UPCLOUD_USER'] +upcloud_password = None \ + if 'RTMP_SERVER_UPCLOUD_PASSWORD' not in os.environ \ + else os.environ['RTMP_SERVER_UPCLOUD_PASSWORD'] +upcloud_region = up.ZONE.SanJose \ + if 'RTMP_SERVER_UPCLOUD_REGION' not in os.environ \ + else os.environ['RTMP_SERVER_UPCLOUD_REGION'] +upcloud_cores = 4 \ + if 'RTMP_SERVER_UPCLOUD_NODE_CORES' not in os.environ \ + else int(os.environ['RTMP_SERVER_UPCLOUD_NODE_CORES']) +upcloud_ram = 4096 \ + if 'RTMP_SERVER_UPCLOUD_NODE_RAM' not in os.environ \ + else int(os.environ['RTMP_SERVER_UPCLOUD_NODE_RAM']) +upcloud_disk = 64 \ + if 'RTMP_SERVER_UPCLOUD_NODE_DISK' not in os.environ \ + else int(os.environ['RTMP_SERVER_UPCLOUD_NODE_DISK']) +rtmp_config_filepath = None +ssh_key_filepath = '~/.ssh/id_rsa.pub' +rtmp_streaming_password = None + +# fetch CLI arguments +try: + opts, args = getopt.getopt(sys.argv[1:],"u:p:c:k:s:h") +except getopt.GetoptError as err: + print(err) + print_help() + exit(2) + +for opt, arg in opts: + if opt == '-u': + upcloud_user = arg + elif opt == '-p': + upcloud_password = arg + elif opt == '-c': + rtmp_config_filepath = arg + elif opt == '-k': + ssh_key_filepath = arg + elif opt == '-s': + rtmp_streaming_password = arg + elif opt == '-h': + print_help() + exit(0) + else: + print('ERROR - Got on unrecognized command line option "{0}"'.format(opt)) + exit(2) + +if upcloud_user is None \ + or upcloud_password is None \ + or rtmp_config_filepath is None \ + or ssh_key_filepath is None \ + or rtmp_streaming_password is None: + print('ERROR - configuration is not complete.') + exit(2) + +# Connect to UpCloud +manager = up.CloudManager(upcloud_user, upcloud_password) +manager.authenticate() +print('Connected to UpCloud API as user "{0}"'.format(upcloud_user)) + +res = subprocess.run( + 'cat {0}'.format(ssh_key_filepath), + shell=True, + stdout=subprocess.PIPE, + text=True, +) +ssh_key_value = res.stdout.strip() +rtmp_user_desc = up.login_user_block( + username='rtmpserver', + ssh_keys=[ssh_key_value], + create_password=False, +) + +rtmp_server_desc = up.Server( + core_number=upcloud_cores, # CPU cores + memory_amount=upcloud_ram, # RAM in MB + zone=upcloud_region, + title='Multiservice RTMP Broadcaster', + # UpCloud strangely requires that every server have a qualified domain + # name. (?!) Using a totaly made up name here. + hostname='multiservice-rtmp-server.com', + storage_devices=[ + up.Storage(os='Ubuntu 18.04', size=upcloud_disk ), + ], + login_user=rtmp_user_desc, # user and ssh-keys + user_data=INIT_SCRIPT, +) + +print('Starting creation of server with these parameters:') +print( + ' cores = {0}\n' + ' RAM = {1} MB\n' + ' disk = {2} GB\n' + ' region = {3}'.format( + upcloud_cores, upcloud_ram, upcloud_disk, upcloud_region + ) +) +rtmp_server = manager.create_server(rtmp_server_desc) +ip_addr = rtmp_server.get_ip() +print( + 'Server creation done.\n' + ' server = {0}\n' + ' IP address = {1}'.format(rtmp_server, ip_addr) +) + +# wait for the server to finish booting and installing docker +print('Waiting 5 minutes for server set up to complete ...') +time.sleep(60) +print('Waiting 4 minutes for server set up to complete ...') +time.sleep(60) +print('Waiting 3 minutes for server set up to complete ...') +time.sleep(60) +print('Waiting 2 minutes for server set up to complete ...') +time.sleep(60) +print('Waiting 1 minutes for server set up to complete ...') +time.sleep(60) + +# Send configuration to server and launch Docker image +print('Adding IP address {0} to known hosts ...'.format(ip_addr)) +res = subprocess.run('ssh-keygen -R {}'.format(ip_addr), shell=True) +if res.returncode != 0: + print('ERROR when removing server IP from known hosts.\n{0}', res.stderr) + exit(1) +else: + print(res.stdout) +res = subprocess.run('ssh-keyscan -T 240 {0} >> ~/.ssh/known_hosts'.format(ip_addr), shell=True) +if res.returncode != 0: + print('ERROR when adding server IP to known hosts.\n{0}', res.stderr) + exit(1) +else: + print(res.stdout) + +# send configuration file to Server +res = subprocess.run( + 'scp {0} rtmpserver@{1}:/home/rtmpserver/rtmp_server_config.json'.format( + rtmp_config_filepath, + ip_addr + ), + shell=True +) +if res.returncode != 0: + print('ERROR when sending RTMP configuration to server.\n{0}', res.stderr) + exit(1) +else: + print(res.stdout) + +# start the docker subprocess +res = subprocess.run( + 'ssh rtmpserver@{0} ' + '"docker run -d -p 80:80 -p 1935:1935 ' + '--env MULTISTREAMING_PASSWORD={1} ' + '-v /home/rtmpserver/rtmp_server_config.json:/rtmp-configuation.json ' + 'kamprath/multistreaming-server:latest"'.format( + ip_addr, rtmp_streaming_password + ), + shell=True, + stdout=subprocess.PIPE, + text=True, +) +docker_container_id = res.stdout[:12] +print('Started Docker container: {0}'.format(docker_container_id)) + +# Finished +print('Finished!\n') +print('The IP address for the Multistreaming Server is:') +print(' {0}\n'.format(ip_addr)) +print('Visit the Multistreaming Server\'s statistics page here:') +print(' http://{0}/stat\n'.format(ip_addr)) +print('Use this command to log into the server (if needed):') +print(' ssh rtmpserver@{0}\n'.format(ip_addr)) +print('When done, terminate this server in the UpCloud Web Console here:') +print(' https://hub.upcloud.com/\n')