From a95d72693b35646d2336978337fbb7f74d77c40b Mon Sep 17 00:00:00 2001 From: Raj Perera Date: Sat, 31 Oct 2020 12:46:33 -0400 Subject: [PATCH] Lots of enhancements (#26) * Migrated to jinja2, use pipfile and supervisord * Add kubernetes manifests, fix nginx logging, streamline the dockerfile * Use the upstream image name, fix typos * Switch to single quotes * Fix typos in template, update readme --- README.md | 53 ++- deployment-scripts/README.md | 17 + .../kubernetes-kustomize/deployment.yaml | 45 +++ .../kubernetes-kustomize/ingress.yaml | 13 + .../kubernetes-kustomize/kustomization.yaml | 19 + .../rtmp-configuration.json | 23 ++ .../kubernetes-kustomize/service.yaml | 19 + multistreaming-server/Dockerfile | 32 +- multistreaming-server/Pipfile | 17 + multistreaming-server/Pipfile.lock | 251 +++++++++++++ multistreaming-server/launch-nginx-server.sh | 5 +- multistreaming-server/nginx-conf/nginx.conf | 2 +- multistreaming-server/nginx-template.conf.j2 | 79 +++++ multistreaming-server/rtmp-conf-generator.py | 335 +++++++++--------- .../stunnel-conf/etc-stunnel-stunnel.conf | 7 +- multistreaming-server/supervisord.conf | 31 ++ 16 files changed, 749 insertions(+), 199 deletions(-) create mode 100644 deployment-scripts/kubernetes-kustomize/deployment.yaml create mode 100644 deployment-scripts/kubernetes-kustomize/ingress.yaml create mode 100644 deployment-scripts/kubernetes-kustomize/kustomization.yaml create mode 100644 deployment-scripts/kubernetes-kustomize/rtmp-configuration.json create mode 100644 deployment-scripts/kubernetes-kustomize/service.yaml create mode 100644 multistreaming-server/Pipfile create mode 100644 multistreaming-server/Pipfile.lock create mode 100644 multistreaming-server/nginx-template.conf.j2 mode change 100644 => 100755 multistreaming-server/rtmp-conf-generator.py create mode 100644 multistreaming-server/supervisord.conf diff --git a/README.md b/README.md index f31a6c1..f373705 100644 --- a/README.md +++ b/README.md @@ -22,31 +22,47 @@ docker run -it -p 80:80 -p 1935:1935 \ If this is a host than where you built the docker image, you will need to push the docker image to that host (or build it there). Alternatively, you could use the DockerHub build of this image by pulling and using the `kamprath/multistreaming-server:latest` [Docker image](https://hub.docker.com/repository/docker/kamprath/multistreaming-server). Also, if you plan on doing any transcoding when rebroadcasting a stream, you need to ensure that your docker host's CPU is sufficient for the job. It is recommend that the host CPU has at least four cores for each distinct transcoding the multi-streaming server will do. If the host CPU is not sufficient, bit rates on the transcoded streams will suffer. -Note that an environment variable is set when running the Docker image: +Note that some environment variables should be set when running the Docker image: * `MULTISTREAMING_PASSWORD` _(REQUIRED)_ - This is a password you define and will be used by your steaming software. This is a marginally secure way to prevent other people from pushing to your stream. +* `CONFIG_DISABLE_RECORD` _(OPTIONAL)_ - If this env var is set to a non-empty value, the stream recording feature will be disabled. +* `CONFIG_NGINX_DEBUG` _(OPTINAL)_ - If this env var is set to a non-empty value, it will enable the nginx debug logging to stderr. This is useful for debugging. +* `CONFIG_FFMPEG_LOG` _(OPTIONAL)_ - If this env var is set to a non-empty value, each ffmpeg process will start logging their output to `/tmp/ffmpeg-$APP.log` in the container. This is useful for debugging issues related to encoding. +* `CONFIG_FFMPEG_MAX_MUXING_QUEUE_SIZE` _(OPTIONAL)_ - This env var must be a numerical value, it will be used as the global default for `-max_muxing_queue_size` argument for the ffmpeg processes. You may need to set this to a higher value (i.e: `4096`) if you run in to this [bug.](https://trac.ffmpeg.org/ticket/6375) You must also create and JSON file with the RTMP rebroadcasting configuration you desire. This file should get mapped from your local file system to the `/rtmp-configuration.json` file path within the Docker container. The JSON file has the following elements: * `endpoint`- This is the name of the RTMP ingest endpoint that the source stream will be pushed to. Defaults to `live` if not specified. +* `transcodeProfiles` - This is a JSON object with keys indicating a unique identifier for a transcode profile, and the values specifying the supported transcode options. + * `pixels` - The pixel dimension that the stream should be transcoded to. Formatted like "1920x1080". If not specified, defaults to "1280x720". + * `videoBitRate` - The video bit rate that should be used when sending the stream to this destination. Should be a number followed by "k" or "m" for kilo- and mega- bits-per-second. If not specified defaults to "4500k". + * `videoKeyFrameSecs` - The number of seconds between key frames in the transcoded stream. If not specified, defaults to 2. + * `videoFrameRate` - The frames per second which the video stream will be re-encoded with, defaults to "30". + * `audioBitRate` - The bit rate that should be used for the transcoded audio signal. Should be a number followed by "k" or "m" for kilo- and mega- bits-per-second. If not specified, defaults to "160k". If neither `audioBitRate` or `audioSampleRate` are specified, then the audio signal is simply copied from source with no alteration. + * `audioSampleRate` - The sampling rate to be used for the transcoded audio signal. Should be an integer indicating the sampling Hertz. If not specified, defaults to `48000`. If neither `audioBitRate` or `audioSampleRate` are specified, then the audio signal is simply copied from source with no alteration. + * `maxMuxingQueueSize` - Individual override for the `CONFIG_FFMPEG_MAX_MUXING_QUEUE_SIZE` env var. See the env var section for the usage. * `rebroadcastList`- _Required_ Contains a list of JSON objects that each configure a distinct RTMP destination that the stream pushed to the ingest endpoint will be rebroadcasted to. At least one destination should be configured. There is no specific limit on the number of destinations except for the hardware limitations of your host. Each destination is configured with the following JSON elements: * `name` - _Required_ A distinct label for this destination. Can be any alphanumeric string with no white space. Must be distinct from all the other destination names in this list. * `platform` - _Required_ The platform that this specific rebroadcast stream should be pushed to. The default RTMP destinations will be used for each platform. Supported platforms values are: `youtube`, `facebook`, `twitch`, `instagram`, `periscope`, `microsoft-stream`, `mixcloud`, `dlive` and `custom`. Note that specifying `microsoft-stream` or `custom` will cause the `streamKey` element to be ignored if present and instead use the `fullRTMPURL` * `regionCode` - If `periscope` is specified as the platform for this destination, this is the two letter region code that is part of the Periscope server URL. If undefined, it will default to `ca` (the "US West" region) * `streamKey` - This is the stream key that identifies the unique stream on the specified platform. This value is provided by the platform. This element must be provided for all `platform` types except for `custom` and `microsoft-stream`. * `fullRTMPURL` - If `custom` or `microsoft-stream` is specified in the `platform`, the URL specified in this element is used for the forming destination URL. This should include the `rtmp://` prefix. For the `microsoft-stream` platform, this is the full URL provided in their stream set up. This element is ignored for all other platform types. - * `transcode` - If present, the stream will be transcoded before rebroadcasting it to this list item's destination. Note that when using this transcoding, the stream will be trancoded to 30 FPS and CBR bit rate. The value is a JSON object that contains the following configuration elements: - * `pixels` - The pixel dimension that the stream should be transcoded to. Formatted like "1920x1080". If not specified, defaults to "1280x720". - * `videoBitRate` - The video bit rate that should be used when sending the stream to this destination. Should be a number followed by "k" or "m" for kilo- and mega- bits-per-second. If not specified defaults to "4500k". - * `videoKeyFrameSecs` - The number of seconds between key frames in the transcoded stream. If not specified, defaults to 2. - * `audioBitRate` - The bit rate that should be used for the transcoded audio signal. Should be a number followed by "k" or "m" for kilo- and mega- bits-per-second. If not specified, defaults to "160k". If neither `audioBitRate` or `audioSampleRate` are specified, then the audio signal is simply copied from source with no alteration. - * `audioSampleRate` - The sampling rate to be used for the transcoded audio signal. Should be an integer indicating the sampling Hertz. If not specified, defaults to `48000`. If neither `audioBitRate` or `audioSampleRate` are specified, then the audio signal is simply copied from source with no alteration. + * `transcode` - If present, the stream will be transcoded before rebroadcasting it to this list item's destination. Note that when using this transcoding, the stream will be trancoded to 30 FPS and CBR bit rate. The value is a JSON object with the support for same optoins as an entry in `transcodeProfiles`. Prefer to use `transcodeProfile` key if you have multiple outputs using the same encoding configuration. + * `transcodeProfile` - If present, this stream will use the transcode settings provided by the `transcodeProfile` referenced by the string identifier set here. If multiple streams refer to the same `transcodeProfile`, the transcoding will only be done once. This option is mutually exclusive with the `transcode` key. + Here is an example of the JSON configuration file: ``` { "endpoint": "live", - "rebroacastList": [ + "transcodeProfiles": { + "720_60fps": { + "pixels": "1280x720", + "videoBitRate": "4500k", + "videoFrameRate": 60 + } + }, + "rebroadcastList": [ { "name": "youtube", "platform": "youtube", @@ -56,19 +72,13 @@ Here is an example of the JSON configuration file: "name": "facebook-1", "platform": "facebook", "streamKey": "def456", - "transcode": { - "pixels": "1280x720", - "videoBitRate": "4500k" - } + "transcodeProfile": "720_60fps" }, { "name": "facebook-2", "platform": "facebook", "streamKey": "ghi789", - "transcode": { - "pixels": "1280x720", - "videoBitRate": "4500k" - } + "transcodeProfile": "720_60fps" }, { "name": "periscope", @@ -88,7 +98,7 @@ Here is an example of the JSON configuration file: ``` Note that as long as the `name` elements are different, you can have more than one destination pushing to the same `platform`, each with different `streamKey` values. -If you would like to capture a recording of the stream sent to the ingest endpoint, bind a local directory on your host to the `/var/www/html/recordings/` file path within the Docker image when launching the Docker container. You can also see statistics about the various streams this server is pushing by visiting this web page: `http://__docker_host_IP_address__/stat`. +If you would like to capture a recording of the stream sent to the ingest endpoint, bind a local directory on your host to the `/var/www/html/recordings/` file path within the Docker image when launching the Docker container. You can also see statistics about the various streams this server is pushing by visiting this web page: `http://__docker_host_IP_address__/stat`. You can optionally disable recording by setting the env var `CONFIG_DISABLE_RECORD` to a non-empty value. Once the Docker image is running, set up your stream software with the following parameters: @@ -99,6 +109,15 @@ In OBS, you would set the above parameters for a "Custom..." Service in the Stre The next thing to do is start your stream! +## Development + +In order to develop on this repository, you must have a python 3.7 environment and [`Pipenv`](https://pypi.org/project/pipenv/) installed. + +- Clone the repository +- Navigate to the `multistreaming-server/` directory +- Install dependencies and create the virtual env by running `pipenv install` +- Activate the virtual env by `pipenv shell` + ## Future work Goals for future improvements to this project include: diff --git a/deployment-scripts/README.md b/deployment-scripts/README.md index 643c22c..bfe4e2f 100644 --- a/deployment-scripts/README.md +++ b/deployment-scripts/README.md @@ -71,3 +71,20 @@ When running this script, it takes the following arguments: * `-h` - This will display more detailed information on how to use the script, including environment variables that are supported. If the script successfully completes, you will be running a Docker container locally with the Multi-Service RTMP Broadcaster software running. The script also prints some useful information and commands to use at the end of its run. The IP address you should use to configure your streaming software's destination is `127.0.0.1` or `localhost`. + +## Kubernetes + +You will find the required manifests to deploy the container to a Kubernetes cluster under the `./kubernetes-kustomize` directory. + +* Exposed RTMP/stats port: The example manifest in this repo will create a Service of type `NodePort` which will expose the RTMP endpoint on port 33195, and the stats endpoint on port 30080 on all the nodes. If you want to change this behavior, update `service.yaml` with the `nodePort` values you want. +* Stream password: Stream password will be created a Kubernetes Secret in the cluster, to set the password open the file `kustomization.yaml` and find the entry with the name `multistreaming` in the list under the key `secretGenerator`. +* RTMP Configuration JSON: The configuration file must be placed in the same path as the `kustomization.yaml` and should be named `rtmp-configuration.json`. The deployment will automatically read this file and create it in the cluster as a configmap. +* Kubernetes namespace: By default the manifests will be deployed to a namespace called `multistreaming`. You may change this in `kustomization.yaml`. This namespace should be created manually before it can be used for deployments. + +Deployment command: +``` +# Make sure you have your KUBECONFIG set. +$ cd kubernetes-kustomize/ # or the path where you are storing the manifests +$ kubectl create namespace multistreaming # the target namespace, should only be run once +$ kubectl apply -k . # can be run many times to refresh the configuration +``` \ No newline at end of file diff --git a/deployment-scripts/kubernetes-kustomize/deployment.yaml b/deployment-scripts/kubernetes-kustomize/deployment.yaml new file mode 100644 index 0000000..385a82a --- /dev/null +++ b/deployment-scripts/kubernetes-kustomize/deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: multistreaming + labels: + app: multistreaming +spec: + selector: + matchLabels: + app: multistreaming + template: + metadata: + labels: + app: multistreaming + spec: + containers: + - name: multistreaming + image: kamprath/multistreaming-server:latest + imagePullPolicy: Always + volumeMounts: + - name: config-volume + mountPath: /rtmp-configuration.json + subPath: rtmp-configuration.json + env: + - name: MULTISTREAMING_PASSWORD + valueFrom: + secretKeyRef: + name: multistreaming + key: password + - name: CONFIG_NGINX_DEBUG + value: "true" + - name: CONFIG_FFMPEG_LOG + value: "true" + - name: CONFIG_FFMPEG_MAX_MUXING_QUEUE_SIZE + value: "4096" + - name: CONFIG_DISABLE_RECORD + value: "true" + ports: + - containerPort: 80 + - containerPort: 1935 + volumes: + - name: config-volume + configMap: + name: multistreaming-config + diff --git a/deployment-scripts/kubernetes-kustomize/ingress.yaml b/deployment-scripts/kubernetes-kustomize/ingress.yaml new file mode 100644 index 0000000..180b7e9 --- /dev/null +++ b/deployment-scripts/kubernetes-kustomize/ingress.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: multistreaming +spec: + rules: + - host: multistreaming.ls90.co + http: + paths: + - path: / + backend: + serviceName: multistreaming + servicePort: 80 \ No newline at end of file diff --git a/deployment-scripts/kubernetes-kustomize/kustomization.yaml b/deployment-scripts/kubernetes-kustomize/kustomization.yaml new file mode 100644 index 0000000..146a6dd --- /dev/null +++ b/deployment-scripts/kubernetes-kustomize/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -v1 +namespace: multistreaming +commonLabels: + app: multistreaming + version: v1 +resources: +- deployment.yaml +- service.yaml +# - ingress.yaml # Optional, must configure the hostname if you need to expose the stat page as an ingress +configMapGenerator: +- name: multistreaming-config + files: + - rtmp-configuration.json +secretGenerator: +- name: multistreaming + literals: + - password=sekritPassword \ No newline at end of file diff --git a/deployment-scripts/kubernetes-kustomize/rtmp-configuration.json b/deployment-scripts/kubernetes-kustomize/rtmp-configuration.json new file mode 100644 index 0000000..92455c3 --- /dev/null +++ b/deployment-scripts/kubernetes-kustomize/rtmp-configuration.json @@ -0,0 +1,23 @@ +{ + "endpoint": "live", + "transcodeProfiles": { + "720_60fps": { + "pixels": "1280x720", + "videoBitRate": "4500k", + "videoFrameRate": 60 + } + }, + "rebroadcastList": [ + { + "name": "twitch", + "platform": "twitch", + "streamKey": "foobar" + }, + { + "name": "fbpage", + "platform": "facebook", + "streamKey": "secretKey", + "transcodeProfile": "720_60fps" + } + ] +} diff --git a/deployment-scripts/kubernetes-kustomize/service.yaml b/deployment-scripts/kubernetes-kustomize/service.yaml new file mode 100644 index 0000000..84a779b --- /dev/null +++ b/deployment-scripts/kubernetes-kustomize/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: multistreaming +spec: + selector: + app: multistreaming + type: NodePort + ports: + - name: web + protocol: TCP + port: 80 + targetPort: 80 + nodePort: 30080 + - name: rtmp + protocol: TCP + port: 1935 + targetPort: 1935 + nodePort: 31935 \ No newline at end of file diff --git a/multistreaming-server/Dockerfile b/multistreaming-server/Dockerfile index e8384df..ab6c433 100644 --- a/multistreaming-server/Dockerfile +++ b/multistreaming-server/Dockerfile @@ -4,6 +4,9 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath" ARG NGINX_VERSION=1.19.2 ARG RTMP_REPO=uizaio ARG RTMP_MODULE_VERSION=1.4.0.4 +ARG TINI_VERSION=v0.19.0 +ARG SUPERVISORD_VERSION=4.2.1 +ARG PIPENV_PACKAGE_VERSION=2020.8.13 RUN set -x \ && addgroup -S stunnel \ @@ -22,34 +25,33 @@ RUN set -x \ && ./configure --with-http_ssl_module --add-module=../nginx-rtmp-module-${RTMP_MODULE_VERSION} \ && make \ && make install \ + && cp /nginx-rtmp-module-${RTMP_MODULE_VERSION}/stat.xsl /usr/local/nginx/html/ \ && apk del build-deps \ && mkdir -p /var/www/html/recordings \ && mkdir -p /var/run/stunnel/ \ && chown nobody:nobody -R /var/www/html \ - && chown stunnel:stunnel /var/run/stunnel/ + && chown stunnel:stunnel /var/run/stunnel/ \ + && wget -O /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static \ + && chmod +x /tini \ + && pip3 install supervisor==${SUPERVISORD_VERSION} pipenv==${PIPENV_PACKAGE_VERSION} -COPY index.html /usr/local/nginx/html/ -COPY nginx-conf/nginx.conf /base-nginx.conf -COPY launch-nginx-server.sh launch-nginx-server.sh -COPY rtmp-conf-generator.py rtmp-conf-generator.py +COPY Pipfile Pipfile.lock / +RUN pipenv install --system --deploy +COPY supervisord.conf /etc/supervisor/supervisord.conf COPY stunnel-conf/etc-default-stunnel /etc/default/stunnel COPY stunnel-conf/etc-stunnel-conf.d-fb.conf /etc/stunnel/conf.d/fb.conf COPY stunnel-conf/etc-stunnel-conf.d-ig.conf /etc/stunnel/conf.d/ig.conf COPY stunnel-conf/etc-stunnel-stunnel.conf /etc/stunnel/stunnel.conf -# forward request and error logs to docker log collector -RUN ln -sf /dev/stdout /usr/local/nginx/logs/access.log -RUN ln -sf /dev/stderr /usr/local/nginx/logs/error.log - -RUN cp /nginx-rtmp-module-${RTMP_MODULE_VERSION}/stat.xsl /usr/local/nginx/html/ +COPY index.html /usr/local/nginx/html/ +COPY nginx-conf/nginx.conf /base-nginx.conf +COPY launch-nginx-server.sh launch-nginx-server.sh +COPY rtmp-conf-generator.py nginx-template.conf.j2 / EXPOSE 1935 EXPOSE 80 STOPSIGNAL SIGTERM - -# Remove the entry point from jrottenberg/ffmpeg -ENTRYPOINT [] - -CMD ["/bin/sh", "/launch-nginx-server.sh"] +ENTRYPOINT ["/tini", "--"] +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/multistreaming-server/Pipfile b/multistreaming-server/Pipfile new file mode 100644 index 0000000..7696061 --- /dev/null +++ b/multistreaming-server/Pipfile @@ -0,0 +1,17 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +pylint = "*" +black = "*" + +[packages] +jinja2 = "*" + +[requires] +python_version = "3.7" + +[pipenv] +allow_prereleases = true diff --git a/multistreaming-server/Pipfile.lock b/multistreaming-server/Pipfile.lock new file mode 100644 index 0000000..ea3c22a --- /dev/null +++ b/multistreaming-server/Pipfile.lock @@ -0,0 +1,251 @@ +{ + "_meta": { + "hash": { + "sha256": "04efaa28a54c0d1e8e3b9cd66b94cec1914dff6cc5372b8df844639c75ec4a40" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "jinja2": { + "hashes": [ + "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", + "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" + ], + "index": "pypi", + "version": "==2.11.2" + }, + "markupsafe": { + "hashes": [ + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", + "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.1" + } + }, + "develop": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "astroid": { + "hashes": [ + "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703", + "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386" + ], + "markers": "python_version >= '3.5'", + "version": "==2.4.2" + }, + "black": { + "hashes": [ + "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" + ], + "index": "pypi", + "version": "==20.8b1" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "isort": { + "hashes": [ + "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7", + "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58" + ], + "markers": "python_version >= '3.6' and python_version < '4.0'", + "version": "==5.6.4" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.4.3" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "pathspec": { + "hashes": [ + "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", + "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061" + ], + "version": "==0.8.0" + }, + "pylint": { + "hashes": [ + "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210", + "sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f" + ], + "index": "pypi", + "version": "==2.6.0" + }, + "regex": { + "hashes": [ + "sha256:1a16afbfadaadc1397353f9b32e19a65dc1d1804c80ad73a14f435348ca017ad", + "sha256:2308491b3e6c530a3bb38a8a4bb1dc5fd32cbf1e11ca623f2172ba17a81acef1", + "sha256:39a5ef30bca911f5a8a3d4476f5713ed4d66e313d9fb6755b32bec8a2e519635", + "sha256:3d5a8d007116021cf65355ada47bf405656c4b3b9a988493d26688275fde1f1c", + "sha256:4302153abb96859beb2c778cc4662607a34175065fc2f33a21f49eb3fbd1ccd3", + "sha256:463e770c48da76a8da82b8d4a48a541f314e0df91cbb6d873a341dbe578efafd", + "sha256:46ab6070b0d2cb85700b8863b3f5504c7f75d8af44289e9562195fe02a8dd72d", + "sha256:4f5c0fe46fb79a7adf766b365cae56cafbf352c27358fda811e4a1dc8216d0db", + "sha256:60c4f64d9a326fe48e8738c3dbc068e1edc41ff7895a9e3723840deec4bc1c28", + "sha256:671c51d352cfb146e48baee82b1ee8d6ffe357c292f5e13300cdc5c00867ebfc", + "sha256:6cf527ec2f3565248408b61dd36e380d799c2a1047eab04e13a2b0c15dd9c767", + "sha256:7c4fc5a8ec91a2254bb459db27dbd9e16bba1dabff638f425d736888d34aaefa", + "sha256:850339226aa4fec04916386577674bb9d69abe0048f5d1a99f91b0004bfdcc01", + "sha256:8ba3efdd60bfee1aa784dbcea175eb442d059b576934c9d099e381e5a9f48930", + "sha256:8c8c42aa5d3ac9a49829c4b28a81bebfa0378996f9e0ca5b5ab8a36870c3e5ee", + "sha256:8e7ef296b84d44425760fe813cabd7afbb48c8dd62023018b338bbd9d7d6f2f0", + "sha256:a2a31ee8a354fa3036d12804730e1e20d58bc4e250365ead34b9c30bbe9908c3", + "sha256:a63907332531a499b8cdfd18953febb5a4c525e9e7ca4ac147423b917244b260", + "sha256:a8240df4957a5b0e641998a5d78b3c4ea762c845d8cb8997bf820626826fde9a", + "sha256:b8806649983a1c78874ec7e04393ef076805740f6319e87a56f91f1767960212", + "sha256:c077c9d04a040dba001cf62b3aff08fd85be86bccf2c51a770c77377662a2d55", + "sha256:c529ba90c1775697a65b46c83d47a2d3de70f24d96da5d41d05a761c73b063af", + "sha256:d537e270b3e6bfaea4f49eaf267984bfb3628c86670e9ad2a257358d3b8f0955", + "sha256:d629d750ebe75a88184db98f759633b0a7772c2e6f4da529f0027b4a402c0e2f", + "sha256:d9d53518eeed12190744d366ec4a3f39b99d7daa705abca95f87dd8b442df4ad", + "sha256:e490f08897cb44e54bddf5c6e27deca9b58c4076849f32aaa7a0b9f1730f2c20", + "sha256:f579caecbbca291b0fcc7d473664c8c08635da2f9b1567c22ea32311c86ef68c" + ], + "version": "==2020.10.11" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "toml": { + "hashes": [ + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" + ], + "version": "==0.10.1" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "markers": "python_version < '3.8' and implementation_name == 'cpython'", + "version": "==1.4.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + }, + "wrapt": { + "hashes": [ + "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" + ], + "version": "==1.12.1" + } + } +} diff --git a/multistreaming-server/launch-nginx-server.sh b/multistreaming-server/launch-nginx-server.sh index d5cad91..84dd819 100644 --- a/multistreaming-server/launch-nginx-server.sh +++ b/multistreaming-server/launch-nginx-server.sh @@ -4,11 +4,8 @@ export DOLLAR='$' envsubst < base-nginx.conf > /usr/local/nginx/conf/nginx.conf -# start stunnel -/usr/bin/stunnel & - # append nginx conf with RTMP Configuration -python3 /rtmp-conf-generator.py /rtmp-configuration.json >> /usr/local/nginx/conf/nginx.conf +python3 /rtmp-conf-generator.py /rtmp-configuration.json /nginx-template.conf.j2 >> /usr/local/nginx/conf/nginx.conf if [ $? -ne 0 ]; then echo "ERROR encountered when generating RTMP configuration." exit 1 diff --git a/multistreaming-server/nginx-conf/nginx.conf b/multistreaming-server/nginx-conf/nginx.conf index c0a27b9..ba5b409 100644 --- a/multistreaming-server/nginx-conf/nginx.conf +++ b/multistreaming-server/nginx-conf/nginx.conf @@ -12,7 +12,7 @@ http { # '$status $body_bytes_sent "$http_referer" ' # '"$http_user_agent" "$http_x_forwarded_for"'; - #access_log logs/access.log main; + access_log /dev/stdout; sendfile on; #tcp_nopush on; diff --git a/multistreaming-server/nginx-template.conf.j2 b/multistreaming-server/nginx-template.conf.j2 new file mode 100644 index 0000000..dbbe7d4 --- /dev/null +++ b/multistreaming-server/nginx-template.conf.j2 @@ -0,0 +1,79 @@ +{% if nginx_error_log %} error_log /dev/stderr debug; {% endif %} + +rtmp { + server { + listen 1935; + chunk_size 4096; + notify_method get; + + application {{ endpoint_name }} { + on_publish http://127.0.0.1/auth; + live on; + record {{ record_mode }}; + record_path /var/www/html/recordings; + record_unique on; + + # Define the applications to which the stream will be pushed, comment them out to disable the ones not needed: + {% for name in transcode_configs.keys() %} + push rtmp://127.0.0.1:1935/transcode_{{name}}; + {% endfor %} + {% for application_endpoint in push_only_applications %} + push rtmp://127.0.0.1:1935/{{application_endpoint}}; + {% endfor %} + } + + # transcode definitions + {% for name, transcode_config in transcode_configs.items() %} + application transcode_{{name}} { + live on; + record off; + + # Only allow 127.0.0.1 to publish + allow publish 127.0.0.1; + deny publish all; + + # need to transcode + exec ffmpeg -re -i rtmp://127.0.0.1:1935/$app/$name + -c:v libx264 + -s {{transcode_config['pixels']}} + -b:v {{transcode_config['videoBitRate']}} + -bufsize 12M + -r {{transcode_config['videoFrameRate']}} + -x264opts "keyint={{transcode_config['keyFrames']}}:min-keyint={{transcode_config['keyFrames']}}:no-scenecut:nal-hrd=cbr" + {{transcode_config['audioOpts']}} + {% if transcode_config['maxMuxingQueueSize'] %}-max_muxing_queue_size {{transcode_config['maxMuxingQueueSize']}}{% endif %} + -f flv rtmp://127.0.0.1:1935/transcode_output_{{name}}/$name + {% if transcode_config['logFfmpeg'] %} 2>>/tmp/ffmpeg-{{name}}.log {% endif %} + ; + } + + application transcode_output_{{name}} { + live on; + record off; + + # Only allow 127.0.0.1 to publish + allow publish 127.0.0.1; + deny publish all; + + {% for application_endpoint in transcode_config['applicationEndpoints'] %} + push rtmp://127.0.0.1:1935/{{application_endpoint}}; + {% endfor %} + } + {% endfor %} + + # application defintions + {% for name, application_config in application_configs.items() %} + application {{ name }} { + live on; + record off; + + # Only allow 127.0.0.1 to publish + allow publish 127.0.0.1; + deny publish all; + + # Push URL + push {{ application_config['pushUrl'] }}; + } + {% endfor %} + } +} \ No newline at end of file diff --git a/multistreaming-server/rtmp-conf-generator.py b/multistreaming-server/rtmp-conf-generator.py old mode 100644 new mode 100755 index bbb26d7..66248cb --- a/multistreaming-server/rtmp-conf-generator.py +++ b/multistreaming-server/rtmp-conf-generator.py @@ -1,104 +1,86 @@ +#!/usr/bin/env python3 + import json import re import sys +import os +import jinja2 # -# Configuration Templates +# Configurable ENV VARS # -RTMP_CONF_BLOCK = """ -rtmp { - server { - listen 1935; - chunk_size 4096; - notify_method get; - - application %%ENDPOINT_NAME%% { - on_publish http://127.0.0.1/auth; - live on; - record all; - record_path /var/www/html/recordings; - record_unique on; - - # Define the applications to which the stream will be pushed, comment them out to disable the ones not needed: - # RTMP_PUSH_DIRECTIVE_MARKER - } - - # application defintions - # RTMP_PUSH_BLOCK_MARKER - } -} -""" - -RTMP_PUSH_BLOCK = """ - application %%BLOCK_NAME%% { - live on; - record off; - - # Only allow 127.0.0.1 to publish - allow publish 127.0.0.1; - deny publish all; +CONFIG_NGINX_DEBUG = os.getenv('CONFIG_NGINX_DEBUG', False) +CONFIG_FFMPEG_LOG = os.getenv('CONFIG_FFMPEG_LOG', False) +CONFIG_FFMPEG_MAX_MUXING_QUEUE_SIZE = os.getenv( + 'CONFIG_FFMPEG_MAX_MUXING_QUEUE_SIZE', False +) +CONFIG_DISABLE_RECORD = os.getenv('CONFIG_DISABLE_RECORD', False) - # Push URL - push %%PUSH_URL%%; - } - -""" +# +# Defaults +# -RTMP_TRANSCODE_BLOCK = """ - application %%BLOCK_NAME%% { - live on; - record off; +RTMP_TRANSCODE_AUDIO_OPTS_COPY = '-c:a copy' +RTMP_TRANSCODE_AUDIO_OPTS_CUSTOM = ( + '-c:a libfdk_aac -b:a %%AUDIO_BIT_RATE%% -ar %%AUDIO_SAMPLE_RATE%%' +) + +PUSH_URL_YOUTUBE = 'rtmp://a.rtmp.youtube.com/live2/%%STREAM_KEY%%' +PUSH_URL_FACEBOOK = 'rtmp://127.0.0.1:19350/rtmp/%%STREAM_KEY%%' +PUSH_URL_TWITCH = 'rtmp://live-cdg.twitch.tv/app/%%STREAM_KEY%%' +PUSH_URL_INSTAGRAM = 'rtmp://127.0.0.1:19351/rtmp/%%STREAM_KEY%%' +PUSH_URL_PERISCOPE = 'rtmp://%%REGION_CODE%%.pscp.tv:80/x/%%STREAM_KEY%%' +PUSH_URL_MICROSOFT_STREAM = '%%RTMP_URL%% app=live/%%APP_NAME%%' +PUSH_URL_MIXCLOUD = 'rtmp://rtmp.mixcloud.com/broadcast/%%STREAM_KEY%%' +PUSH_URL_DLIVE = 'rtmp://stream.dlive.tv/live/%%STREAM_KEY%%' + + +DEFAULT_TRANSCODE_CONFIG = { + 'pixels': '1280x720', + 'videoBitRate': '4500k', + 'videoFrameRate': 60, + 'keyFrames': 60, + 'audioOpts': '-c:a copy', + 'logFfmpeg': True if CONFIG_FFMPEG_LOG else False, + 'applicationEndpoints': set(), + 'maxMuxingQueueSize': int(CONFIG_FFMPEG_MAX_MUXING_QUEUE_SIZE) + if CONFIG_FFMPEG_MAX_MUXING_QUEUE_SIZE + else None, +} - # Only allow 127.0.0.1 to publish - allow publish 127.0.0.1; - deny publish all; +DEFAULT_AUDIO_OPTS = { + 'audioBitRate': '160k', + 'audioSampleRate': '48000', +} - # need to transcode - exec ffmpeg -re -i rtmp://127.0.0.1:1935/$app/$name - -c:v libx264 -s %%PIXEL_SIZE%% -b:v %%VIDEO_BIT_RATE%% -bufsize 12M -r 30 -x264opts "keyint=%%KFS%%:min-keyint=%%KFS%%:no-scenecut:nal-hrd=cbr" - %%AUDIO_OPTS%% - -f flv rtmp://127.0.0.1:1935/%%DEST_BLOCK_NAME%%/$name; - } -""" - -RTMP_TRANSCODE_AUDIO_OPTS_COPY = "-c:a copy" -RTMP_TRANSCODE_AUDIO_OPTS_CUSTOM = "-c:a libfdk_aac -b:a %%AUDIO_BIT_RATE%% -ar %%AUDIO_SAMPLE_RATE%%" - -PUSH_URL_YOUTUBE = "rtmp://a.rtmp.youtube.com/live2/%%STREAM_KEY%%" -PUSH_URL_FACEBOOK = "rtmp://127.0.0.1:19350/rtmp/%%STREAM_KEY%%" -PUSH_URL_TWITCH = "rtmp://live-cdg.twitch.tv/app/%%STREAM_KEY%%" -PUSH_URL_INSTAGRAM = "rtmp://127.0.0.1:19351/rtmp/%%STREAM_KEY%%" -PUSH_URL_PERISCOPE = "rtmp://%%REGION_CODE%%.pscp.tv:80/x/%%STREAM_KEY%%" -PUSH_URL_MICROSOFT_STREAM = "%%RTMP_URL%% app=live/%%APP_NAME%%" -PUSH_URL_MIXCLOUD = "rtmp://rtmp.mixcloud.com/broadcast/%%STREAM_KEY%%" -PUSH_URL_DLIVE = "rtmp://stream.dlive.tv/live/%%STREAM_KEY%%" -# -# -# def generatePlatormPushURL(block_config): if 'platform' not in block_config: - print("ERROR - Application block is missing platform element.", file=sys.stderr) + print('ERROR - Application block is missing platform element.', file=sys.stderr) exit(1) push_url = 'push-it-real-good' if block_config['platform'] == 'youtube': push_url = PUSH_URL_YOUTUBE.replace('%%STREAM_KEY%%', block_config['streamKey']) elif block_config['platform'] == 'facebook': # must push through stunnel. Push through Facebook stunnel port. - push_url = PUSH_URL_FACEBOOK.replace('%%STREAM_KEY%%', block_config['streamKey']) + push_url = PUSH_URL_FACEBOOK.replace( + '%%STREAM_KEY%%', block_config['streamKey'] + ) elif block_config['platform'] == 'twitch': push_url = PUSH_URL_TWITCH.replace('%%STREAM_KEY%%', block_config['streamKey']) elif block_config['platform'] == 'instagram': # must push through stunnel. Push through Instagram stunnel port. - push_url = PUSH_URL_INSTAGRAM.replace('%%STREAM_KEY%%', block_config['streamKey']) + push_url = PUSH_URL_INSTAGRAM.replace( + '%%STREAM_KEY%%', block_config['streamKey'] + ) elif block_config['platform'] == 'periscope': - region_code = block_config['regionCode'] if 'regionCode' in block_config else 'ca' + region_code = ( + block_config['regionCode'] if 'regionCode' in block_config else 'ca' + ) push_url = PUSH_URL_PERISCOPE.replace( '%%STREAM_KEY%%', block_config['streamKey'] - ).replace( - '%%REGION_CODE%%', region_code - ) + ).replace('%%REGION_CODE%%', region_code) elif block_config['platform'] == 'custom': push_url = block_config['customRTMPURL'] elif block_config['platform'] == 'microsoft-stream': @@ -106,102 +88,137 @@ def generatePlatormPushURL(block_config): ms_rtmp_url = re.search(r'^(.*)/live/', ms_source_url).group(1) ms_app_name = re.search(r'/live/(.*)$', ms_source_url).group(1) push_url = PUSH_URL_MICROSOFT_STREAM.replace( - '%%RTMP_URL%%', ms_rtmp_url - ).replace( - '%%APP_NAME%%', ms_app_name - ) + '%%RTMP_URL%%', ms_rtmp_url + ).replace('%%APP_NAME%%', ms_app_name) elif block_config['platform'] == 'mixcloud': - push_url = PUSH_URL_MIXCLOUD.replace('%%STREAM_KEY%%', block_config['streamKey']) + push_url = PUSH_URL_MIXCLOUD.replace( + '%%STREAM_KEY%%', block_config['streamKey'] + ) elif block_config['platform'] == 'dlive': push_url = PUSH_URL_DLIVE.replace('%%STREAM_KEY%%', block_config['streamKey']) else: - print("ERROR - an unsupported platform type was provided in destination configation", file=sys.stderr) + print( + 'ERROR - an unsupported platform type was provided in destination configation', + file=sys.stderr, + ) exit(1) return push_url -def createRTMPApplicationBlocks(block_name, block_config): - app_block = '' - primary_block_name = block_name - if 'transcode' in block_config: - primary_block_name += '_transcoded' - tc_conf = block_config['transcode'] - pixel_size = tc_conf['pixels'] if 'pixels' in tc_conf else '1280x720' - video_bit_rate = tc_conf['videoBitRate'] if 'videoBitRate' in tc_conf else '4500k' - key_frames = 30 * \ - tc_conf['videoKeyFrameSecs'] if 'videoKeyFrameSecs' in tc_conf else 60 - if ('audioBitRate' in tc_conf) or ('audioSampleRate' in tc_conf): - audio_bit_rate = tc_conf['audioBitRate'] if 'audioBitRate' in tc_conf else '160k' - audio_sample_rate = str(tc_conf['audioSampleRate']) if 'audioSampleRate' in tc_conf else '48000' - audio_opts = RTMP_TRANSCODE_AUDIO_OPTS_CUSTOM.replace( - '%%AUDIO_BIT_RATE%%', audio_bit_rate - ).replace( - '%%AUDIO_SAMPLE_RATE%%', audio_sample_rate - ) - else: - audio_opts = RTMP_TRANSCODE_AUDIO_OPTS_COPY - - app_block += RTMP_TRANSCODE_BLOCK.replace( - '%%BLOCK_NAME%%', block_name - ).replace( - '%%DEST_BLOCK_NAME%%', primary_block_name - ).replace( - '%%PIXEL_SIZE%%', pixel_size - ).replace( - '%%VIDEO_BIT_RATE%%', video_bit_rate - ).replace( - '%%KFS%%', str(key_frames) - ).replace( - '%%AUDIO_OPTS%%', audio_opts - ) - app_block += RTMP_PUSH_BLOCK.replace( - '%%BLOCK_NAME%%', primary_block_name +def generateTranscodeConfig(transcode_config_name, block_config, config): + block_config_name = block_config['name'] + default_transcode_config = DEFAULT_TRANSCODE_CONFIG.copy() + transcode_config_block = config.get('transcodeProfiles', {}).get( + transcode_config_name, block_config.get('transcode') + ) + if not transcode_config_block: + print( + f'ERROR - unable to resolve transcode profile for {transcode_config_name} in block {block_config_name}', + file=sys.stderr, + ) + exit(1) + transcode_config = { + key: transcode_config_block.get(key, DEFAULT_TRANSCODE_CONFIG[key]) + for key in default_transcode_config.keys() + } + + if 'videoKeyFrameSecs' in transcode_config_block: + transcode_config['keyFrames'] = 30 * transcode_config_block['videoKeyFrameSecs'] + + if ('audioBitRate' in transcode_config_block) or ( + 'audioSampleRate' in transcode_config_block + ): + transcode_config['audioOpts'] = RTMP_TRANSCODE_AUDIO_OPTS_CUSTOM.replace( + '%%AUDIO_BIT_RATE%%', + transcode_config_block.get( + 'audioBitRate', DEFAULT_AUDIO_OPTS['audioBitRate'] + ), ).replace( - '%%PUSH_URL%%', generatePlatormPushURL(block_config) + '%%AUDIO_SAMPLE_RATE%%', + transcode_config_block.get( + 'audioSampleRate', DEFAULT_AUDIO_OPTS['audioSampleRate'] + ), ) - return app_block + return transcode_config -def addRTMPPushConfiguration(orig_rtmp_conf, block_config, endpoint_name): - if 'name' not in block_config: - print("ERROR - Application block is missing name element.", file=sys.stderr) - exit(1) - block_name = endpoint_name + '-' + block_config['name'] - push_pos = orig_rtmp_conf.index(' # RTMP_PUSH_DIRECTIVE_MARKER') - rtmp_conf = orig_rtmp_conf[:push_pos] \ - + ' push rtmp://127.0.0.1/' \ - + block_name \ - + ';\n' \ - + orig_rtmp_conf[push_pos:] - block_pos = rtmp_conf.index(' # RTMP_PUSH_BLOCK_MARKER') - rtmp_conf = rtmp_conf[:block_pos] \ - + createRTMPApplicationBlocks(block_name, block_config) \ - + rtmp_conf[block_pos:] - return rtmp_conf - - -if len(sys.argv) != 2: - print("Must pass a single argument of the JSON configuration file path.", file=sys.stderr) - sys.exit(1) - -try: - with open(sys.argv[1], 'r') as f: - config = json.load(f) -except json.decoder.JSONDecodeError as err: - print('ERROR decoding JSON config file "{0}": {1}'.format(sys.argv[1], err), file=sys.stderr) - exit(1) -except: - print('ERROR loading JSON config file "{0}"'.format(sys.argv[1]), file=sys.stderr) - exit(1) - -endpoint_name = config['endpoint'] if 'endpoint' in config else 'live' -rtmp_conf = RTMP_CONF_BLOCK.replace('%%ENDPOINT_NAME%%', endpoint_name, 1) - -# unfortunate hack to make code compatible with previous spelling error -config_list_key = 'rebroacastList' if 'rebroacastList' in config else 'rebroadcastList' -for block_config in config[config_list_key]: - rtmp_conf = addRTMPPushConfiguration( - rtmp_conf, block_config, endpoint_name +def loadJsonConfig(path): + try: + with open(path, 'r') as f: + config = json.load(f) + except json.decoder.JSONDecodeError as err: + print( + "ERROR decoding JSON config file '{0}': {1}".format(path, err), + file=sys.stderr, ) + exit(1) + except: + print("ERROR loading JSON config file '{0}'".format(path), file=sys.stderr) + exit(1) + + return config + -print(rtmp_conf) +def generateConfig(config_file, nginx_config_template): + config = loadJsonConfig(config_file) + # unfortunate hack to make code compatible with previous spelling error + config_list_key = ( + 'rebroacastList' if 'rebroacastList' in config else 'rebroadcastList' + ) + + record_mode = 'off' if CONFIG_DISABLE_RECORD else 'all' + nginx_error_log = True if CONFIG_NGINX_DEBUG else False + + endpoint_name = config['endpoint'] if 'endpoint' in config else 'live' + + application_configs = {} + transcode_configs = {} + push_only_applications = set() + + for block_config in config[config_list_key]: + if block_config.get('disabled', False): + continue + + block_config_name = block_config['name'] + application_configs[block_config_name] = { + 'pushUrl': generatePlatormPushURL(block_config) + } + + if ('transcodeProfile' in block_config) or 'transcode' in block_config: + transcode_config_name = block_config.get( + 'transcodeProfile', f'inline_{block_config_name}' + ) + + if transcode_config_name not in transcode_configs: + transcode_configs[transcode_config_name] = generateTranscodeConfig( + transcode_config_name, block_config, config + ) + transcode_configs[transcode_config_name]['applicationEndpoints'].add( + block_config_name + ) + + else: + push_only_applications.add(block_config_name) + + with open(nginx_config_template) as fh: + template = jinja2.Template(fh.read()) + + return template.render( + nginx_error_log=nginx_error_log, + endpoint_name=endpoint_name, + record_mode=record_mode, + transcode_configs=transcode_configs, + push_only_applications=push_only_applications, + application_configs=application_configs, + ) + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print( + 'Must pass two arguments of the JSON configuration file path and the nginx config.', + file=sys.stderr, + ) + sys.exit(1) + rtmp_conf = generateConfig(sys.argv[1], sys.argv[2]) + print(rtmp_conf) diff --git a/multistreaming-server/stunnel-conf/etc-stunnel-stunnel.conf b/multistreaming-server/stunnel-conf/etc-stunnel-stunnel.conf index 3401826..1a94fd0 100644 --- a/multistreaming-server/stunnel-conf/etc-stunnel-stunnel.conf +++ b/multistreaming-server/stunnel-conf/etc-stunnel-stunnel.conf @@ -1,5 +1,6 @@ +foreground = yes setuid = stunnel setgid = stunnel -pid=/tmp/stunnel.pid -output = /var/run/stunnel/stunnel.log -include = /etc/stunnel/conf.d +pid= /tmp/stunnel.pid +output = /tmp/stunnel.log +include = /etc/stunnel/conf.d \ No newline at end of file diff --git a/multistreaming-server/supervisord.conf b/multistreaming-server/supervisord.conf new file mode 100644 index 0000000..25d2067 --- /dev/null +++ b/multistreaming-server/supervisord.conf @@ -0,0 +1,31 @@ +[supervisord] +nodaemon=true +user=root + +[supervisorctl] + +[program:stunnel] +command=/usr/bin/stunnel +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +autorestart=true + + +[program:stunnel-logs] +command=tail -f /tmp/stunnel.log +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +autorestart=true + + +[program:nginx] +command=sh /launch-nginx-server.sh +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +autorestart=true