From 598ec853535e8a8d791966d74a2fb700263ce31c Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Mon, 17 Jul 2023 16:07:17 +1000 Subject: [PATCH 1/4] WIP: Fixing new headscale issues --- packages/grid/docker-compose.yml | 4 + packages/grid/vpn/config.yaml | 202 +++++++++++++----- packages/grid/vpn/connect_vpn.py | 2 +- packages/grid/vpn/headscale.dockerfile | 13 +- packages/grid/vpn/headscale.py | 4 +- packages/grid/vpn/headscale.sh | 17 +- packages/grid/vpn/tailscale.dockerfile | 2 +- .../src/syft/service/vpn/headscale_client.py | 5 +- .../src/syft/service/vpn/tailscale_client.py | 8 +- 9 files changed, 189 insertions(+), 68 deletions(-) diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 2309749dfe8..375668f7ce7 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -24,6 +24,7 @@ services: - HTTPS_PORT=${HTTPS_PORT} ports: - "${HTTP_PORT}:81" + - "41641/udp" extra_hosts: - "host.docker.internal:host-gateway" @@ -211,6 +212,9 @@ services: - RELEASE=${RELEASE:-production} - NETWORK_NAME=omnet - STACK_API_KEY=$STACK_API_KEY + ports: + - "3478/udp" + - "8080:8080" # seaweedfs: # profiles: diff --git a/packages/grid/vpn/config.yaml b/packages/grid/vpn/config.yaml index 00fe10d42f1..d09248f00e0 100644 --- a/packages/grid/vpn/config.yaml +++ b/packages/grid/vpn/config.yaml @@ -10,23 +10,28 @@ # # https://myheadscale.example.com:443 # -server_url: http://127.0.0.1:8080 +server_url: http://0.0.0.0:8080 # Address to listen to / bind to on the server # +# For production: +# listen_addr: 0.0.0.0:8080 listen_addr: 0.0.0.0:8080 # Address to listen to /metrics, you may want # to keep this endpoint private to your internal # network # -metrics_listen_addr: 127.0.0.1:9090 +metrics_listen_addr: 0.0.0.0:9090 # Address to listen for gRPC. # gRPC is used for controlling a headscale server # remotely with the CLI # Note: Remote access _only_ works if you have # valid certificates. +# +# For production: +# grpc_listen_addr: 0.0.0.0:50443 grpc_listen_addr: 0.0.0.0:50443 # Allow the gRPC admin interface to run in INSECURE @@ -35,15 +40,30 @@ grpc_listen_addr: 0.0.0.0:50443 # are doing. grpc_allow_insecure: false -# Private key used encrypt the traffic between headscale +# Private key used to encrypt the traffic between headscale # and Tailscale clients. -# The private key file which will be -# autogenerated if it's missing -private_key_path: /headscale/data/private_v2.key +# The private key file will be autogenerated if it's missing. +# +private_key_path: /headscale/data/private.key + +# The Noise section includes specific configuration for the +# TS2021 Noise protocol +noise: + # The Noise private key is used to encrypt the + # traffic between headscale and Tailscale clients when + # using the new Noise-based protocol. It must be different + # from the legacy private key. + private_key_path: /headscale/data/noise_private.key # List of IP prefixes to allocate tailaddresses from. # Each prefix consists of either an IPv4 or IPv6 address, # and the associated prefix length, delimited by a slash. +# It must be within IP ranges supported by the Tailscale +# client - i.e., subnets of 100.64.0.0/10 and fd7a:115c:a1e0::/48. +# See below: +# IPv6: https://github.com/tailscale/tailscale/blob/22ebb25e833264f58d7c3f534a8b166894a89536/net/tsaddr/tsaddr.go#LL81C52-L81C71 +# IPv4: https://github.com/tailscale/tailscale/blob/22ebb25e833264f58d7c3f534a8b166894a89536/net/tsaddr/tsaddr.go#L33 +# Any other range is NOT supported, and it will cause unexpected issues. ip_prefixes: # - fd7a:115c:a1e0::/48 - 100.64.0.0/10 @@ -58,7 +78,7 @@ derp: server: # If enabled, runs the embedded DERP server and merges it into the rest of the DERP config # The Headscale server_url defined above MUST be using https, DERP requires TLS to be in place - enabled: true + enabled: false # Region ID to use for the embedded DERP server. # The local DERP prevails if the region ID collides with other region ID coming from @@ -69,7 +89,7 @@ derp: region_code: "headscale" region_name: "Headscale Embedded DERP" - # Listens in UDP at the configured address for STUN connections to help on NAT traversal. + # Listens over UDP at the configured address for STUN connections - to help with NAT traversal. # When the embedded DERP server is enabled stun_listen_addr MUST be defined. # # For more details on how this works, check this great article: https://tailscale.com/blog/how-tailscale-works/ @@ -103,11 +123,20 @@ disable_check_updates: false # Time before an inactive ephemeral node is deleted? ephemeral_node_inactivity_timeout: 30m +# Period to check for node updates within the tailnet. A value too low will severely affect +# CPU consumption of Headscale. A value too high (over 60s) will cause problems +# for the nodes, as they won't get updates or keep alive messages frequently enough. +# In case of doubts, do not touch the default 10s. +node_update_check_interval: 10s + # SQLite config db_type: sqlite3 + +# For production: db_path: /headscale/data/db.sqlite # # Postgres config +# If using a Unix socket to connect to Postgres, set the socket path in the 'host' field and leave 'port' blank. # db_type: postgres # db_host: localhost # db_port: 5432 @@ -115,6 +144,10 @@ db_path: /headscale/data/db.sqlite # db_user: foo # db_pass: bar +# If other 'sslmode' is required instead of 'require(true)' and 'disabled(false)', set the 'sslmode' you need +# in the 'db_ssl' field. Refers to https://www.postgresql.org/docs/current/libpq-ssl.html Table 34.1. +# db_ssl: false + ### TLS configuration # ## Let's encrypt / ACME @@ -123,39 +156,36 @@ db_path: /headscale/data/db.sqlite # TLS for a domain with Let's Encrypt. # # URL to ACME directory -acme_url: https://acme-v02.api.letsencrypt.org/directory - -# Email to register with ACME provider -acme_email: "" - -# Domain name to request a TLS certificate for: -tls_letsencrypt_hostname: "" - -# Client (Tailscale/Browser) authentication mode (mTLS) -# Acceptable values: -# - disabled: client authentication disabled -# - relaxed: client certificate is required but not verified -# - enforced: client certificate is required and verified -tls_client_auth_mode: relaxed - -# Path to store certificates and metadata needed by -# letsencrypt -tls_letsencrypt_cache_dir: /var/lib/headscale/cache - -# Type of ACME challenge to use, currently supported types: -# HTTP-01 or TLS-ALPN-01 -# See [docs/tls.md](docs/tls.md) for more information -tls_letsencrypt_challenge_type: HTTP-01 -# When HTTP-01 challenge is chosen, letsencrypt must set up a -# verification endpoint, and it will be listning on: -# :http = port 80 -tls_letsencrypt_listen: ":http" - -## Use already defined certificates: +# acme_url: https://acme-v02.api.letsencrypt.org/directory + +# # Email to register with ACME provider +# acme_email: "" + +# # Domain name to request a TLS certificate for: +# tls_letsencrypt_hostname: "" + +# # Path to store certificates and metadata needed by +# # letsencrypt +# # For production: +# tls_letsencrypt_cache_dir: /var/lib/headscale/cache + +# # Type of ACME challenge to use, currently supported types: +# # HTTP-01 or TLS-ALPN-01 +# # See [docs/tls.md](docs/tls.md) for more information +# tls_letsencrypt_challenge_type: HTTP-01 +# # When HTTP-01 challenge is chosen, letsencrypt must set up a +# # verification endpoint, and it will be listening on: +# # :http = port 80 +# tls_letsencrypt_listen: ":http" + +# ## Use already defined certificates: tls_cert_path: "" tls_key_path: "" -log_level: info +log: + # Output formatting for logs: text or json + format: text + level: info # Path to a file containg ACL policies. # ACLs can be defined as YAML or HUJSON. @@ -172,10 +202,26 @@ acl_policy_path: "" # - https://tailscale.com/blog/2021-09-private-dns-with-magicdns/ # dns_config: + # Whether to prefer using Headscale provided DNS or use local. + # override_local_dns: true + override_local_dns: false + # List of DNS servers to expose to clients. nameservers: - 1.1.1.1 + # NextDNS (see https://tailscale.com/kb/1218/nextdns/). + # "abc123" is example NextDNS ID, replace with yours. + # + # With metadata sharing: + # nameservers: + # - https://dns.nextdns.io/abc123 + # + # Without metadata sharing: + # nameservers: + # - 2a07:a8c0::ab:c123 + # - 2a07:a8c1::ab:c123 + # Split DNS (see https://tailscale.com/kb/1054/dns/), # list of search domains and the DNS to query for each one. # @@ -189,20 +235,31 @@ dns_config: # Search domains to inject. domains: [] + # Extra DNS records + # so far only A-records are supported (on the tailscale side) + # See https://github.com/juanfont/headscale/blob/main/docs/dns-records.md#Limitations + # extra_records: + # - name: "grafana.myvpn.example.com" + # type: "A" + # value: "100.64.0.3" + # + # # you can also put it in one line + # - { name: "prometheus.myvpn.example.com", type: "A", value: "100.64.0.3" } + # Whether to use [MagicDNS](https://tailscale.com/kb/1081/magicdns/). # Only works if there is at least a nameserver defined. - magic_dns: true + magic_dns: false + # magic_dns: true # Defines the base domain to create the hostnames for MagicDNS. # `base_domain` must be a FQDNs, without the trailing dot. # The FQDN of the hosts will be - # `hostname.namespace.base_domain` (e.g., _myhost.mynamespace.example.com_). - base_domain: openmined.org + # `hostname.user.base_domain` (e.g., _myhost.myuser.example.com_). + base_domain: example.com # Unix socket used for the CLI to connect without authentication -# Note: for local development, you probably want to change this to: -# unix_socket: ./headscale.sock -unix_socket: /var/run/headscale.sock +# Note: for production you will want to set this to something like: +unix_socket: /var/run/headscale/headscale.sock unix_socket_permission: "0770" # # headscale supports experimental OpenID connect support, @@ -210,13 +267,62 @@ unix_socket_permission: "0770" # help us test it. # OpenID Connect # oidc: +# only_start_if_oidc_is_available: true # issuer: "https://your-oidc.issuer.com/path" # client_id: "your-oidc-client-id" # client_secret: "your-oidc-client-secret" +# # Alternatively, set `client_secret_path` to read the secret from the file. +# # It resolves environment variables, making integration to systemd's +# # `LoadCredential` straightforward: +# client_secret_path: "${CREDENTIALS_DIRECTORY}/oidc_client_secret" +# # client_secret and client_secret_path are mutually exclusive. +# +# # The amount of time from a node is authenticated with OpenID until it +# # expires and needs to reauthenticate. +# # Setting the value to "0" will mean no expiry. +# expiry: 180d # -# If `strip_email_domain` is set to `true`, the domain part of the username email address will be removed. -# This will transform `first-name.last-name@example.com` to the namespace `first-name.last-name` -# If `strip_email_domain` is set to `false` the domain part will NOT be removed resulting to the following -# namespace: `first-name.last-name.example.com` +# # Use the expiry from the token received from OpenID when the user logged +# # in, this will typically lead to frequent need to reauthenticate and should +# # only been enabled if you know what you are doing. +# # Note: enabling this will cause `oidc.expiry` to be ignored. +# use_expiry_from_token: false +# +# # Customize the scopes used in the OIDC flow, defaults to "openid", "profile" and "email" and add custom query +# # parameters to the Authorize Endpoint request. Scopes default to "openid", "profile" and "email". +# +# scope: ["openid", "profile", "email", "custom"] +# extra_params: +# domain_hint: example.com +# +# # List allowed principal domains and/or users. If an authenticated user's domain is not in this list, the +# # authentication request will be rejected. +# +# allowed_domains: +# - example.com +# # Note: Groups from keycloak have a leading '/' +# allowed_groups: +# - /headscale +# allowed_users: +# - alice@example.com +# +# # If `strip_email_domain` is set to `true`, the domain part of the username email address will be removed. +# # This will transform `first-name.last-name@example.com` to the user `first-name.last-name` +# # If `strip_email_domain` is set to `false` the domain part will NOT be removed resulting to the following +# user: `first-name.last-name.example.com` # # strip_email_domain: true + +# Logtail configuration +# Logtail is Tailscales logging and auditing infrastructure, it allows the control panel +# to instruct tailscale nodes to log their activity to a remote server. +logtail: + # Enable logtail for this headscales clients. + # As there is currently no support for overriding the log server in headscale, this is + # disabled by default. Enabling this will make your clients send logs to Tailscale Inc. + enabled: false + +# Enabling this option makes devices prefer a random port for WireGuard traffic over the +# default static port 41641. This option is intended as a workaround for some buggy +# firewall devices. See https://tailscale.com/kb/1181/firewalls/ for more information. +randomize_client_port: false diff --git a/packages/grid/vpn/connect_vpn.py b/packages/grid/vpn/connect_vpn.py index d7ca16c42d2..fb62961d0ed 100644 --- a/packages/grid/vpn/connect_vpn.py +++ b/packages/grid/vpn/connect_vpn.py @@ -57,7 +57,7 @@ def connect_with_key(tailscale_host: str, headscale_host: str, authkey: str) -> # /etc/resolv.conf has the mDNS ip nameserver 127.0.0.11 data = { "args": [ - "-login-server", + "--login-server", f"{headscale_host}", "--reset", "--force-reauth", diff --git a/packages/grid/vpn/headscale.dockerfile b/packages/grid/vpn/headscale.dockerfile index e1f8b31dcf6..f5fd15f106f 100644 --- a/packages/grid/vpn/headscale.dockerfile +++ b/packages/grid/vpn/headscale.dockerfile @@ -1,9 +1,11 @@ -FROM headscale/headscale:0.15.0-alpine +FROM headscale/headscale:0.22.3 -RUN --mount=type=cache,target=/var/cache/apk \ - apk -U upgrade || true; \ - apk fix || true; \ - apk add --no-cache python3 py3-pip curl bash || true +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + DEBIAN_FRONTEND=noninteractive \ + apt-get update && \ + apt-get install -yqq \ + python3 python3-pip curl procps && \ + rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip @@ -19,6 +21,7 @@ RUN --mount=type=cache,target=/root/.cache \ COPY ./headscale.sh /headscale/headscale.sh COPY ./config.yaml /etc/headscale/config.yaml COPY ./headscale.py /headscale/headscale.py +RUN mkdir -p /headscale/data ENV NETWORK_NAME="omnet" diff --git a/packages/grid/vpn/headscale.py b/packages/grid/vpn/headscale.py index 0bd406d0ede..e3d358647dd 100644 --- a/packages/grid/vpn/headscale.py +++ b/packages/grid/vpn/headscale.py @@ -57,7 +57,7 @@ def generate_key_callback(context: Dict, future: Future) -> None: shell2http.register_command( endpoint="generate_key", - command_name=f"headscale -n {network_name} preauthkeys create -o json", + command_name=f"headscale -u {network_name} preauthkeys create -o json", callback_fn=generate_key_callback, decorators=[basic_auth_check], ) @@ -70,7 +70,7 @@ def list_nodes_callback(context: Dict, future: Future) -> None: shell2http.register_command( endpoint="list_nodes", - command_name=f"headscale -n {network_name} nodes list -o json", + command_name=f"headscale -u {network_name} nodes list -o json", callback_fn=list_nodes_callback, decorators=[basic_auth_check], ) diff --git a/packages/grid/vpn/headscale.sh b/packages/grid/vpn/headscale.sh index 32fdd620c82..55ce42ba66e 100755 --- a/packages/grid/vpn/headscale.sh +++ b/packages/grid/vpn/headscale.sh @@ -1,9 +1,8 @@ -#!/bin/ash +#!/bin/bash export PATH="/root/.local/bin:${PATH}" export FLASK_APP=headscale export NETWORK_NAME="${1}" -flask run -p 4000 --host=0.0.0.0& # start server in background headscale serve& @@ -11,11 +10,15 @@ headscale serve& # Wait for headscale to start waitforit -address=http://localhost:8080/health -status=200 -timeout=60 -- echo "server started" -# create namespace -headscale namespaces create $NETWORK_NAME || true +# create users +headscale users create $NETWORK_NAME || true + +# +# headscale -u omnet preauthkeys create -o json # kill background process -pgrep headscale | xargs kill -9 +# pgrep headscale | xargs kill -9 -# start in foreground -headscale serve +# # start in foreground +# headscale serve +flask run -p 4000 --host=0.0.0.0 \ No newline at end of file diff --git a/packages/grid/vpn/tailscale.dockerfile b/packages/grid/vpn/tailscale.dockerfile index 8da87fe844a..70960b3da01 100644 --- a/packages/grid/vpn/tailscale.dockerfile +++ b/packages/grid/vpn/tailscale.dockerfile @@ -1,4 +1,4 @@ -FROM tailscale/tailscale:v1.22.0 +FROM tailscale/tailscale:v1.38.4 RUN --mount=type=cache,target=/var/cache/apk \ apk -U upgrade || true; \ diff --git a/packages/syft/src/syft/service/vpn/headscale_client.py b/packages/syft/src/syft/service/vpn/headscale_client.py index 4776517a9ae..7aa872067ad 100644 --- a/packages/syft/src/syft/service/vpn/headscale_client.py +++ b/packages/syft/src/syft/service/vpn/headscale_client.py @@ -20,7 +20,7 @@ class HeadscaleAuthToken(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 id: Optional[UID] - namespace: str + user: str key: str @@ -64,7 +64,8 @@ def generate_token( return SyftError(message=result.error) result = json.loads(command_result.report) + print("got result", result) return HeadscaleAuthToken( key=result["key"], - namespace=result["namespace"], + user=result["user"], ) diff --git a/packages/syft/src/syft/service/vpn/tailscale_client.py b/packages/syft/src/syft/service/vpn/tailscale_client.py index 7d027472fa0..d172d00956e 100644 --- a/packages/syft/src/syft/service/vpn/tailscale_client.py +++ b/packages/syft/src/syft/service/vpn/tailscale_client.py @@ -157,10 +157,14 @@ def connect( self, headscale_host: str, headscale_auth_token: str ) -> Union[SyftSuccess, SyftError]: CONNECT_TIMEOUT = 60 - + print(">>> connect to headscale host:", headscale_host) + # headscale_host = "http://192.168.4.65:9080/vpn" + headscale_host = "http://host.docker.internal:9080/vpn" + # headscale_host = "http://host.docker.internal:8080" + print(">>> change connect to headscale host:", headscale_host) command_args = { "args": [ - "-login-server", + "--login-server", f"{headscale_host}", "--reset", "--force-reauth", From 55bd3f04ebf1d2e7a4c11a8cf189ee54090be470 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Tue, 18 Jul 2023 16:23:44 +1000 Subject: [PATCH 2/4] Rebuild headscale container with updated python debian - Switched to working tailscale client --- packages/grid/vpn/headscale.dockerfile | 28 +++++++++++++++++++++++--- packages/grid/vpn/tailscale.dockerfile | 4 +++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/grid/vpn/headscale.dockerfile b/packages/grid/vpn/headscale.dockerfile index f5fd15f106f..aa038f20cb2 100644 --- a/packages/grid/vpn/headscale.dockerfile +++ b/packages/grid/vpn/headscale.dockerfile @@ -1,10 +1,19 @@ -FROM headscale/headscale:0.22.3 +ARG PYTHON_VERSION='3.11' +FROM python:3.11-slim as build + +# set UTC timezone +ENV TZ=Etc/UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN mkdir -p /root/.local + +RUN apt-get update && apt-get upgrade -y RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ DEBIAN_FRONTEND=noninteractive \ apt-get update && \ - apt-get install -yqq \ - python3 python3-pip curl procps && \ + apt-get install -y --no-install-recommends \ + curl procps && \ rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip @@ -13,6 +22,19 @@ ENV WAITFORIT_VERSION="v2.4.1" RUN curl -o /usr/local/bin/waitforit -sSL https://github.com/maxcnunes/waitforit/releases/download/$WAITFORIT_VERSION/waitforit-linux_amd64 && \ chmod +x /usr/local/bin/waitforit +ENV HEADSCALE_VERSION="0.22.3" +RUN --mount=type=cache,target=/root/.cache if [ $(uname -m) != "x86_64" ]; then \ + curl -o /bin/headscale -sSL https://github.com/juanfont/headscale/releases/download/v${HEADSCALE_VERSION}/headscale_${HEADSCALE_VERSION}_linux_arm64; \ + fi + +RUN --mount=type=cache,target=/root/.cache if [ $(uname -m) == "x86_64" ]; then \ + curl -o /bin/headscale -sSL https://github.com/juanfont/headscale/releases/download/v${HEADSCALE_VERSION}/headscale_${HEADSCALE_VERSION}_linux_amd64; \ + fi + +RUN chmod +x /bin/headscale + +RUN mkdir -p /var/run/headscale + WORKDIR /headscale COPY ./requirements.txt /headscale/requirements.txt RUN --mount=type=cache,target=/root/.cache \ diff --git a/packages/grid/vpn/tailscale.dockerfile b/packages/grid/vpn/tailscale.dockerfile index 70960b3da01..be75f7324a4 100644 --- a/packages/grid/vpn/tailscale.dockerfile +++ b/packages/grid/vpn/tailscale.dockerfile @@ -1,4 +1,6 @@ -FROM tailscale/tailscale:v1.38.4 +FROM tailscale/tailscale:v1.29.125 + +RUN apk update && apk upgrade --available RUN --mount=type=cache,target=/var/cache/apk \ apk -U upgrade || true; \ From 99ab041d0314bf712557b7d49312af2e198e5438 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Tue, 18 Jul 2023 16:40:33 +1000 Subject: [PATCH 3/4] Removed unused ports for now --- packages/grid/docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 375668f7ce7..2309749dfe8 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -24,7 +24,6 @@ services: - HTTPS_PORT=${HTTPS_PORT} ports: - "${HTTP_PORT}:81" - - "41641/udp" extra_hosts: - "host.docker.internal:host-gateway" @@ -212,9 +211,6 @@ services: - RELEASE=${RELEASE:-production} - NETWORK_NAME=omnet - STACK_API_KEY=$STACK_API_KEY - ports: - - "3478/udp" - - "8080:8080" # seaweedfs: # profiles: From 8e3a430c042fc9524a5cc0ade244bc518b90de28 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Tue, 18 Jul 2023 16:51:19 +1000 Subject: [PATCH 4/4] Remove test work around --- packages/syft/src/syft/service/vpn/tailscale_client.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/syft/src/syft/service/vpn/tailscale_client.py b/packages/syft/src/syft/service/vpn/tailscale_client.py index d172d00956e..133fbd9993e 100644 --- a/packages/syft/src/syft/service/vpn/tailscale_client.py +++ b/packages/syft/src/syft/service/vpn/tailscale_client.py @@ -157,11 +157,6 @@ def connect( self, headscale_host: str, headscale_auth_token: str ) -> Union[SyftSuccess, SyftError]: CONNECT_TIMEOUT = 60 - print(">>> connect to headscale host:", headscale_host) - # headscale_host = "http://192.168.4.65:9080/vpn" - headscale_host = "http://host.docker.internal:9080/vpn" - # headscale_host = "http://host.docker.internal:8080" - print(">>> change connect to headscale host:", headscale_host) command_args = { "args": [ "--login-server",