Merge pull request #100 from b2network/multinodes

Support multi nodes

Dockerfile

@@ -26,5 +26,7 @@ WORKDIR /
 # Copy over binaries from the build-env
 COPY --from=build-env /go/src/github.com/evmos/ethermint/build/ethermintd /usr/bin/ethermintd
 
+EXPOSE 26656 26657 9090 1317 8545 8546
+
 # Run ethermintd by default
 CMD ["ethermintd","start"]

Makefile

@@ -473,20 +473,20 @@ proto-check-breaking:
 localnet-build:
 	@$(MAKE) -C networks/local
 
+# Generate 4 nodes config
+localnet-config:
+	@bash ./init-multi-nodes.sh
+
 # Start a 4-node testnet locally
 localnet-start: localnet-stop
 ifeq ($(OS),Windows_NT)
-	mkdir localnet-setup &
 	@$(MAKE) localnet-build
-
-	IF not exist "build/node0/$(ETHERMINT_BINARY)/config/genesis.json" docker run --rm -v $(CURDIR)/build\ethermint\Z ethermintd/node "./ethermintd testnet --v 4 -o /ethermint --keyring-backend=test --ip-addresses ethermintdnode0,ethermintdnode1,ethermintdnode2,ethermintdnode3"
-	docker-compose up -d
+	IF not exist "build/node0/$(ETHERMINT_BINARY)/config/genesis.json" docker run --rm -v $(CURDIR)/build\ethermint\Z ethermintd/node "/usr/bin/ethermintd testnet init-files --v 4 -o /ethermint --keyring-backend=test --starting-ip-address 192.167.10.2"
+	docker-compose -f networks/local/ethermintnode/docker-compose.yml up -d
 else
-	mkdir -p localnet-setup
 	@$(MAKE) localnet-build
-
-	if ! [ -f localnet-setup/node0/$(ETHERMINT_BINARY)/config/genesis.json ]; then docker run --rm -v $(CURDIR)/localnet-setup:/ethermint:Z ethermintd/node "./ethermintd testnet --v 4 -o /ethermint --keyring-backend=test --ip-addresses ethermintdnode0,ethermintdnode1,ethermintdnode2,ethermintdnode3"; fi
-	docker-compose up -d
+	if ! [ -f build/node0/$(ETHERMINT_BINARY)/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/ethermint:Z ethermintd/node "/usr/bin/ethermintd testnet init-files --v 4 -o /ethermint --keyring-backend=test --starting-ip-address 192.167.10.2"; fi
+	docker-compose -f networks/local/ethermintnode/docker-compose.yml up -d
 endif
 
 # Stop testnet

docker-compose.yml

@@ -1,54 +1,75 @@
-version: "3"
+version: '3'
 
 services:
   node0:
+    container_name: b2-node0
     image: ${ETHERMINT_IMAGE}
     ports:
-      - "26657:26657"
-      - "8545:8545"
-      - "8546:8546"
-      - "8125:8125"
+      - "26656-26657:26656-26657"
     volumes:
-      - ./client-data/node0/ethermintd:/root/.ethermintd
+      - ./build/node0/ethermintd:/ethermint:Z
+    command:
+      - ethermintd
+      - start
+      - --home
+      - /ethermint
     networks:
-      - ethermint
+      localnet:
+        ipv4_address: 192.167.10.2
 
   node1:
+    container_name: b2-node1
     image: ${ETHERMINT_IMAGE}
     ports:
-      - "26658:26657"
-      - "8555:8545"
-      - "8556:8546"
-      - "8126:8125"
+      - "26659-26660:26656-26657"
     volumes:
-      - /root/.ethermintd:/root/.ethermintd
-      # - ./client-data/node1/ethermintd:/root/.ethermintd
+      - ./build/node1/ethermintd:/ethermint:Z
+    command:
+      - ethermintd
+      - start
+      - --home
+      - /ethermint
     networks:
-      - ethermint
+      localnet:
+        ipv4_address: 192.167.10.3
 
   node2:
+    container_name: b2-node2
     image: ${ETHERMINT_IMAGE}
     ports:
-      - "26659:26657"
-      - "8565:8545"
-      - "8566:8546"
-      - "8127:8125"
+      - "26661-26662:26656-26657"
     volumes:
-      - ./client-data/node2/ethermintd:/root/.ethermintd
+      - ./build/node2/ethermintd:/ethermint:Z
+    command:
+      - ethermintd
+      - start
+      - --home
+      - /ethermint
     networks:
-      - ethermint
+      localnet:
+        ipv4_address: 192.167.10.4
 
   node3:
+    container_name: b2-node3
     image: ${ETHERMINT_IMAGE}
     ports:
-      - "26660:26657"
-      - "8575:8545"
-      - "8576:8546"
-      - "8128:8125"
+      - "26663-26664:26656-26657"
     volumes:
-      - ./client-data/node3/ethermintd:/root/.ethermintd
+      - ./build/node3/ethermintd:/ethermint:Z
+    command:
+      - ethermintd
+      - start
+      - --home
+      - /ethermint
     networks:
-      - ethermint
+      localnet:
+        ipv4_address: 192.167.10.5
 
 networks:
-  ethermint:
+  localnet:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        -
+          subnet: 192.167.10.0/16

init-multi-nodes.sh

@@ -0,0 +1,144 @@
+#!/bin/bash
+HOME_DIR="$HOME/.ethermint"
+KEY="mykey"
+CHAINID="bsqhub_1113-1"
+MONIKER="localtestnet"
+KEYRING="test"
+KEYALGO="eth_secp256k1"
+LOGLEVEL="info"
+# trace evm
+TRACE="--trace"
+# TRACE=""
+NODES=(
+    "$HOME_DIR/node0/ethermintd/config"
+    "$HOME_DIR/node1/ethermintd/config"
+    "$HOME_DIR/node2/ethermintd/config"
+    "$HOME_DIR/node3/ethermintd/config"
+)
+NODE=$HOME_DIR/node
+
+KEYS=(
+    "node0"
+    "node1"
+    "node2"
+    "node3"
+)
+
+# validate dependencies are installed
+command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; }
+
+# remove existing daemon and client
+rm -rf ~/.ethermint*
+
+make install
+
+ethermintd config keyring-backend $KEYRING
+ethermintd config chain-id $CHAINID
+
+# if $KEY exists it should be deleted
+# ethermintd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
+
+# Generate 4 nodes config
+ethermintd testnet init-files --v 4 --chain-id $CHAINID --node-dir-prefix node --keyring-backend $KEYRING  --output-dir $HOME_DIR
+
+# Copy each node's config to the temp directory
+echo ">>> Copying each node's config to the temp directory"
+mkdir -p $NODE/config $NODE/keyring-test
+cp $HOME_DIR/node0/ethermintd/config/genesis.json $NODE/config/genesis.json
+for i in {0..3}; do
+    echo "Copying node$i's config to the temp directory"
+    cp $HOME_DIR/node$i/ethermintd/keyring-test/node$i.info $NODE/keyring-test/node$i.info
+done
+
+# Change parameter token denominations to bsq
+echo ">>> Change parameter token denominations to bsq"
+jq '.app_state["staking"]["params"]["bond_denom"]="bsq"' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+jq '.app_state["crisis"]["constant_fee"]["denom"]="bsq"' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="bsq"' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+jq '.app_state["mint"]["params"]["mint_denom"]="bsq"' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+
+# Set gas limit in genesis
+echo "Set gas limit in genesis"
+jq '.consensus_params["block"]["max_gas"]="20000000"' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+
+# Remove existing genesis accounts
+echo ">>> Remove existing genesis accounts"
+jq '.app_state["auth"]["accounts"]=[]' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+
+# Remove existing genesis balanaces
+echo "Remove existing genesis balanaces"
+jq '.app_state["bank"]["balances"]=[]' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+
+# Allocate genesis accounts (cosmos formatted addresses)
+echo ">>> Allocate genesis accounts"
+for i in {0..3}; do
+    echo "Allocate genesis accounts for ${KEYS[i]}"
+    ethermintd add-genesis-account ${KEYS[i]} 100000000000000000000000000bsq --keyring-backend $KEYRING --home $NODE
+done
+
+# Remove existing genesis gentxs
+echo "Remove existing genesis gentxs"
+jq '.app_state["genutil"]["gen_txs"]=[]' $NODE/config/genesis.json > $NODE/config/tmp_genesis.json && mv $NODE/config/tmp_genesis.json $NODE/config/genesis.json
+
+# Sign genesis transaction
+echo ">>> Sign genesis transaction"
+for i in {0..3}; do
+  echo "Sign genesis transaction for ${KEYS[i]}"
+  ethermintd gentx ${KEYS[i]} 1000000000000000000000bsq --moniker node$i  --keyring-backend $KEYRING --chain-id $CHAINID --home $NODE
+  mv $NODE/config/node_key.json $NODE/config/node$i_key.json
+  mv $NODE/config/priv_validator_key.json $NODE/config/priv_validator$i_key.json
+done
+
+# Collect genesis tx
+echo ">>> Collect genesis tx"
+ethermintd collect-gentxs --home $NODE
+
+# Run this to ensure everything worked and that the genesis file is setup correctly
+ethermintd validate-genesis --home $NODE
+
+# disable produce empty block and enable prometheus metrics
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $NODE/config/config.toml
+    sed -i '' 's/prometheus = false/prometheus = true/' $NODE/config/config.toml
+    sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time  = 1000000000000/g' $NODE/config/app.toml
+    sed -i '' 's/enabled = false/enabled = true/g' $NODE/config/app.toml
+else
+    sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $NODE/config/config.toml
+    sed -i 's/prometheus = false/prometheus = true/' $NODE/config/config.toml
+    sed -i 's/prometheus-retention-time  = "0"/prometheus-retention-time  = "1000000000000"/g' $NODE/config/app.toml
+    sed -i 's/enabled = false/enabled = true/g' $NODE/config/app.toml
+fi
+
+if [[ $1 == "pending" ]]; then
+    echo "pending mode is on, please wait for the first block committed."
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $NODE/config/config.toml
+        sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $NODE/config/config.toml
+    else
+        sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $NODE/config/config.toml
+        sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $NODE/config/config.toml
+    fi
+fi
+
+# Copy the config files to the nodes' directories
+echo ">>> Copy the config files to the nodes' directories"
+for i in {0..3}; do
+    echo "Copying node$i's config to the temp directory"
+    cp $NODE/config/genesis.json $HOME_DIR/node$i/ethermintd/config/genesis.json
+    cp $NODE/config/node$i_key.json $HOME_DIR/node$i/ethermintd/config/node_key.json
+    cp $NODE/config/priv_validator$i_key.json $HOME_DIR/node$i/ethermintd/config/priv_validator_key.json
+done

networks/local/ethermintnode/Dockerfile

@@ -15,17 +15,17 @@ COPY . .
 RUN make build-linux
 
 # Final image
-FROM golang:1.19 as final
+FROM golang:1.20 as final
 
 WORKDIR /
 
 RUN apt-get update
 
 # Copy over binaries from the build-env
-COPY --from=build-env /go/src/github.com/evmos/ethermint/build/ethermintd /
+COPY --from=build-env /go/src/github.com/evmos/ethermint/build/ethermintd /usr/bin/ethermintd
 COPY --from=build-env /go/src/github.com/evmos/ethermint/scripts/start-docker.sh /
 
-EXPOSE 26656 26657 1317 8545 8546
+EXPOSE 26656 26657 1317 9090 8545 8546
 
 # Run ethermintd by default, omit entrypoint to ease using container with ethermintd
 ENTRYPOINT ["/bin/bash", "-c"]

networks/local/ethermintnode/docker-compose.yml

@@ -0,0 +1,63 @@
+version: '3'
+
+services:
+  node0:
+    container_name: b2-node0
+    image: ethermintd/node 
+    ports:
+      - "26656-26657:26656-26657"
+    volumes:
+      - ../../../build/node0/ethermintd:/ethermint:Z
+    command:
+      - ethermintd start --home /ethermint
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.2
+
+  node1:
+    container_name: b2-node1
+    image: ethermintd/node 
+    ports:
+      - "26659-26660:26656-26657"
+    volumes:
+      - ../../../build/node1/ethermintd:/ethermint:Z
+    command:
+      - ethermintd start --home /ethermint
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.3
+
+  node2:
+    container_name: b2-node2
+    image: ethermintd/node 
+    ports:
+      - "26661-26662:26656-26657"
+    volumes:
+      - ../../../build/node2/ethermintd:/ethermint:Z
+    command:
+      - ethermintd start --home /ethermint
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.4
+
+  node3:
+    container_name: b2-node3
+    image: ethermintd/node 
+    ports:
+      - "26663-26664:26656-26657"
+    volumes:
+      - ../../../build/node3/ethermintd:/ethermint:Z
+    command:
+      - ethermintd start --home /ethermint
+    networks:
+      localnet:
+        ipv4_address: 192.167.10.5
+
+networks:
+  localnet:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        -
+          subnet: 192.167.10.0/16