-
Notifications
You must be signed in to change notification settings - Fork 180
/
Makefile
330 lines (269 loc) · 11.5 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
#
# Usage: `make help`
#
SHELL := /bin/bash
DEPLOY_DIR = ./deploy/dev/local
SCRIPTS_DIR = ./scripts
BUILD_DIR = ./cmd
BUILD_SRC = $(BUILD_DIR)/aisnode/main.go
ifeq ($(shell go env GOOS),linux)
ifeq ($(strip $(GORACE)),)
CGO_DISABLE = CGO_ENABLED=0
endif
endif
AISTORE_PATH = $(shell git rev-parse --show-toplevel)
CLI_VERSION := $(shell ais version 2>/dev/null)
# Do not print enter/leave directory when doing 'make -C DIR <target>'
MAKEFLAGS += --no-print-directory
# Uncomment to cross-compile aisnode and cli, respectively:
# CROSS_COMPILE = docker run --rm -v $(AISTORE_PATH):/go/src/n -w /go/src/n golang:1.23
# CROSS_COMPILE_CLI = docker run -e $(CGO_DISABLE) --rm -v $(AISTORE_PATH)/cmd/cli:/go/src/n -w /go/src/n golang:1.23
# Build version, flags, and tags
VERSION = $(shell git rev-parse --short HEAD)
BUILD = $(shell date +%FT%T%z)
BUILD_FLAGS += $(if $(strip $(GORACE)),-race,)
BUILD_DEST = $(GOPATH)/bin
ifdef GOBIN
BUILD_DEST = $(GOBIN)
endif
ifdef TAGS
BUILD_TAGS = $(AIS_BACKEND_PROVIDERS) $(TAGS)
else
BUILD_TAGS = $(AIS_BACKEND_PROVIDERS)
endif
# Race detection
ifneq ($(strip $(GORACE)),)
WRD = GORACE=$(GORACE)
ifneq ($(findstring log_path,$(GORACE)),log_path)
@echo
@echo "Expecting GORACE='log_path=...', run 'make help' for usage examples"
@exit 1
endif
endif
# Profiling
# Example usage: MEM_PROFILE=/tmp/mem make kill clean deploy <<< $'5\n5\n4\ny\ny\nn\n'
# Note that MEM_PROFILE (memprofile) option requires graceful shutdown (see `kill:`)
ifdef MEM_PROFILE
export AIS_NODE_FLAGS += -memprofile=$(MEM_PROFILE)
BUILD_SRC = $(BUILD_DIR)/aisnodeprofile/main.go
endif
ifdef CPU_PROFILE
export AIS_NODE_FLAGS += -cpuprofile=$(CPU_PROFILE)
BUILD_SRC = $(BUILD_DIR)/aisnodeprofile/main.go
endif
# Intra-cluster networking: two alternative ways to build AIS `transport` package:
# 1) using Go net/http, or
# 2) with a 3rd party github.com/valyala/fasthttp aka "fasthttp"
#
# The second option is the current default.
# To build with net/http, use `nethttp` build tag, for instance:
# TAGS=nethttp make deploy <<< $'5\n5\n4\ny\ny\nn\n'
ifeq ($(MODE),debug)
# Debug mode
GCFLAGS = -gcflags="all=-N -l"
LDFLAGS = -ldflags "-X 'main.build=$(VERSION)' -X 'main.buildtime=$(BUILD)'"
BUILD_TAGS += debug
else
# Production mode
GCFLAGS =
LDFLAGS = -ldflags "-w -s -X 'main.build=$(VERSION)' -X 'main.buildtime=$(BUILD)'"
endif
ifdef AIS_DEBUG
BUILD_TAGS += debug
endif
# mono: utilize go:linkname for monotonic time source
# !mono: use standard Go runtime
BUILD_TAGS += mono
# Colors
cyan = $(shell { tput setaf 6 || tput AF 6; } 2>/dev/null)
term-reset = $(shell { tput sgr0 || tput me; } 2>/dev/null)
$(call make-lazy,cyan)
$(call make-lazy,term-reset)
.PHONY: all node cli cli-autocompletions authn aisloader xmeta
all: node cli authn aisloader ## Build all main binaries
node: ## Build 'aisnode' binary
@echo "Building aisnode $(VERSION) [build tags:$(BUILD_TAGS)]"
ifdef WRD
@echo "Deploying with race detector, writing reports to $(subst log_path=,,$(GORACE)).<pid>"
endif
ifdef CROSS_COMPILE
@$(CROSS_COMPILE) go build -o ./aisnode $(BUILD_FLAGS) -tags="$(BUILD_TAGS)" $(GCFLAGS) $(LDFLAGS) $(BUILD_SRC)
@mv ./aisnode $(BUILD_DEST)/.
else
@$(WRD) go build -o $(BUILD_DEST)/aisnode $(BUILD_FLAGS) -tags="$(BUILD_TAGS)" $(GCFLAGS) $(LDFLAGS) $(BUILD_SRC)
endif
@echo "done."
cli: ## Build CLI binary. NOTE: 1) a separate go.mod, 2) static linkage with cgo disabled
@echo "Building ais (CLI) [build tags:$(BUILD_TAGS)]"
ifdef CROSS_COMPILE_CLI
cd $(BUILD_DIR)/cli && \
$(CROSS_COMPILE_CLI) go build -o ./ais -tags="$(BUILD_TAGS)" $(BUILD_FLAGS) $(LDFLAGS) *.go && mv ./ais $(BUILD_DEST)/.
else
@cd $(BUILD_DIR)/cli && $(CGO_DISABLE) go build -o $(BUILD_DEST)/ais -tags="$(BUILD_TAGS)" $(BUILD_FLAGS) $(LDFLAGS) *.go
endif
@echo "*** To enable autocompletions in your current shell, run:"
ifeq ($(AISTORE_PATH),$(PWD))
@echo "*** 'source cmd/cli/autocomplete/bash' or 'source cmd/cli/autocomplete/zsh'"
else
@echo "*** source $(AISTORE_PATH)/cmd/cli/autocomplete/[bash|zsh]"
endif
cli-autocompletions: ## Add CLI autocompletions
@echo "Adding CLI autocomplete..."
@./$(BUILD_DIR)/cli/autocomplete/install.sh
authn: build-authn ## Build AuthN
aisloader: build-aisloader ## Build aisloader
xmeta: build-xmeta ## Build xmeta
aisinit: build-aisinit ## Build aisinit
build-%:
@echo -n "Building $*... "
ifdef CROSS_COMPILE
@$(CROSS_COMPILE) go build -o ./$* $(BUILD_FLAGS) $(LDFLAGS) $(BUILD_DIR)/$*/*.go
@mv ./$* $(BUILD_DEST)/.
else
@go build -o $(BUILD_DEST)/$* $(BUILD_FLAGS) $(LDFLAGS) $(BUILD_DIR)/$*/*.go
endif
@echo "done."
#
# local deployment (intended for developers)
#
.PHONY: deploy run restart
## Build 'aisnode' and parse command-line to generate configurations
## and deploy the specified numbers of local AIS (proxy and target) daemons
deploy:
@"$(DEPLOY_DIR)/deploy.sh" $(RUN_ARGS)
## Same as `deploy` except for (not) generating daemon configurations.
## Use `make run` to restart cluster and utilize existing daemon configs.
run:
@"$(DEPLOY_DIR)/deploy.sh" $(RUN_ARGS) --dont-generate-configs
restart: kill run
#
# cleanup local deployment (cached objects, logs, and executables)
#
.PHONY: kill clean
kill: ## Kill all locally deployed clusters (all targets and all proxies)
ifndef CLI_VERSION
@echo "Warning: missing CLI (ais) executable for proper graceful shutdown"
else
@ais cluster shutdown -y 2>/dev/null || true
endif
@"$(DEPLOY_DIR)/kill.sh"
clean: ## Remove all AIS related files and binaries
@echo -n "Cleaning... "
@"$(DEPLOY_DIR)/clean.sh"
@echo "done."
#
# modules
#
.PHONY: mod-all mod-clean mod-tidy
mod-all: mod-clean mod-tidy
@echo "CLI ..." && cd cmd/cli && $(MAKE) mod-tidy
@echo "ishard ..." && cd cmd/ishard && $(MAKE) mod-tidy
# cleanup go-mod cache
mod-clean:
go clean --modcache
# in particular, remove unused
mod-tidy:
go mod tidy
# for local docker deployment
.PHONY: deploy-docker stop-docker
deploy-docker: ## Deploy AIS cluster inside the dockers
# pass -d=3 because need 3 mountpaths for some tests
@cd ./deploy/dev/docker && ./deploy_docker.sh -d=3
stop-docker: ## Stop deployed AIS docker cluster
ifeq ($(FLAGS),)
$(warning missing environment variable: FLAGS="stop docker flags")
endif
@./deploy/dev/docker/stop_docker.sh $(FLAGS)
#
# tests
#
.PHONY: test-bench test-aisloader test-envcheck test-short test-long test-run test-docker test
# Target for benchmark tests
test-bench: ## Run benchmarking tests
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" test-bench
test-envcheck:
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" test-env
test-short: test-envcheck ## Run short tests
@RE="$(RE)" BUCKET="$(BUCKET)" TESTS_DIR="$(TESTS_DIR)" AIS_ENDPOINT="$(AIS_ENDPOINT)" $(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" test-short
@cd $(BUILD_DIR)/cli && go test -v -tags=debug ./...
test-tracing-unit:
@cd tracing && go test -v -tags=oteltracing ./...
test-assorted: test-envcheck # Run specific tests
@RE="ETLBucket|ETLConnectionError|ETLInitCode" BUCKET="$(BUCKET)" TESTS_DIR="$(TESTS_DIR)" AIS_ENDPOINT="$(AIS_ENDPOINT)" $(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" test-long
test-long: test-envcheck ## Run all integration tests
@RE="$(RE)" BUCKET="$(BUCKET)" TESTS_DIR="$(TESTS_DIR)" AIS_ENDPOINT="$(AIS_ENDPOINT)" $(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" test-long
@cd $(BUILD_DIR)/cli && go test -v -tags=debug ./...
test-aisloader:
@./bench/tools/aisloader/test/ci-test.sh $(FLAGS)
test-run: test-envcheck # runs tests matching a specific regex
ifeq ($(RE),)
$(error missing environment variable: RE="testnameregex")
endif
@RE="$(RE)" BUCKET="$(BUCKET)" TESTS_DIR="$(TESTS_DIR)" AIS_ENDPOINT="$(AIS_ENDPOINT)" $(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" test-run
test-docker: ## Run tests inside docker
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" test-docker
ci: spell-check fmt-check lint test-short ## Run CI related checkers and linters (requires BUCKET variable to be set)
# Target for linters
.PHONY: lint-update lint install-python-deps fmt-check fmt-fix spell-check spell-fix cyclo msgp-update
## Removes the previous version of `golangci-lint` and installs the latest (compare with lint-update-ci below)
lint-update:
@rm -f $(GOPATH)/bin/golangci-lint
@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin latest
## Install specific `golangci-lint` version (hardcoded)
## See also: .github/workflows/lint.yml
lint-update-ci:
@rm -f $(GOPATH)/bin/golangci-lint
@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.62.2
lint:
@([[ -x "$(command -v golangci-lint)" ]] && echo "Cannot find golangci-lint, run 'make lint-update' to install" && exit 1) || true
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" lint
@$(MAKE) -C $(BUILD_DIR)/cli lint
install-python-deps:
@pip3 install -r ./python/aistore/common_requirements
fmt-check: install-python-deps ## Check code formatting
@pip3 install --upgrade black[jupyter] -q
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" fmt
fmt-fix: ## Fix code formatting
@pip3 install --upgrade black[jupyter] -q
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" fmt --fix
spell-check: ## Run spell checker on the project
@GOOS="" go install github.com/client9/misspell/cmd/misspell@latest
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" spell
spell-fix: ## Fix spell checking issues
@GOOS="" go install github.com/client9/misspell/cmd/misspell@latest
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" spell --fix
msgp-update: ## MessagePack generator
@GOOS="" go install github.com/tinylib/msgp@latest
# Misc Targets
.PHONY: cpuprof flamegraph dev-init
# run benchmarks 10 times to generate cpu.prof
cpuprof:
@go test -v -run=XXX -bench=. -count 10 -cpuprofile=/tmp/cpu.prof
flamegraph: cpuprof
@go tool pprof -http ":6060" /tmp/cpu.prof
# To help with working with a non-github remote
# It replaces existing github.com AIStore remote with the one specified by REMOTE
dev-init:
@$(SHELL) "$(SCRIPTS_DIR)/bootstrap.sh" dev-init
.PHONY: help
help:
@echo "Usage:"
@echo " [VAR=foo VAR2=bar...] make [target...]"
@echo ""
@echo "Useful commands:"
@grep -Eh '^[a-zA-Z._-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " $(cyan)%-30s$(term-reset) %s\n", $$1, $$2}'
@echo ""
@echo "Examples:"
@printf " $(cyan)%s$(term-reset)\n %s\n\n" \
"make deploy" "Deploy cluster locally" \
"make kill clean" "Stop locally deployed cluster and cleanup all cluster-related data and bucket metadata (but not cluster map)" \
"make kill deploy <<< $$'7\n2\n4\ny\ny\n'" "Shutdown and then (non-interactively) generate local configs and deploy a cluster consisting of 7 targets (4 mountpaths each) and 2 proxies; build 'aisnode' executable with AWS and GCP backends" \
"TAGS=\"aws gcp\" make kill deploy <<< $$'7\n2\n'" "Same as above (see docs/getting_started.md for many more examples)" \
"GORACE='log_path=/tmp/race' make deploy" "Deploy cluster with race detector, write reports to /tmp/race.<PID>" \
"MODE=debug make deploy" "Deploy cluster with 'aisnode' (AIS target and proxy) executable built with debug symbols and debug asserts enabled" \
"BUCKET=tmp make test-short" "Run all short tests" \
"BUCKET=<existing-cloud-bucket> make test-long" "Run all tests" \
"BUCKET=tmp make ci" "Run style, lint, and spell checks, as well as all short tests" \
"MEM_PROFILE=/tmp/mem make deploy" "Deploy cluster with memory profiling enabled, write reports to /tmp/mem.<PID> (and make sure to stop gracefully)" \
"CPU_PROFILE=/tmp/cpu make deploy" "Build and deploy cluster instrumented for CPU profiling, write reports to /tmp/cpu.<PID>" \
"TAGS=nethttp make deploy" "Build 'transport' package with net/http (see transport/README.md) and deploy cluster locally" \