BaseTools: ninja build system for faster builds

The current makefiles are executed sequentially. The VfrCompile
build also includes an antlr build that generates code over a long
period of time. Ninja allows to parallelise builds of different
utilities and build them 2.5 times faster.

Signed-off-by: Alexander Gryanko <[email protected]>

BaseTools/GNUmakefile

@@ -13,13 +13,16 @@ SOURCE_SUBDIRS := $(patsubst %,Source/%,$(sort $(LANGUAGES)))
 SUBDIRS := $(SOURCE_SUBDIRS) Tests
 CLEAN_SUBDIRS := $(patsubst %,%-clean,$(sort $(SUBDIRS)))
 
-.PHONY: subdirs $(SUBDIRS)
+.PHONY: subdirs $(SUBDIRS) ninja
 subdirs: $(SUBDIRS)
 $(SUBDIRS):
 	$(MAKE) -C $@
 
 Tests: $(SOURCE_SUBDIRS)
 
+ninja:
+	for dir in $(SUBDIRS); do $(MAKE) -C $$dir ninja; done
+
 .PHONY: $(CLEAN_SUBDIRS)
 $(CLEAN_SUBDIRS):
 	-$(MAKE) -C $(@:-clean=) clean

BaseTools/Source/C/GNUmakefile

@@ -41,7 +41,10 @@ endif
 
 export HOST_ARCH
 
-MAKEROOT = .
+MAKEROOT ?= $(CURDIR)
+export MAKEROOT
+
+NINJA := $(shell which ninja-build 2>/dev/null || which ninja 2>/dev/null)
 
 include Makefiles/header.makefile
 
@@ -85,6 +88,17 @@ $(patsubst %,%-clean,$(sort $(SUBDIRS))):
 $(VFRAUTOGEN): VfrCompile/VfrSyntax.g
 	$(MAKE) -C VfrCompile VfrLexer.h
 
+.PHONY: ninja
+ninja:
+	for dir in $(LIBRARIES); do $(MAKE) -C $$dir ninja_lib; done
+	for dir in $(APPLICATIONS); do $(MAKE) -C $$dir ninja_app; done
+	@echo "pool link_pool" > build.ninja
+	@echo "  depth = 10" >> build.ninja
+	@echo "" >> build.ninja
+	for dir in $(LIBRARIES); do echo "subninja $${dir}/build.ninja" >> build.ninja; done
+	for dir in $(APPLICATIONS); do echo "subninja $${dir}/build.ninja" >> build.ninja; done
+	$(NINJA)
+
 clean:  $(patsubst %,%-clean,$(sort $(SUBDIRS)))
 
 clean: localClean

BaseTools/Source/C/GenFfs/GNUmakefile

@@ -10,7 +10,8 @@ APPNAME = GenFfs
 
 OBJECTS = GenFfs.o
 
+LIBS = -lCommon
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
-LIBS = -lCommon
 

BaseTools/Source/C/LzmaCompress/GNUmakefile

@@ -22,6 +22,18 @@ OBJECTS = \
   $(SDK_C)/7zStream.o \
   $(SDK_C)/Bra86.o
 
+override NINJA_CSRCS := \
+  LzmaCompress.c \
+  $(SDK_C)/Alloc.c \
+  $(SDK_C)/LzFind.c \
+  $(SDK_C)/LzmaDec.c \
+  $(SDK_C)/LzmaEnc.c \
+  $(SDK_C)/7zFile.c \
+  $(SDK_C)/7zStream.c \
+  $(SDK_C)/Bra86.c
+
+override NINJA_CXXSRCS := 
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
 CFLAGS += -D_7ZIP_ST

BaseTools/Source/C/Makefiles/app.makefile

@@ -11,12 +11,26 @@ include $(MAKEROOT)/Makefiles/header.makefile
 
 APPLICATION = $(MAKEROOT)/bin/$(APPNAME)
 
-.PHONY:all
+NINJA_DEPS = 
+ifneq ($(LIBS),)
+  NINJA_DEPS +=  || 
+  NINJA_DEPS += $(foreach lib,$(LIBS),"$(MAKEROOT)/libs/lib$(subst -l,,$(lib)).a")
+endif
+
+.PHONY:all ninja
 all: $(MAKEROOT)/bin $(APPLICATION)
 
 $(APPLICATION): $(OBJECTS)
 	$(LINKER) -o $(APPLICATION) $(LDFLAGS) $(OBJECTS) -L$(MAKEROOT)/libs $(LIBS)
 
 $(OBJECTS): $(MAKEROOT)/Include/Common/BuildVersion.h
 
+ninja_app: ninja_build
+	@echo "" >> build.ninja
+	@echo "rule link_$(NINJA_ID)" >> build.ninja
+	@echo "  pool = link_pool" >> build.ninja
+	@echo "  command = $(LINKER) -o \044out $(LDFLAGS) \044in -L$(MAKEROOT)/libs $(LIBS)" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "build $(APPLICATION): link_$(NINJA_ID) $(NINJA_OBJECTS)$(NINJA_DEPS)" >> build.ninja
+
 include $(MAKEROOT)/Makefiles/footer.makefile

BaseTools/Source/C/Makefiles/footer.makefile

@@ -10,7 +10,12 @@ DEPFILES = $(OBJECTS:%.o=%.d)
 $(MAKEROOT)/libs-$(HOST_ARCH):
 	mkdir -p $(MAKEROOT)/libs-$(HOST_ARCH)
 
-.PHONY: install
+NINJA_CSRCS = $(wildcard *.c)
+NINJA_CXXSRCS = $(wildcard *.cpp)
+NINJA_COBJECTS := $(foreach file,$(NINJA_CSRCS),"build $(NINJA_DIR)/$(subst .c,,$(file)).o: cc_$(NINJA_ID) $(NINJA_DIR)/$(file)")
+NINJA_CXXOBJECTS := $(foreach file,$(NINJA_CXXSRCS),"build $(NINJA_DIR)/$(subst .c,,$(file)).o: cxx_$(NINJA_ID) $(NINJA_DIR)/$(file)")
+
+.PHONY: install ninja
 install: $(MAKEROOT)/libs-$(HOST_ARCH) $(LIBRARY)
 	cp $(LIBRARY) $(MAKEROOT)/libs-$(HOST_ARCH)
 
@@ -23,8 +28,25 @@ $(LIBRARY): $(OBJECTS)
 %.o : %.cpp
 	$(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< -o $@
 
+ninja_build:
+	@echo "" > build.ninja
+	@echo "rule cc_$(NINJA_ID)" >> build.ninja
+	@echo "  deps = $(CC)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(CC) -c $(CPPFLAGS) $(CFLAGS) \044in -o \044out" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "rule cxx_$(NINJA_ID)" >> build.ninja
+	@echo "  deps = $(CC)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(CC) -c $(CPPFLAGS) $(CXXFLAGS) \044in -o \044out" >> build.ninja
+	@echo "" >> build.ninja
+	@printf "%s\n" $(NINJA_COBJECTS) >> build.ninja
+	@printf "%s\n" $(NINJA_CXXOBJECTS) >> build.ninja
+	@echo "" >> build.ninja
+
 .PHONY: clean
 clean:
 	@rm -f $(OBJECTS) $(LIBRARY) $(DEPFILES)
+	@rm -f build.ninja
 
 -include $(DEPFILES)

BaseTools/Source/C/Makefiles/header.makefile

@@ -134,6 +134,19 @@ CXXFLAGS += $(BUILD_OPTFLAGS)
 # keep EXTRA_LDFLAGS last
 LDFLAGS += $(EXTRA_LDFLAGS)
 
+NINJA_DIR = 
+ifneq ($(APPNAME),)
+  NINJA_DIR = $(APPNAME)
+else ifneq ($(LIBNAME),)
+  NINJA_DIR = $(LIBNAME)
+else ifneq ($(APPLICATIONS),)
+# The root makefile cannot define APPNAME/LIBNAME. APPLICATIONS is a unique
+#   variable for the root makefile that is unlikely to be used in subfolders.
+  $(error "Neither APPNAME nor LIBNAME is defined. At least one of them must be defined")
+endif
+NINJA_ID := $(shell tr -dc A-Za-z0-9 </dev/random | head -c 5)
+NINJA_OBJECTS := $(foreach obj,$(OBJECTS),"$(NINJA_DIR)/$(obj)")
+
 .PHONY: all
 .PHONY: install
 .PHONY: clean

BaseTools/Source/C/Makefiles/lib.makefile

@@ -11,4 +11,12 @@ LIBRARY = $(MAKEROOT)/libs/lib$(LIBNAME).a
 
 all: $(MAKEROOT)/libs $(LIBRARY)
 
+ninja_lib: ninja_build
+	@echo "" >> build.ninja
+	@echo "rule link_$(NINJA_ID)" >> build.ninja
+	@echo "  pool = link_pool" >> build.ninja
+	@echo "  command = $(AR) crs \044out \044in" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "build $(LIBRARY): link_$(NINJA_ID) $(NINJA_OBJECTS)" >> build.ninja
+
 include $(MAKEROOT)/Makefiles/footer.makefile

BaseTools/Source/C/VfrCompile/GNUmakefile

@@ -11,7 +11,7 @@ APPNAME = VfrCompile
 
 LIBS = -lCommon
 
-TOOL_INCLUDE = -I Pccts/h
+TOOL_INCLUDE = -I $(MAKEROOT)/$(APPNAME)/Pccts/h
 
 #OBJECTS = VfrSyntax.o VfrServices.o DLGLexer.o EfiVfrParser.o ATokenBuffer.o DLexerBase.o AParser.o
 OBJECTS = AParser.o DLexerBase.o ATokenBuffer.o EfiVfrParser.o VfrLexer.o VfrSyntax.o \
@@ -38,7 +38,42 @@ include $(MAKEROOT)/Makefiles/header.makefile
 
 APPLICATION = $(MAKEROOT)/bin/$(APPNAME)
 
-.PHONY:all
+override NINJA_CSRCS :=
+override NINJA_CXXSRCS :=
+
+ANTLR_OBJECTS = \
+  $(NINJA_DIR)/Pccts/antlr/antlr.o \
+  $(NINJA_DIR)/Pccts/antlr/scan.o \
+  $(NINJA_DIR)/Pccts/antlr/err.o \
+  $(NINJA_DIR)/Pccts/antlr/bits.o \
+  $(NINJA_DIR)/Pccts/antlr/build.o \
+  $(NINJA_DIR)/Pccts/antlr/fset2.o \
+  $(NINJA_DIR)/Pccts/antlr/fset.o \
+  $(NINJA_DIR)/Pccts/antlr/gen.o \
+  $(NINJA_DIR)/Pccts/antlr/globals.o \
+  $(NINJA_DIR)/Pccts/antlr/hash.o \
+  $(NINJA_DIR)/Pccts/antlr/lex.o \
+  $(NINJA_DIR)/Pccts/antlr/main.o \
+  $(NINJA_DIR)/Pccts/antlr/misc.o \
+  $(NINJA_DIR)/Pccts/antlr/pred.o \
+  $(NINJA_DIR)/Pccts/antlr/egman.o \
+  $(NINJA_DIR)/Pccts/antlr/mrhoist.o \
+  $(NINJA_DIR)/Pccts/antlr/fcache.o
+
+DLG_OBJECTS = \
+  $(NINJA_DIR)/Pccts/dlg/dlg_p.o \
+  $(NINJA_DIR)/Pccts/dlg/dlg_a.o \
+  $(NINJA_DIR)/Pccts/dlg/main.o \
+  $(NINJA_DIR)/Pccts/dlg/err.o \
+  $(NINJA_DIR)/Pccts/dlg/support.o \
+  $(NINJA_DIR)/Pccts/dlg/output.o \
+  $(NINJA_DIR)/Pccts/dlg/relabel.o \
+  $(NINJA_DIR)/Pccts/dlg/automata.o
+
+ANTLR_BUILDS := $(foreach file,$(ANTLR_OBJECTS),"build $(file): cc_tools_$(NINJA_ID) $(subst .o,,$(file)).c")
+DLG_BUILDS := $(foreach file,$(DLG_OBJECTS),"build $(file): cc_tools_$(NINJA_ID) $(subst .o,,$(file)).c")
+
+.PHONY:all ninja
 all: $(MAKEROOT)/bin $(APPLICATION)
 
 $(APPLICATION): $(OBJECTS)
@@ -72,6 +107,60 @@ AParser.o: Pccts/h/AParser.cpp
 VfrSyntax.o: VfrSyntax.cpp
 	$(CXX) -c $(VFR_CPPFLAGS) $(INC) $(VFR_CXXFLAGS) $? -o $@
 
+
+# The source code licence for Pccts is not clearly defined. I'm not sure
+#   if editing its makefiles is allowed. The build is invoked without
+#   automatically generating a ninja file.
+ninja_antlr_tools:
+	@echo "rule link_tools_$(NINJA_ID)" > build.ninja
+	@echo "  pool = link_pool" >> build.ninja
+	@echo "  command = $(LINKER) -o \044out $(LDFLAGS) \044in" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "rule cc_tools_$(NINJA_ID)" >> build.ninja
+	@echo "  deps = $(CC)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(CC) -O -I$(NINJA_DIR)/Pccts/dlg -I$(NINJA_DIR)/Pccts/support/set -I$(NINJA_DIR)/Pccts/h -DUSER_ZZSYN -DZZLEXBUFSIZE=65536 -c \044in -o \044out" >> build.ninja
+	@echo "" >> build.ninja
+	@printf "%s\n" $(ANTLR_BUILDS) >> build.ninja
+	@printf "%s\n" $(DLG_BUILDS) >> build.ninja
+	@echo "build $(NINJA_DIR)/Pccts/support/set/set.o: cc_tools_$(NINJA_ID) $(NINJA_DIR)/Pccts/support/set/set.c" >> build.ninja
+	@echo "build $(NINJA_DIR)/Pccts/antlr/antlr: link_tools_$(NINJA_ID) $(NINJA_DIR)/Pccts/support/set/set.o $(ANTLR_OBJECTS)" >> build.ninja
+	@echo "build $(NINJA_DIR)/Pccts/dlg/dlg: link_tools_$(NINJA_ID) $(NINJA_DIR)/Pccts/support/set/set.o $(DLG_OBJECTS)" >> build.ninja
+
+ninja_app: ninja_antlr_tools
+	@echo "" >> build.ninja
+	@echo "rule antlr_$(NINJA_ID)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(NINJA_DIR)/Pccts/antlr/antlr -CC -e3 -ck 3 -k 2 -fl VfrParser.dlg -ft VfrTokens.h -o $(NINJA_DIR)/ \044in" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "rule dlg_$(NINJA_ID)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(NINJA_DIR)/Pccts/dlg/dlg -C2 -i -CC -cl VfrLexer -o $(NINJA_DIR)/ \044in" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "rule cxx_vfr_$(NINJA_ID)" >> build.ninja
+	@echo "  deps = $(CC)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(CXX) -c $(VFR_CPPFLAGS) $(INC) $(VFR_CXXFLAGS) \044in -o \044out" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "build $(NINJA_DIR)/AParser.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/Pccts/h/AParser.cpp" >> build.ninja
+	@echo "build $(NINJA_DIR)/DLexerBase.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/Pccts/h/DLexerBase.cpp" >> build.ninja
+	@echo "build $(NINJA_DIR)/ATokenBuffer.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/Pccts/h/ATokenBuffer.cpp" >> build.ninja
+	@echo "build $(NINJA_DIR)/EfiVfrParser.cpp $(NINJA_DIR)/VfrParser.dlg: antlr_$(NINJA_ID) $(NINJA_DIR)/VfrSyntax.g || $(NINJA_DIR)/AParser.o $(NINJA_DIR)/DLexerBase.o $(NINJA_DIR)/ATokenBuffer.o $(NINJA_DIR)/Pccts/antlr/antlr" >> build.ninja
+	@echo "build $(NINJA_DIR)/EfiVfrParser.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/EfiVfrParser.cpp || $(NINJA_DIR)/EfiVfrParser.cpp" >> build.ninja
+	@echo "build $(NINJA_DIR)/VfrSyntax.cpp $(NINJA_DIR)/VfrLexer.cpp $(NINJA_DIR)/VfrFormPkg.cpp $(NINJA_DIR)/VfrError.cpp $(NINJA_DIR)/VfrUtilityLib.cpp $(NINJA_DIR)/VfrCompiler.cpp: dlg_$(NINJA_ID) $(NINJA_DIR)/VfrParser.dlg || $(NINJA_DIR)/EfiVfrParser.o $(NINJA_DIR)/Pccts/dlg/dlg" >> build.ninja
+	@echo "build $(NINJA_DIR)/VfrLexer.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/VfrLexer.cpp || $(NINJA_DIR)/EfiVfrParser.o" >> build.ninja
+	@echo "build $(NINJA_DIR)/VfrSyntax.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/VfrSyntax.cpp || $(NINJA_DIR)/EfiVfrParser.o" >> build.ninja
+	@echo "build $(NINJA_DIR)/VfrFormPkg.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/VfrFormPkg.cpp || $(NINJA_DIR)/EfiVfrParser.o" >> build.ninja
+	@echo "build $(NINJA_DIR)/VfrError.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/VfrError.cpp || $(NINJA_DIR)/EfiVfrParser.o" >> build.ninja
+	@echo "build $(NINJA_DIR)/VfrUtilityLib.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/VfrUtilityLib.cpp || $(NINJA_DIR)/EfiVfrParser.o" >> build.ninja 
+	@echo "build $(NINJA_DIR)/VfrCompiler.o: cxx_vfr_$(NINJA_ID) $(NINJA_DIR)/VfrCompiler.cpp || $(NINJA_DIR)/EfiVfrParser.o" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "rule link_$(NINJA_ID)" >> build.ninja
+	@echo "  pool = link_pool" >> build.ninja
+	@echo "  command = $(LINKER) -o \044out $(LDFLAGS) \044in -L$(MAKEROOT)/libs $(LIBS)" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "build $(APPLICATION): link_$(NINJA_ID) $(NINJA_OBJECTS)" >> build.ninja
+
 clean: localClean
 
 localClean:

BaseTools/Source/Python/GNUmakefile

@@ -5,7 +5,9 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 
-all:
+all: ninja
+
+ninja:
 
 clean:
 	find . -name '*.pyc' -exec rm '{}' ';'

BaseTools/Tests/GNUmakefile

@@ -5,11 +5,13 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 
-all: test
+all: test ninja
 
 test:
 	@if command -v "${PYTHON_COMMAND}" >/dev/null 2>&1; then ${PYTHON_COMMAND} RunTests.py; else python RunTests.py; fi
 
+ninja: test
+
 clean:
 	find . -name '*.pyc' -exec rm '{}' ';'