|
- #!/usr/bin/env bash
- set -e
-
- BUILD_TYPE=Release
- MGE_WITH_CUDA=OFF
- MGE_INFERENCE_ONLY=ON
- MGE_WINDOWS_BUILD_ARCH=x64
- MGE_WINDOWS_BUILD_MARCH=m64
- MGE_ARCH=x86_64
- REMOVE_OLD_BUILD=false
- NINJA_VERBOSE=OFF
- BUILD_DEVELOP=ON
- NINJA_DRY_RUN=OFF
- SPECIFIED_TARGET="install/strip"
- READLINK=readlink
-
- OS=$(uname -s)
- if [[ $OS =~ "NT" ]]; then
- echo "Windows no need strip, caused by pdb file always split with exe"
- SPECIFIED_TARGET="install"
- fi
-
- if [ $OS = "Darwin" ];then
- READLINK=greadlink
- fi
-
- SRC_DIR=$($READLINK -f "`dirname $0`/../../")
- source ${SRC_DIR}/scripts/cmake-build/utils/utils.sh
- if [[ $OS =~ "NT" ]]; then
- source ${SRC_DIR}/scripts/whl/windows/config.sh
- fi
- config_ninja_default_max_jobs
-
- echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
-
- function usage() {
- echo "$0 args1 args2 .."
- echo "available args detail:"
- echo "-d : Build with Debug mode, default Release mode"
- echo "-c : Build with CUDA, default without CUDA"
- echo "-t : Build with training mode, default inference only"
- echo "-m : Build with m32 mode(only for windows build), default m64"
- echo "-r : remove old build dir before make, default off"
- echo "-v : ninja with verbose and explain, default off"
- echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel"
- echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
- echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2"
- echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)"
- echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS"
- echo "-h : show usage"
- echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:"
- echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d"
- exit -1
- }
-
- while getopts "lnsrhdctmve:j:" arg
- do
- case $arg in
- j)
- NINJA_MAX_JOBS=$OPTARG
- echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}"
- ;;
- l)
- echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS"
- show_cmakelist_options
- exit 0
- ;;
- d)
- echo "Build with Debug mode"
- BUILD_TYPE=Debug
- ;;
- c)
- echo "Build with CUDA"
- MGE_WITH_CUDA=ON
- ;;
- t)
- echo "Build with training mode"
- MGE_INFERENCE_ONLY=OFF
- ;;
- h)
- echo "show usage"
- usage
- ;;
- r)
- echo "config REMOVE_OLD_BUILD=true"
- REMOVE_OLD_BUILD=true
- ;;
- s)
- echo "config BUILD_DEVELOP=OFF"
- BUILD_DEVELOP=OFF
- ;;
- v)
- echo "config NINJA_VERBOSE=ON"
- NINJA_VERBOSE=ON
- ;;
- n)
- echo "config NINJA_DRY_RUN=ON"
- NINJA_DRY_RUN=ON
- ;;
- m)
- echo "build for m32(only valid use for windows)"
- MGE_WINDOWS_BUILD_ARCH=x86
- MGE_WINDOWS_BUILD_MARCH=m32
- MGE_ARCH=i386
- ;;
- e)
- SPECIFIED_TARGET=$OPTARG
- ;;
- ?)
- echo "unkonw argument"
- usage
- ;;
- esac
- done
- echo "------------------------------------"
- echo "build config summary:"
- echo "BUILD_TYPE: $BUILD_TYPE"
- echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
- echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
- echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}"
- echo "NINJA_MAX_JOBS: ${NINJA_MAX_JOBS}"
- echo "------------------------------------"
-
- if [ $OS = "Darwin" ];then
- if [ $MGE_WITH_CUDA = "ON" ];then
- echo "MACOS DO NOT SUPPORT TensorRT, ABORT NOW!!"
- exit -1
- fi
- elif [[ $OS =~ "NT" ]]; then
- echo "BUILD in NT ..."
- fi
-
- if [ ${MGE_INFERENCE_ONLY} = "ON" ]; then
- echo "config BUILD_DEVELOP=OFF when MGE_INFERENCE_ONLY=ON"
- BUILD_DEVELOP=OFF
- fi
-
- function cmake_build() {
- BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build
- INSTALL_DIR=$BUILD_DIR/../install
- MGE_WITH_CUDA=$1
- MGE_INFERENCE_ONLY=$2
- BUILD_TYPE=$3
- echo "build dir: $BUILD_DIR"
- echo "install dir: $INSTALL_DIR"
- echo "build type: $BUILD_TYPE"
- echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
- echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
- try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
-
- echo "create build dir"
- mkdir -p $BUILD_DIR
- mkdir -p $INSTALL_DIR
- cd_real_build_dir $BUILD_DIR
- # fork a new bash to handle EXTRA_CMAKE_ARGS env with space
- bash -c "cmake -G Ninja \
- -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
- -DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \
- -DMGE_WITH_CUDA=$MGE_WITH_CUDA \
- -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
- ${EXTRA_CMAKE_ARGS} \
- ${SRC_DIR} "
-
- config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS}
- bash -c "${NINJA_CMD}"
- }
-
- function windows_env_err() {
- echo "check windows env failed!!"
- echo "please install env refs for: scripts/cmake-build/BUILD_README.md"
- echo "also refs for: scripts/whl/windows/env_prepare.sh"
- exit -1
- }
-
- function prepare_env_for_windows_build() {
- echo "check Visual Studio install path env..."
- # check VS_INSTALL_PATH is valid or not
- if [ ! -d ${VS_INSTALL_PATH}/Licenses ];then
- echo "can not find ${VS_INSTALL_PATH}/Licenses, pls check VS_INSTALL_PATH env"
- echo "pls install VisualStudio by scripts/whl/windows/env_prepare.sh"
-
- windows_env_err
- else
- echo "use ${VS_INSTALL_PATH}"
- fi
-
- # only use cmake/Ninja install from Visual Studio, if not, may build failed
- # some user env may install cmake/clang-cl/Ninja at windows-git-bash env, so we put Visual Studio
- # path at the head of PATH, and check the valid
- echo "check cmake install..."
- export PATH=${VS_INSTALL_PATH}/Common7/IDE/CommonExtensions/Microsoft/CMake/CMake/bin/:$PATH
- which cmake
- cmake_loc=`which cmake`
- if [[ $cmake_loc =~ ${MEGENGINE_DEV_TOOLS_PREFIX_DIR} ]]; then
- echo "use cmake: $cmake_loc"
- else
- echo "cmake Invalid: ..."
- windows_env_err
- fi
-
- echo "check clang-cl install..."
- # llvm install by Visual Studio have some issue, eg, link crash on large project, so we
- # use official LLVM download from https://releases.llvm.org/download.html
- # check LLVM_MEGENGINE_DEV_DIR is valid or not
- if [ ! -f ${LLVM_MEGENGINE_DEV_DIR}/bin/clang-cl.exe ];then
- echo "can not find ${LLVM_MEGENGINE_DEV_DIR}/bin/clang-cl.exe, pls check LLVM_MEGENGINE_DEV_DIR env"
- echo "pls install LLVM by scripts/whl/windows/env_prepare.sh"
- windows_env_err
- else
- echo "use ${LLVM_MEGENGINE_DEV_DIR}"
- fi
- echo ${LLVM_MEGENGINE_DEV_DIR}
- export PATH=${LLVM_MEGENGINE_DEV_DIR}/bin/:$PATH
- clang_loc=`which clang-cl`
- if [[ $clang_loc =~ ${VS_INSTALL_PATH} ]]; then
- echo "clang-cl Invalid: we do not support use LLVM installed by Visual Studio"
- windows_env_err
- else
- echo "clang-cl valid ..."
- fi
- if [[ $clang_loc =~ ${LLVM_MEGENGINE_DEV_DIR} ]]; then
- echo "use clang-cl : $clang_loc"
- else
- echo "clang-cl Invalid: ..."
- windows_env_err
- fi
-
- echo "check Ninja install..."
- export PATH=${VS_INSTALL_PATH}/Common7/IDE/CommonExtensions/Microsoft/CMake/Ninja/:$PATH
- which Ninja
- ninja_loc=`which Ninja`
- if [[ $ninja_loc =~ ${MEGENGINE_DEV_TOOLS_PREFIX_DIR} ]]; then
- echo "use Ninja: $ninja_loc"
- else
- echo "Ninja Invalid: ..."
- windows_env_err
- fi
-
- echo "put vcvarsall.bat path to PATH env.."
- # check vcvarsall.bat is valid or not
- if [ ! -f ${VS_INSTALL_PATH}/VC/Auxiliary/Build/vcvarsall.bat ];then
- echo "can not find ${VS_INSTALL_PATH}/VC/Auxiliary/Build/vcvarsall.bat, pls check VS_INSTALL_PATH env"
- echo "pls install VisualStudio by scripts/whl/windows/env_prepare.sh"
- windows_env_err
- else
- echo "use ${VS_INSTALL_PATH}"
- fi
- export PATH=$VS_INSTALL_PATH/VC/Auxiliary/Build:$PATH
-
- if [ $MGE_WITH_CUDA = "ON" ];then
- echo "config cuda/cudnn/TensorRT env..."
- if [[ -z ${CUDA_ROOT_DIR} ]]; then
- echo "CUDA_ROOT_DIR is not set, use default: ${CUDA_DFT_ROOT}"
- export CUDA_ROOT_DIR=${CUDA_DFT_ROOT}
- fi
- # check CUDA_ROOT_DIR is valid or not
- if [ ! -f "${CUDA_ROOT_DIR}/bin/nvcc.exe" ];then
- echo "can not find ${CUDA_ROOT_DIR}/bin/nvcc.exe, pls check env"
- windows_env_err
- else
- echo "use CUDA_ROOT_DIR: ${CUDA_ROOT_DIR}"
- # put cuda/bin to PATH env
- export PATH=${CUDA_ROOT_DIR}/bin/:$PATH
- fi
-
- if [[ -z ${CUDNN_ROOT_DIR} ]]; then
- echo "CUDNN_ROOT_DIR is not set, use default: ${CUDNN_DFT_ROOT}"
- export CUDNN_ROOT_DIR=${CUDNN_DFT_ROOT}
- fi
- # check CUDNN_ROOT_DIR is valid or not
- if [ ! -f "${CUDNN_ROOT_DIR}/include/cudnn.h" ];then
- echo "can not find ${CUDNN_ROOT_DIR}/include/cudnn.h, pls check env"
- windows_env_err
- else
- echo "use CUDNN_ROOT_DIR: ${CUDNN_ROOT_DIR}"
- fi
-
- if [[ -z ${TRT_ROOT_DIR} ]]; then
- echo "TRT_ROOT_DIR is not set, use default: ${TRT_DFT_ROOT}"
- export TRT_ROOT_DIR=${TRT_DFT_ROOT}
- fi
- # check TRT_ROOT_DIR is valid or not
- if [ ! -f "${TRT_ROOT_DIR}/include/NvInfer.h" ];then
- echo "can not find ${TRT_ROOT_DIR}/include/NvInfer.h, pls check env"
- windows_env_err
- else
- echo "use TRT_ROOT_DIR: ${TRT_ROOT_DIR}"
- fi
- fi
-
- # python version will be config by whl build script or ci script, we need
- # a DFT version for build success when we just call host_build.sh
- if [[ -z ${ALREADY_CONFIG_PYTHON_VER} ]]
- then
- echo "config a default python3"
- if [ ! -f "${DFT_PYTHON_BIN}/python3.exe" ]; then
- echo "ERR: can not find ${DFT_PYTHON_BIN}/python3.exe , Invalid env"
- windows_env_err
- else
- echo "put python3 to env..."
- export PATH=${DFT_PYTHON_BIN}:$PATH
- which python3
- fi
- fi
-
- # check swig is valid or not
- if [ ! -f ${SWIG_INSTALL_DIR}/swig.exe ];then
- echo "can not find ${SWIG_INSTALL_DIR}/swig.exe, pls check SWIG_INSTALL_DIR env"
- echo "pls install swig by scripts/whl/windows/env_prepare.sh"
- windows_env_err
- else
- echo "use swig: ${SWIG_INSTALL_DIR}"
- export PATH=${SWIG_INSTALL_DIR}:$PATH
- fi
- }
-
- function cmake_build_windows() {
- # windows do not support long path, so we cache the BUILD_DIR ASAP
- prepare_env_for_windows_build
- BUILD_DIR=$SRC_DIR/build_dir/host/build
- # ninja have compat issue with bash env, which fork from windows-git
- # which will map C: to /c/c/ dir, which will lead to install file to /c/c/..
- # as a solution, we map INSTALL_DIR to INSTALL_DIR_WIN (/c --> C:)
- INSTALL_DIR=${BUILD_DIR}/../install
-
- INSTALL_DIR_PREFIX=${INSTALL_DIR:0:2}
- if [ ${INSTALL_DIR_PREFIX} = "/c" ];then
- echo "INSTALL_DIR_PREFIX is ${INSTALL_DIR_PREFIX}, map to C:"
- INSTALL_DIR_WIN="C:${INSTALL_DIR:2}"
- else
- INSTALL_DIR_WIN=${INSTALL_DIR}
- fi
- MGE_WITH_CUDA=$1
- MGE_INFERENCE_ONLY=$2
- BUILD_TYPE=$3
- echo "build dir: $BUILD_DIR"
- echo "install dir: $INSTALL_DIR"
- echo "install dir for ninja: $INSTALL_DIR_WIN"
- echo "build type: $BUILD_TYPE"
- echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
- echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
- try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
-
- echo "create build dir"
- mkdir -p $BUILD_DIR
- cd_real_build_dir $BUILD_DIR
- echo "now try build windows native with cmake/clang-ci/Ninja/Visual Studio ....."
- export CFLAGS=-$MGE_WINDOWS_BUILD_MARCH
- export CXXFLAGS=-$MGE_WINDOWS_BUILD_MARCH
- cmd.exe /C " \
- vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH -vcvars_ver=${CVARS_VER_NEED} && cmake -G "Ninja" \
- -DMGE_ARCH=$MGE_ARCH \
- -DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \
- -DMGE_WITH_CUDA=$MGE_WITH_CUDA \
- -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
- -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_DIR_WIN \
- -DCMAKE_C_COMPILER=clang-cl.exe \
- -DCMAKE_CXX_COMPILER=clang-cl.exe \
- -DCMAKE_MAKE_PROGRAM=ninja.exe \
- ${EXTRA_CMAKE_ARGS} ../../.. "
-
- config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS}
- cmd.exe /C " vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH -vcvars_ver=${CVARS_VER_NEED} && ${NINJA_CMD} "
- }
-
- if [[ $OS =~ "NT" ]]; then
- if [ ${MGE_ARCH} = "i386" ] && [ ${MGE_INFERENCE_ONLY} = "OFF" ]; then
- echo "ERR: training mode(-t) only support 64 bit mode"
- echo "pls remove -t or remove -m"
- exit -1
- fi
- cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
- else
- cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
- fi
|