diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..dcefbc3 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,71 @@ +name: Build Tensornet + +on: + push: + branches: + - '**' # matches every branch + tags: + - 'v[0-9]+' + - 'v[0-9]+\.[0-9]+' + - 'v[0-9]+\.[0-9]+\.[0-9]+' + pull_request: + +jobs: + tn_build: + runs-on: ubuntu-latest + steps: + - name: checkout repository + uses: actions/checkout@v4 + + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: '1.5.8-0' + environment-file: config/tn_build.yaml + init-shell: bash + cache-downloads: true + post-cleanup: 'none' + - name: Run custom command in micromamba environment + run: ./manager build + shell: micromamba-shell {0} + + - name: Create setup dist + run: ./manager create_dist + shell: micromamba-shell {0} + + - name: Store wheels + uses: actions/upload-artifact@v4 + with: + path: dist/ + retention-days: 7 + + publish-to-test-pypi: + name: Upload to test-pypi + needs: + - tn_build + permissions: + id-token: write # mandatory for trusted publishing + steps: + - name: Download wheels + uses: actions/download-artifact@v4 + with: + path: dist/ + - name: Publish wheels to test-PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + publish-to-pypi: + name: Upload to pypi + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + permissions: + id-token: write # mandatory for trusted publishing + steps: + - name: Download wheels + uses: actions/download-artifact@v4 + with: + path: dist/ + - name: Publish wheels to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..4d15b92 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,32 @@ +tn_build: + stage: build + tags: + - image-unlocked + image: + name: r.addops.soft.360.cn/sycp-container/centos7.2:base + + variables: + NEED_PREPARE_ENV: "true" + + before_script: + - mkdir -p ${HOME}/.config/pip + - | + cat > "${HOME}/.config/pip/pip.conf" <&2 "[ERROR]$err_fmt %s\n" "$*" + exit $err +} + +_prepare_mamba_env(){ + if ! type micromamba >/dev/null 2>&1;then + HTTPS_PROXY=${PROXY_URL:=${HTTPS_PROXY}} "${SHELL}" <(curl -L micro.mamba.pm/install.sh) + fi + _mamba_source + [[ -z ${NEXUS3_HEADER} ]] || { + ${MAMBA_EXE} config set --file "${MAMBA_ROOT_PREFIX}/.mambarc" channel_alias ${NEXUS3_HEADER}/conda + } + micromamba create -y -f ${WORKSPACE_DIR}/config/${TN_BUILD_ENV_NAME}.yaml + micromamba activate ${TN_BUILD_ENV_NAME} +} + +_mamba_source() { + [[ -e ${MAMBA_EXE} ]] || { echo "no micromamba exe found, run ./manager prepare_build_env to create env"; exit 1;} + __mamba_setup="$("$MAMBA_EXE" shell hook --shell bash --root-prefix "$MAMBA_ROOT_PREFIX" 2> /dev/null)" + if [ $? -eq 0 ]; then + eval "$__mamba_setup" + else + alias micromamba="$MAMBA_EXE" # Fallback on help from mamba activate + fi + unset __mamba_setup +} + +_activate_env() { + _mamba_source + micromamba activate ${TN_BUILD_ENV_NAME} +} + +_prepare_compile_env() { + CUR_ENV_PATH=$(ompi_info --parsable --path prefix 2>/dev/null | awk -F":" '{print $NF}') + export C_INCLUDE_PATH=${CUR_ENV_PATH}/include + export CPLUS_INCLUDE_PATH=${CUR_ENV_PATH}/include +} + +_build_config(){ + CUR_ENV_PATH=$(ompi_info --parsable --path prefix 2>/dev/null | awk -F":" '{print $NF}') + cd ${WORKSPACE_DIR}; bash configure.sh --openmpi_path ${CUR_ENV_PATH} + _prepare_compile_env +} + +start_build(){ + [[ ${NEED_PREPARE_ENV} == true ]] && _prepare_mamba_env + [[ ${NEED_ACTIVATE_ENV} == true ]] && _activate_env + _build_config + extra_opts=("$@") + [[ ${DEBUG-} != true ]] || extra_opts+=(--sandbox_debug) + bazel build "${extra_opts[@]}" -c opt //core:_pywrap_tn.so +} + +only_build(){ + [[ ${NEED_ACTIVATE_ENV} == true ]] && _activate_env + _prepare_compile_env + extra_opts=("$@") + [[ ${DEBUG-} != true ]] || extra_opts+=(--sandbox_debug) + bazel build "${extra_opts[@]}" -c opt //core:_pywrap_tn.so +} + + +start_copy_libs(){ + rm -f tensornet/core/_pywrap_tn.so || true + cp bazel-bin/core/_pywrap_tn.so tensornet/core/_pywrap_tn.so +} + +start_test(){ + python -c "import tensorflow as tf;import tensornet as tn;tn.core.init()" +} + + +start_only_upload(){ + [[ ${NEED_ACTIVATE_ENV} == true ]] && _activate_env + export TWINE_USERNAME=${TWINE_USERNAME:=${NEXUS3_USERNAME}} + export TWINE_PASSWORD=${TWINE_PASSWORD:=${NEXUS3_PASSWORD}} + if [[ -z "$TWINE_USERNAME" || -z "$TWINE_PASSWORD" ]];then + echo "need username/password auth, no env " + echo "export NEXUS3_USERNAME=xxxx" + echo "export NEXUS3_PASSWORD=xxxx" + exit 0 + fi + [[ -z ${NEXUS3_PYPI_HOST} ]] && { echo "need pypi host address, export NEXUS3_PYPI_HOST=xxx"; exit 0; } + twine upload --verbose --repository-url ${NEXUS3_PYPI_HOST} dist/* +} + +start_create_dist(){ + [[ ${NEED_PREPARE_ENV} == true ]] && _prepare_mamba_env + [[ ${NEED_ACTIVATE_ENV} == true ]] && _activate_env + rm -rf dist/* || true + start_copy_libs + [[ $# > 0 ]] && export TN_VERSION=$1 + PY_VERSION=$(python -c "import sys; print('cp' + ''.join(map(str, sys.version_info[:2])))") + python setup.py bdist_wheel --plat-name manylinux2010_x86_64 --python-tag ${PY_VERSION} +} + +start_upload(){ + start_create_dist + start_only_upload +} + +case "$1" in +(prepare_build_env) + _prepare_mamba_env + ;; +(build) + shift 1 + start_build "$@" + ;; +(only-build) + shift 1 + only_build "$@" + ;; +(deploy) + shift 1 + start_upload "$@" + ;; +(copy-libs) + start_copy_libs + ;; +(create_dist) + shift 1 + start_create_dist "$@" + ;; +(help) + cmd=$(basename -- "$0") + cat <<-END + Usage: + $cmd help - Print this help. + + $cmd prepare_build_env - install micromamba environment. + + $cmd build [args..] - Build tn so file. + + $cmd only-build [args..] - Build tn so file without config mpi + + $cmd deploy [version] - deploy tn to pypi + + $cmd create_dist [version] - create setup dist without upload +END + ;; +(*) die Unknown command "$1" ;; +esac diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..552bce2 --- /dev/null +++ b/setup.py @@ -0,0 +1,27 @@ +import os +from setuptools import setup, find_packages + +version = os.environ.get('TN_VERSION', '0.1.2') + +setup( + name='qihoo-tensornet', + version=version, + description='tensornet', + author='jiangxinglei', + author_email='jiangxinglei@360.cn', + url='https://github.com/Qihoo360/tensornet', + packages=find_packages(), + package_data = { + "tensornet.core": ["_pywrap_tn.so"], + }, + install_requires=[ + 'tensorflow>=2.2,<2.3' + ], + python_requires='>=3.7, <3.8', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3.7' + ], +) diff --git a/thirdparty/openmpi/BUILD b/thirdparty/openmpi/BUILD index 5c1f8e6..81ba592 100644 --- a/thirdparty/openmpi/BUILD +++ b/thirdparty/openmpi/BUILD @@ -2,7 +2,6 @@ cc_library( name = "openmpi", hdrs = glob(["include/**"]), includes = ["include"], - srcs = glob(["lib/lib*.so"]), + linkopts = ["-lmpi"], visibility = ["//visibility:public"], ) -