MgeConvert のインストール CREStereo

pip install megengine==1.8.2

git clone https://github.com/MegEngine/mgeconvert.git && cd mgeconvert
git checkout 54fcb679d00ea31beca54d92de2e7d48fdb5af09

sudo chmod -R 777 .
pip install . --user --install-option="--targets=onnx"

${HOME}/.local/bin/convert --init
source ${HOME}/.local/share/bash-completion/completions/mgeconvert
cd ..

PINTO

${HOME}/.local/bin/convert -h
usage: convert [-h] [--init] {} ...

use "convert {mge_to_onnx, tracedmodule_to_onnx} -h" for more details

positional arguments:
  {}

optional arguments:
  -h, --help  show this help message and exit
  --init      init the bash completion

PINTO

${HOME}/.local/bin/convert mge_to_onnx -h

usage: convert mge_to_onnx [-h] -i INPUT -o OUTPUT [--opset OPSET] [--graph_name GRAPH_NAME] [--end_point END_POINT]

optional arguments:
  -h, --help            show this help message and exit
  -i INPUT, --input INPUT
                        Input megengine dump model file or onnx model
  -o OUTPUT, --output OUTPUT
                        Output onnx .onnx file or megengine model file(mge c++ model or traced module)
  --opset OPSET         Onnx opset version
  --graph_name GRAPH_NAME
                        Onnx graph name
  --end_point END_POINT
                        end_point is used to specify which part of the mge model should be converted

PINTO

${HOME}/.local/bin/convert mge_to_onnx \
-i crestereo_eth3d.mge \
-o crestereo_eth3d.onnx \
--opset 11

/home/user/.local/lib/python3.8/site-packages/megengine/core/tensor/megbrain_graph.py:508: ResourceWarning: unclosed file <_io.BufferedReader name='crestereo_eth3d.mge'>
  buf = open(fpath, "rb").read()
ResourceWarning: Enable tracemalloc to get the object allocation traceback
Traceback (most recent call last):
  File "/home/user/.local/bin/convert", line 525, in <module>
    main()
  File "/home/user/.local/bin/convert", line 518, in main
    args.func(args)
  File "/home/user/.local/bin/convert", line 283, in convert_func
    converter_map[target](
  File "/home/user/.local/lib/python3.8/site-packages/mgeconvert/converters/mge_to_onnx.py", line 50, in mge_to_onnx
    irgraph = MGE_FrontEnd(mge_fpath, outspec=outspec).resolve()
  File "/home/user/.local/lib/python3.8/site-packages/mgeconvert/frontend/mge_to_ir/mge_frontend.py", line 21, in __init__
    _, outputs = load_comp_graph_from_file(model_path)
  File "/home/user/.local/lib/python3.8/site-packages/mgeconvert/frontend/mge_to_ir/mge_utils.py", line 106, in load_comp_graph_from_file
    ret = G.load_graph(path)
  File "/home/user/.local/lib/python3.8/site-packages/megengine/core/tensor/megbrain_graph.py", line 511, in load_graph
    cg, metadata = _imperative_rt.load_graph(buf, output_vars_map, output_vars_list)
RuntimeError: access invalid Maybe value

backtrace:
/home/user/.local/lib/python3.8/site-packages/megengine/core/lib/libmegengine_shared.so(_ZN3mgb13MegBrainErrorC1ERKSs+0x4a) [0x7f3b39dfe1fa]
/home/user/.local/lib/python3.8/site-packages/megengine/core/lib/libmegengine_shared.so(_ZN3mgb17metahelper_detail27on_maybe_invalid_val_accessEv+0x34) [0x7f3b39f060f4]
/home/user/.local/lib/python3.8/site-packages/megengine/core/_imperative_rt.cpython-38-x86_64-linux-gnu.so(+0x14c605) [0x7f3b94873605]
/home/user/.local/lib/python3.8/site-packages/megengine/core/_imperative_rt.cpython-38-x86_64-linux-gnu.so(+0x14c823) [0x7f3b94873823]
/home/user/.local/lib/python3.8/site-packages/megengine/core/_imperative_rt.cpython-38-x86_64-linux-gnu.so(+0x11d62e) [0x7f3b9484462e]
/usr/bin/python3(PyCFunction_Call+0x59) [0x5f5e79]
/usr/bin/python3(_PyObject_MakeTpCall+0x296) [0x5f6a46]
/usr/bin/python3(_PyEval_EvalFrameDefault+0x5d3f) [0x570a1f]
/usr/bin/python3(_PyFunction_Vectorcall+0x1b6) [0x5f6226]
/usr/bin/python3(_PyEval_EvalFrameDefault+0x5706) [0x5703e6]

PINTO

仕切り直し

docker run --gpus all -it --rm \
-v `pwd`:/home/user/workdir \
ghcr.io/pinto0309/openvino2tensorflow:latest

wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
sudo apt-get update
sudo apt-get install -y intel-mkl-64bit-2020.0-088

git clone https://github.com/MegEngine/MegEngine && cd MegEngine
git checkout b6ad457269b8b57f317c7123de1c00c5520101e7
git submodule update --init

export MKL_ROOT=/opt/intel/mkl
export MKL_INCLUDE=$MKL_ROOT/include
export MKL_LIBRARY=$MKL_ROOT/lib/intel64
source /opt/intel/mkl/bin/mklvars.sh intel64
source /opt/intel/bin/compilervars.sh intel64
export CMAKE_INCLUDE_PATH=$MKL_INCLUDE:$CMAKE_INCLUDE_PATH
export CMAKE_LIBRARY_PATH=$MKL_LIBRARY:$CMAKE_LIBRARY_PATH
export CMAKE_PREFIX_PATH=/opt/intel/mkl

sed -i 's/PYTHON_DIR=/#PYTHON_DIR=/g' scripts/whl/manylinux2014/do_build_common.sh
sed -i '116i \    PYTHON_DIR=/usr/' scripts/whl/manylinux2014/do_build_common.sh
sed -i 's/\/usr\/lib64\/libatomic.so.1/\/usr\/lib\/x86_64-linux-gnu\/libatomic.so.1/g' scripts/whl/manylinux2014/do_build_common.sh
sed -i 's/cd \/home\/output/cd output/g' scripts/whl/manylinux2014/do_build_common.sh
sed -i '169i \    mkdir -p output && cd output' scripts/whl/manylinux2014/do_build_common.sh
sed -i '170d' scripts/whl/manylinux2014/do_build_common.sh
sed -i 's/cd \/home\/output/cd ${BUILD_DIR}\/staging\/output/g' scripts/whl/manylinux2014/do_build_common.sh

sudo rm -rf build_dir
sudo apt-get install -y libatomic1 patchelf
ALL_PYTHON="38" BUILD_WHL_CPU_ONLY="ON" scripts/whl/manylinux2014/do_build_common.sh
pip install ./scripts/whl/manylinux2014/output/wheelhouse/MegEngine-1.8.0+cpu-cp38-cp38-manylinux2014_x86_64.whl
cp ./scripts/whl/manylinux2014/output/wheelhouse/MegEngine-1.8.0+cpu-cp38-cp38-manylinux2014_x86_64.whl ${HOME}/workdir
cd ..

git clone https://github.com/MegEngine/mgeconvert.git && cd mgeconvert
git checkout 54fcb679d00ea31beca54d92de2e7d48fdb5af09
sudo chmod -R 777 .
pip install . --user --install-option="--targets=onnx"
cd ..
${HOME}/.local/bin/convert --init
source ${HOME}/.local/share/bash-completion/completions/mgeconvert

PINTO

${HOME}/.local/bin/convert mge_to_onnx \
-i crestereo_eth3d.mge \
-o crestereo_eth3d.onnx \
--opset 11

/home/user/.local/lib/python3.8/site-packages/megengine/core/tensor/megbrain_graph.py:508: ResourceWarning: unclosed file <_io.BufferedReader name='crestereo_eth3d.mge'>
  buf = open(fpath, "rb").read()
ResourceWarning: Enable tracemalloc to get the object allocation traceback
Traceback (most recent call last):
  File "/home/user/.local/bin/convert", line 525, in <module>
    main()
  File "/home/user/.local/bin/convert", line 518, in main
    args.func(args)
  File "/home/user/.local/bin/convert", line 283, in convert_func
    converter_map[target](
  File "/home/user/.local/lib/python3.8/site-packages/mgeconvert/converters/mge_to_onnx.py", line 50, in mge_to_onnx
    irgraph = MGE_FrontEnd(mge_fpath, outspec=outspec).resolve()
  File "/home/user/.local/lib/python3.8/site-packages/mgeconvert/frontend/mge_to_ir/mge_frontend.py", line 21, in __init__
    _, outputs = load_comp_graph_from_file(model_path)
  File "/home/user/.local/lib/python3.8/site-packages/mgeconvert/frontend/mge_to_ir/mge_utils.py", line 106, in load_comp_graph_from_file
    ret = G.load_graph(path)
  File "/home/user/.local/lib/python3.8/site-packages/megengine/core/tensor/megbrain_graph.py", line 511, in load_graph
    cg, metadata = _imperative_rt.load_graph(buf, output_vars_map, output_vars_list)
RuntimeError: access invalid Maybe value

backtrace:
/home/user/.local/lib/python3.8/site-packages/megengine/core/lib/libmegengine_shared.so(_ZN3mgb13MegBrainErrorC1ERKSs+0x4a) [0x7f3b39dfe1fa]
/home/user/.local/lib/python3.8/site-packages/megengine/core/lib/libmegengine_shared.so(_ZN3mgb17metahelper_detail27on_maybe_invalid_val_accessEv+0x34) [0x7f3b39f060f4]
/home/user/.local/lib/python3.8/site-packages/megengine/core/_imperative_rt.cpython-38-x86_64-linux-gnu.so(+0x14c605) [0x7f3b94873605]
/home/user/.local/lib/python3.8/site-packages/megengine/core/_imperative_rt.cpython-38-x86_64-linux-gnu.so(+0x14c823) [0x7f3b94873823]
/home/user/.local/lib/python3.8/site-packages/megengine/core/_imperative_rt.cpython-38-x86_64-linux-gnu.so(+0x11d62e) [0x7f3b9484462e]
/usr/bin/python3(PyCFunction_Call+0x59) [0x5f5e79]
/usr/bin/python3(_PyObject_MakeTpCall+0x296) [0x5f6a46]
/usr/bin/python3(_PyEval_EvalFrameDefault+0x5d3f) [0x570a1f]
/usr/bin/python3(_PyFunction_Vectorcall+0x1b6) [0x5f6226]
/usr/bin/python3(_PyEval_EvalFrameDefault+0x5706) [0x5703e6]

PINTO

仕切り直し２

docker run --gpus all -it --rm \
-v `pwd`:/home/user/workdir \
ghcr.io/pinto0309/openvino2tensorflow:latest

wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
sudo apt-get update
sudo apt-get install -y intel-mkl-64bit-2020.0-088

git clone https://github.com/MegEngine/MegEngine && cd MegEngine
git checkout b6ad457269b8b57f317c7123de1c00c5520101e7
git submodule update --init

export MKL_ROOT=/opt/intel/mkl
export MKL_INCLUDE=$MKL_ROOT/include
export MKL_LIBRARY=$MKL_ROOT/lib/intel64
source /opt/intel/mkl/bin/mklvars.sh intel64
source /opt/intel/bin/compilervars.sh intel64
export CMAKE_INCLUDE_PATH=$MKL_INCLUDE:$CMAKE_INCLUDE_PATH
export CMAKE_LIBRARY_PATH=$MKL_LIBRARY:$CMAKE_LIBRARY_PATH
export CMAKE_PREFIX_PATH=/opt/intel/mkl
export CUDNN_LIB_DIR="/usr/lib/x86_64-linux-gnu/"
export CUDA_LIB_DIR="/usr/local/cuda-11.4/targets/x86_64-linux/lib/"
export CUDA_COPY_LIB_LIST="${CUDA_LIB_DIR}/libnvrtc.so.11.2:${CUDA_LIB_DIR}/libcublasLt.so.11:${CUDA_LIB_DIR}/libcublas.so.11:${CUDNN_LIB_DIR}/libcudnn_adv_infer.so.8:${CUDNN_LIB_DIR}/libcudnn_adv_train.so.8:${CUDNN_LIB_DIR}/libcudnn_cnn_infer.so.8:${CUDNN_LIB_DIR}/libcudnn_cnn_train.so.8:${CUDNN_LIB_DIR}/libcudnn_ops_infer.so.8:${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:${CUDNN_LIB_DIR}/libcudnn.so.8"
export EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON -DMGE_CUDA_GENCODE=\"-gencode arch=compute_86,code=sm_86 -gencode arch=compute_86,code=compute_86\" "

sed -i 's/PYTHON_DIR=/#PYTHON_DIR=/g' scripts/whl/manylinux2014/do_build_common.sh
sed -i '116i \    PYTHON_DIR=/usr/' scripts/whl/manylinux2014/do_build_common.sh
sed -i 's/\/usr\/lib64\/libatomic.so.1/\/usr\/lib\/x86_64-linux-gnu\/libatomic.so.1/g' scripts/whl/manylinux2014/do_build_common.sh
sed -i 's/cd \/home\/output/cd output/g' scripts/whl/manylinux2014/do_build_common.sh
sed -i '169i \    mkdir -p output && cd output' scripts/whl/manylinux2014/do_build_common.sh
sed -i '170d' scripts/whl/manylinux2014/do_build_common.sh
sed -i 's/cd \/home\/output/cd ${BUILD_DIR}\/staging\/output/g' scripts/whl/manylinux2014/do_build_common.sh

sudo rm -rf build_dir
sudo apt-get install -y libatomic1 patchelf
ALL_PYTHON="38" BUILD_WHL_CPU_ONLY="ON" scripts/whl/manylinux2014/do_build_common.sh
pip install ./scripts/whl/manylinux2014/output/wheelhouse/MegEngine-1.8.0+cpu-cp38-cp38-manylinux2014_x86_64.whl
cp ./scripts/whl/manylinux2014/output/wheelhouse/MegEngine-1.8.0+cpu-cp38-cp38-manylinux2014_x86_64.whl ${HOME}/workdir
cd ..

git clone https://github.com/MegEngine/mgeconvert.git && cd mgeconvert
git checkout 54fcb679d00ea31beca54d92de2e7d48fdb5af09
sudo chmod -R 777 .
pip install . --user --install-option="--targets=onnx"
cd ..
${HOME}/.local/bin/convert --init
source ${HOME}/.local/share/bash-completion/completions/mgeconvert

PINTO