将conda虚拟环境打包并集成到singularity镜像中

1. 使用yml文件打包

bash 复制代码

conda activate your_env
conda env export > environment.yml

编写cond.def文件

bash 复制代码

Bootstrap: docker

From: continuumio/miniconda3

%files
    environment.yml

%post
    /opt/conda/bin/conda env create -f environment.yml

%runscript
    exec /opt/conda/envs/$(head -n 1 environment.yml | cut -f 2 -d ' ')/bin/"$@"

生成镜像:

bash 复制代码

singularity build conda.sif conda.def

2. 利用tar包

2.1 安装conda-pack

bash 复制代码

pip install conda-pack

版本需要0.7以上。

2.2 导出tar包

bash 复制代码

conda-pack -n <MY_ENV> -o packed_environment.tar.gz

编写conda.def文件:

bash 复制代码

Bootstrap: docker

From: continuumio/miniconda3

%files
    packed_environment.tar.gz /packed_environment.tar.gz

%post
    tar xvzf /packed_environment.tar.gz -C /opt/conda
    conda-unpack
    rm /packed_environment.tar.gz

生成镜像:

bash 复制代码

singularity build --fakeroot <OUTPUT_CONTAINER.sif> conda.def

3. 在已有基础上构建

def:

bash 复制代码

Bootstrap: localimage
From: local_image.sif

%environment
    # set up environment for when using the container
    . /opt/conda/etc/profile.d/conda.sh
    conda activate

%post
    apt-get update -y
    apt-get install -y \
            build-essential \
		    wget \
            cmake \
            g++ \
            r-base-dev \
			make
	    

    R -e "install.packages('cowsay', dependencies=TRUE, repos='http://cran.rstudio.com/')"
	
    # install miniconda
    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
    bash Miniconda3-latest-Linux-x86_64.sh -b -f -p /opt/conda
    rm Miniconda3-latest-Linux-x86_64.sh

    # install conda components - add the packages you need here
    . /opt/conda/etc/profile.d/conda.sh
    conda activate
    conda install -y -c conda-forge numpy cowpy
    conda update --all

4. 沙盒模式

4.1 构建沙河目录

bash 复制代码

singularity build --sandbox lolcow/ library://sylabs-jms/testing/lolcow

4.2 进入沙盒

bash 复制代码

singularity shell --writable lolcow/

4.3 将沙盒打包成sif

bash 复制代码

singularity build lolcow.sif lolcow/

5. 设置环境变量

pytorchcmake未设置cuda环境变量

bash 复制代码

SET(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} "path\\boost_1_80_0")
SET(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} "path\\boost_1_80_0\\libs")

可以通过如下设置:

bash 复制代码

%environment
    export CUDA_INCLUDE_DIRS=/opt/conda/cuda/include
    export CUDA_CUDART_LIBRARY=/opt/conda/cuda/lib

    export LIBRARY_PATH=/opt/conda/cuda/lib:$LIBRARY_PATH
    export CPATH=/opt/conda/cuda/include:$CPATH
    export PATH=/opt/conda/cuda:$PATH
%post
   mkdir -p /opt/conda/cuda
   conda install cuda -c nvidia -p /opt/conda/cuda
   
   mkdir -p /opt/conda/cudnn
   conda install -c anaconda cudnn -p /opt/conda/cudnn

   export PATH=/opt/conda/cuda:$PATH