From 8fb3c0a3f96024786305a1234fe5cfb70e6f00c0 Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Mon, 9 Mar 2020 03:37:34 +0100 Subject: Initial release --- .gitignore | 2 ++ build/buildah.sh | 34 +++++++++++++++++++++++++++++++++ build/extract.sh | 9 +++++++++ build/pre-setup/requirements.txt | 6 ++++++ build/setup/build/astra.sh | 11 +++++++++++ build/setup/build/ccpi.sh | 13 +++++++++++++ build/setup/build/cmake.sh | 9 +++++++++ build/setup/build/python.sh | 9 +++++++++ build/setup/provision.sh | 15 +++++++++++++++ build/setup/repos.sh | 41 ++++++++++++++++++++++++++++++++++++++++ build/update.sh | 16 ++++++++++++++++ run/attach-devel.sh | 1 + run/devel.sh | 9 +++++++++ run/run-devel.sh | 5 +++++ run/run.sh | 5 +++++ run/test.sh | 6 ++++++ run/update-from-devel.sh | 12 ++++++++++++ 17 files changed, 203 insertions(+) create mode 100644 .gitignore create mode 100644 build/buildah.sh create mode 100644 build/extract.sh create mode 100644 build/pre-setup/requirements.txt create mode 100644 build/setup/build/astra.sh create mode 100644 build/setup/build/ccpi.sh create mode 100644 build/setup/build/cmake.sh create mode 100644 build/setup/build/python.sh create mode 100644 build/setup/provision.sh create mode 100644 build/setup/repos.sh create mode 100644 build/update.sh create mode 100644 run/attach-devel.sh create mode 100644 run/devel.sh create mode 100644 run/run-devel.sh create mode 100644 run/run.sh create mode 100644 run/test.sh create mode 100644 run/update-from-devel.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6207b24 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/repos +/data diff --git a/build/buildah.sh b/build/buildah.sh new file mode 100644 index 0000000..45ee576 --- /dev/null +++ b/build/buildah.sh @@ -0,0 +1,34 @@ +#! /usr/bin/env bash + +set -o errexit + +#container=$(buildah from nvidia/cuda:10.1-base-ubuntu18.04) +container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04) + +buildah config --label maintainer="Suren A. Chilingaryan " $container +buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=19.10 --env HOME=/ccpi/data $container + +buildah run $container sh -c 'echo "ssh:x:101:" >> /etc/group' +buildah run $container sh -c 'echo "messagebus:x:102:" >> /etc/group' +buildah run $container sh -c 'echo "rdma:x:103:" >> /etc/group' +buildah run $container sh -c 'echo "messagebus:x:101:102:messagebus:/dev/null:/sbin/nologin" >> /etc/passwd' + +buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git' +buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3' +buildah run $container sh -c 'apt-get install -y cmake autoconf automake libtool libboost-all-dev' + +buildah run $container sh -c 'update-alternatives --remove python /usr/bin/python2' +buildah run $container sh -c 'update-alternatives --install /usr/bin/python python /usr/bin/python3 10' + +buildah copy $container pre-setup /root/pre-setup +buildah run $container pip3 install -r /root/pre-setup/requirements.txt + +buildah copy $container setup /root/setup +buildah run $container bash /root/setup/repos.sh + +buildah commit --format docker $container ccpi:latest + + + +echo $container + diff --git a/build/extract.sh b/build/extract.sh new file mode 100644 index 0000000..5321db8 --- /dev/null +++ b/build/extract.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env bash + +set -o errexit + +container=$(buildah from localhost/ccpi) +path=$(buildah mount $container) +mkdir -p ../repos/ +cp -ra "$path/ccpi/repos"/* ../repos/ +buildah umount $container diff --git a/build/pre-setup/requirements.txt b/build/pre-setup/requirements.txt new file mode 100644 index 0000000..87e3dca --- /dev/null +++ b/build/pre-setup/requirements.txt @@ -0,0 +1,6 @@ +setuptools +numpy +scipy +matplotlib +h5py +pillow diff --git a/build/setup/build/astra.sh b/build/setup/build/astra.sh new file mode 100644 index 0000000..ed79a1d --- /dev/null +++ b/build/setup/build/astra.sh @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +set -o errexit + +( + cd build/linux + ./autogen.sh + ./configure --with-cuda=/usr/local/cuda --with-python=/usr/bin/python3 + make + make install +) diff --git a/build/setup/build/ccpi.sh b/build/setup/build/ccpi.sh new file mode 100644 index 0000000..329fbdf --- /dev/null +++ b/build/setup/build/ccpi.sh @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +real_script=$(readlink "$0") +[ -z "$real_script" ] && real_script="$0" +script_path=$(dirname "$real_script") + +set -o errexit + +# Do both (as framework's cmake install python files in invalid location) +[ -f CMakeLists.txt ] && bash "$script_path/cmake.sh" +[ -d Wrappers/Python ] && bash "$script_path/python.sh" + +exit 0 diff --git a/build/setup/build/cmake.sh b/build/setup/build/cmake.sh new file mode 100644 index 0000000..09beccf --- /dev/null +++ b/build/setup/build/cmake.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env bash + +set -o errexit + +pypath=$(python3 -c 'import site; print(site.getsitepackages()[0])') +#rm CMakeCache.txt +cmake -D "PYTHON_DEST_DIR:PATH=$pypath" . +make +make install diff --git a/build/setup/build/python.sh b/build/setup/build/python.sh new file mode 100644 index 0000000..de08a2c --- /dev/null +++ b/build/setup/build/python.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env bash + +set -o errexit + +( + cd Wrappers/Python + python3 setup.py install +) + diff --git a/build/setup/provision.sh b/build/setup/provision.sh new file mode 100644 index 0000000..8066399 --- /dev/null +++ b/build/setup/provision.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +set -o errexit + +setup_path="$1" +repos_path="$2" +[ -z "$setup_path" ] && setup_path="/root/setup" +[ -z "$repos_path" ] && repos_path="/ccpi/repos" + +for name in "$repos_path"/*; do + ( + cd $name + [ -f "ands_install.sh" ] && bash "ands_install.sh" + ) +done diff --git a/build/setup/repos.sh b/build/setup/repos.sh new file mode 100644 index 0000000..9e76208 --- /dev/null +++ b/build/setup/repos.sh @@ -0,0 +1,41 @@ +#! /usr/bin/env bash + +set -o errexit + +setup_path="$1" +repos_path="$2" +[ -z "$setup_path" ] && setup_path="/root/setup" +[ -z "$repos_path" ] && repos_path="/ccpi/repos" + +repos=( \ + "https://github.com/astra-toolbox/astra-toolbox.git tags/v1.8.3 astra.sh" \ + "https://github.com/vais-ral/CCPi-Framework.git master ccpi.sh" \ + "https://github.com/vais-ral/CCPi-FrameworkPlugins.git master ccpi.sh" \ + "https://github.com/vais-ral/CCPi-astra.git update_projectors ccpi.sh" \ + "https://github.com/vais-ral/CCPi-Regularisation-Toolkit.git master ccpi.sh" \ +) + +function install_repo { + repo="$1" + name=$(basename "$repo" .git) + branch="$2" + script="$3" + + [ -d "$name" ] || git clone "$repo" + + ( + cd "$name" + git pull origin "$branch" + git checkout "$branch" + bash "$setup_path/build/$script" + ln -sf "$setup_path/build/$script" ands_install.sh + ) +} + +mkdir -p "$repos_path" +( + cd "$repos_path" + for repo in "${repos[@]}"; do + install_repo $repo + done +) diff --git a/build/update.sh b/build/update.sh new file mode 100644 index 0000000..7f5dcd0 --- /dev/null +++ b/build/update.sh @@ -0,0 +1,16 @@ +#! /usr/bin/env bash + +set -o errexit + +#container=cuda-working-container-15 +container=$(buildah from localhost/ccpi) + +#buildah run $container bash +#exit + +buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data $container +buildah copy $container setup /root/setup +buildah run $container bash /root/setup/repos.sh "/root/setup" + +#buildah commit --format docker $container ccpi:latest +echo "buildah commit --format docker $container ccpi:latest" diff --git a/run/attach-devel.sh b/run/attach-devel.sh new file mode 100644 index 0000000..6facdd4 --- /dev/null +++ b/run/attach-devel.sh @@ -0,0 +1 @@ +podman exec -it "ccpi-devel" bash diff --git a/run/devel.sh b/run/devel.sh new file mode 100644 index 0000000..57ec7f0 --- /dev/null +++ b/run/devel.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +set -o errexit + +if [ ! -d ../repos ]; then + buildah unshare bash ../build/extract.sh +fi + +podman run --name "ccpi-devel" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/ccpi bash -c "bash /root/setup/provision.sh; bash" diff --git a/run/run-devel.sh b/run/run-devel.sh new file mode 100644 index 0000000..2adb95c --- /dev/null +++ b/run/run-devel.sh @@ -0,0 +1,5 @@ +#! /bin/bash + +set -o errexit + +podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data localhost/ccpi:devel bash diff --git a/run/run.sh b/run/run.sh new file mode 100644 index 0000000..5ebb075 --- /dev/null +++ b/run/run.sh @@ -0,0 +1,5 @@ +#! /bin/bash + +set -o errexit + +podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data localhost/ccpi bash diff --git a/run/test.sh b/run/test.sh new file mode 100644 index 0000000..90624c0 --- /dev/null +++ b/run/test.sh @@ -0,0 +1,6 @@ +#! /bin/bash + +set -o errexit + +#podman exec -it "ccpi-devel" bash /ccpi/data/run.sh +podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data -e "CUDA_VISIBLE_DEVICES=2" localhost/ccpi:devel bash /ccpi/data/run.sh diff --git a/run/update-from-devel.sh b/run/update-from-devel.sh new file mode 100644 index 0000000..19aee6b --- /dev/null +++ b/run/update-from-devel.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +set -o errexit + +if [ ! -d ../repos ]; then + buildah unshare bash ../build/extract.sh +fi + +podman run --name "ccpi-devel" -it --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/ccpi bash /root/setup/provision.sh +podman container cp ../repos/ ccpi-devel:/ccpi/ +podman container commit ccpi-devel ccpi:devel +podman rm ccpi-devel -- cgit v1.2.3