From ea2245b35cbfddb8c7473453b7d977b0a64f61b2 Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Thu, 8 Sep 2022 02:16:52 +0200 Subject: Update containers to ubuntu 22.04, gcc-12; include perf/gdb; add python libraries for opencv, machine learning, etc. --- build/buildah-astra.sh | 7 +++++-- build/buildah-base.sh | 26 ++++++++++++++++++++++---- build/buildah-ccpi.sh | 7 +++++-- build/buildah-remote.sh | 11 +++++++---- build/buildah-tomo.sh | 7 +++++-- build/buildah-ufo.sh | 7 +++++-- build/buildah.sh | 12 +++++++----- build/update.sh | 12 ++++++++---- patches/ccpi-readme.txt | 7 +++++++ run/config.sh | 4 +++- 10 files changed, 74 insertions(+), 26 deletions(-) create mode 100644 patches/ccpi-readme.txt diff --git a/build/buildah-astra.sh b/build/buildah-astra.sh index 1282e28..6f6e7b1 100644 --- a/build/buildah-astra.sh +++ b/build/buildah-astra.sh @@ -2,9 +2,12 @@ set -o errexit -container=$(buildah from pycuda:latest) +tag=latest +[ -n "$1" ] && tag="$1" + +container=$(buildah from pycuda:$tag) buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container buildah run $container bash /root/setup/repos.sh "astra" -buildah commit --format docker $container astra:latest +buildah commit --format docker $container astra:$tag diff --git a/build/buildah-base.sh b/build/buildah-base.sh index 319ed6b..fcb3200 100644 --- a/build/buildah-base.sh +++ b/build/buildah-base.sh @@ -2,8 +2,13 @@ set -o errexit +tag=latest +[ -n "$1" ] && tag="$1" + #container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04) -container=$(buildah from nvidia/cuda:11.3.0-devel-ubuntu20.04) +[ $tag == "20.04" ] && container=$(buildah from nvidia/cuda:11.3.0-devel-ubuntu20.04) +[ $tag == "22.04" ] && container=$(buildah from nvidia/cuda:11.7.1-devel-ubuntu22.04) +[ -n "$container" ] && container=$(buildah from nvidia/cuda:11.7.1-devel-ubuntu$tag) buildah config --label maintainer="Suren A. Chilingaryan " $container buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container @@ -24,12 +29,23 @@ buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/d buildah run $container sh -c 'echo "systemd-timesync:x:107:" >> /etc/group' buildah run $container sh -c 'echo "systemd-timesync:x:107:107:systemd-timesync:/dev/null:/sbin/nologin" >> /etc/passwd' +# https://stackoverflow.com/questions/71941032/why-i-cannot-run-apt-update-inside-a-fresh-ubuntu22-04 +buildah run $container sh -c 'sed -i -e "s/^APT/# APT/" -e "s/^DPkg/# DPkg/" /etc/apt/apt.conf.d/docker-clean' -buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server' +buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc rar ca-certificates git openssh-server' buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3' buildah run $container sh -c 'apt-get install -y cmake make autoconf automake libtool gcc g++ libboost-all-dev' + +[ $tag == "22.04" ] && buildah run $container sh -c 'apt-get install -y gcc-12 g++-12' # First gcc version supporting _Float16 data type +[ $tag != "20.04" ] && buildah run $container sh -c 'apt-get install -y screen perf-tools-unstable gdb htop' # Debugging/Profilling tools buildah run $container sh -c 'apt-get install -y ca-certificates pkg-config gobject-introspection liblapack-dev libjpeg-dev libtiff-dev libglib2.0-dev libjson-glib-dev libopenmpi-dev libhdf5-dev libclfft-dev libgsl-dev libgirepository1.0-dev fftw3-dev zlib1g-dev' -buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy python-gobject tifffile' + +# Problems with python3-tifffile (this should be ran after installing pkgconfig/gobject stuff) +buildah run $container sh -c 'sed -i -e "s/^post-invoke/# post-invoke/" /etc/dpkg/dpkg.cfg.d/pkg-config-hook-config' + +buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy python3-gi python3-gi-cairo python3-tifffile' + +[ $tag != "20.04" ] && buildah run $container sh -c 'apt-get install -y python3-astropy python3-opencv python3-willow python3-pypillowfight python3-scipy python3-sklearn python3-skorch python3-tomopy' # Not needed for ccpi/ufo, but rather additional image-processing code for python buildah run $container sh -c 'apt-get install -y ocl-icd-opencl-dev clinfo' buildah run $container sh -c 'apt-get install -y jupyter-notebook' @@ -39,8 +55,10 @@ buildah run $container sh -c 'update-alternatives --install /usr/bin/python pyth buildah run $container mkdir -p /etc/OpenCL/vendors buildah run $container sh -c 'echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd' +[ $tag != "20.04" ] && buildah run $container sh -c 'rm /usr/bin/perf; ln -s /usr/lib/linux-tools/*/perf /usr/bin/perf' + buildah copy $container pre-setup /root/pre-setup buildah run $container pip3 install -r /root/pre-setup/requirements.txt buildah copy $container setup /root/setup -buildah commit --format docker $container pycuda:latest +buildah commit --format docker $container pycuda:$tag diff --git a/build/buildah-ccpi.sh b/build/buildah-ccpi.sh index 661e0ef..6912476 100644 --- a/build/buildah-ccpi.sh +++ b/build/buildah-ccpi.sh @@ -2,9 +2,12 @@ set -o errexit -container=$(buildah from astra:latest) +tag=latest +[ -n "$1" ] && tag="$1" + +container=$(buildah from astra:$tag) buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=21.0.0 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container buildah run $container bash /root/setup/repos.sh "ccpi" -buildah commit --format docker $container ccpi:latest +buildah commit --format docker $container ccpi:$tag diff --git a/build/buildah-remote.sh b/build/buildah-remote.sh index 5e9d038..7cb0eeb 100644 --- a/build/buildah-remote.sh +++ b/build/buildah-remote.sh @@ -3,9 +3,12 @@ set -o errexit image="$1" -[ -n "$image" ] || { echo "Usage: $0 "; exit 1; } +[ -n "$image" ] || { echo "Usage: $0 [tag]"; exit 1; } -container=$(buildah from ${image}:latest) +tag=latest +[ -n "$2" ] && tag="$2" + +container=$(buildah from ${image}:${tag}) buildah config --port 2222/tcp --port 8888/tcp $container buildah run $container sh -c 'apt-get install -y supervisor' @@ -36,6 +39,6 @@ exitcodes=0 EOF ' -buildah commit --format docker $container ${image}-remote:latest +buildah commit --format docker $container ${image}-remote:${tag} -echo "$container build from ${image} and saved as ${image}-remote" +echo "$container build from ${image}:${tag} and saved as ${image}-remote:${tag}" diff --git a/build/buildah-tomo.sh b/build/buildah-tomo.sh index ed9287e..486f182 100644 --- a/build/buildah-tomo.sh +++ b/build/buildah-tomo.sh @@ -2,10 +2,13 @@ set -o errexit -container=$(buildah from ccpi:latest) +tag=latest +[ -n "$1" ] && tag="$1" + +container=$(buildah from ccpi:$tag) buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container buildah run $container bash /root/setup/repos.sh "ufo" -buildah commit --format docker $container tomo:latest +buildah commit --format docker $container tomo:$tag diff --git a/build/buildah-ufo.sh b/build/buildah-ufo.sh index ee8e328..bdc6331 100644 --- a/build/buildah-ufo.sh +++ b/build/buildah-ufo.sh @@ -2,10 +2,13 @@ set -o errexit -container=$(buildah from pycuda:latest) +tag=latest +[ -n "$1" ] && tag="$1" + +container=$(buildah from pycuda:$tag) buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container buildah run $container bash /root/setup/repos.sh "ufo" -buildah commit --format docker $container ufo:latest +buildah commit --format docker $container ufo:$tag diff --git a/build/buildah.sh b/build/buildah.sh index 8e84457..2ba99e5 100644 --- a/build/buildah.sh +++ b/build/buildah.sh @@ -1,7 +1,9 @@ set -o errexit -bash buildah-base.sh -bash buildah-astra.sh -bash buildah-ccpi.sh -bash buildah-tomo.sh -bash buildah-remote.sh tomo +tag=22.04 + +bash buildah-base.sh $tag +bash buildah-astra.sh $tag +bash buildah-ccpi.sh $tag +bash buildah-tomo.sh $tag +bash buildah-remote.sh tomo $tag diff --git a/build/update.sh b/build/update.sh index a3dc9ff..6a8ac7a 100644 --- a/build/update.sh +++ b/build/update.sh @@ -3,10 +3,14 @@ set -o errexit image="$1" -[ -n "$image" ] || { echo "Usage: $0 "; exit 1; } +[ -n "$image" ] || { echo "Usage: $0 [tag]"; exit 1; } + +tag=latest +[ -n "$2" ] && tag="$2" + #container=cuda-working-container-15 -container=$(buildah from localhost/${image}) +container=$(buildah from localhost/${image}:${tag}) #buildah run $container bash #exit @@ -15,5 +19,5 @@ buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PA buildah copy $container setup /root/setup buildah run $container bash /root/setup/repos.sh "all" "/root/setup" -#buildah commit --format docker $container ccpi:latest -echo "buildah commit --format docker $container ${image}:latest" +#buildah commit --format docker $container ccpi:${tag} +echo "buildah commit --format docker $container ${image}:${tag}" diff --git a/patches/ccpi-readme.txt b/patches/ccpi-readme.txt new file mode 100644 index 0000000..4399a22 --- /dev/null +++ b/patches/ccpi-readme.txt @@ -0,0 +1,7 @@ + - There is also extra memory consumed in FISTA implementation, here: + /ccpi/repos/CCPi-Framework/Wrappers/Python/cil/optimisation/algorithms + We might reduce further 1-2 volumes if rewritten in C and temporarily variables + are properly eleminated. We further might introduce simplified memory manager + which will allow to give up further temporarily in FISTA and allocated them in + regularize and, then, vice-versa. + diff --git a/run/config.sh b/run/config.sh index bc1d6c5..d2b01dd 100644 --- a/run/config.sh +++ b/run/config.sh @@ -6,7 +6,9 @@ web_port="${port}80" image="$1" -tag="latest" +#tag="latest" +tag="22.04" + [ -n "$image" ] || image="localhost/tomo-remote:$tag" # tomo-remote:latest tomo-remote:devel tomo-remote:evelina (also ccpi, ufo, tomo) options="--cap-add SYS_ADMIN --cap-add=SYS_PTRACE --hooks-dir /usr/share/containers/oci/hooks.d" -- cgit v1.2.3