Support for ufo & jupyter notebooks

13 files changed, 112 insertions, 45 deletions

diff --git a/build/buildah-astra.sh b/build/buildah-astra.sh
index fa69db6..1282e28 100644
--- a/build/buildah-astra.sh
+++ b/build/buildah-astra.sh
@@ -2,32 +2,9 @@
 
 set -o errexit
 
-container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
+container=$(buildah from pycuda:latest)
 
-buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
-buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
-
-buildah run $container sh -c 'echo "ssh:x:101:" >> /etc/group'
-buildah run $container sh -c 'echo "sshd:x:101:101:sshd:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "messagebus:x:102:" >> /etc/group'
-buildah run $container sh -c 'echo "messagebus:x:102:102:messagebus:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "rdma:x:103:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-journal:x:104:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-network:x:105:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-network:x:105:105:systemd-network:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "systemd-resolve:x:106:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/dev/null:/sbin/nologin" >> /etc/passwd'
-
-buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
-buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
-buildah run $container sh -c 'apt-get install -y cmake autoconf automake libtool libboost-all-dev'
-
-buildah run $container sh -c 'update-alternatives --remove python /usr/bin/python2'
-buildah run $container sh -c 'update-alternatives --install /usr/bin/python python /usr/bin/python3 10'
-
-buildah copy $container pre-setup /root/pre-setup
-buildah run $container pip3 install -r /root/pre-setup/requirements.txt
-
-buildah copy $container setup /root/setup
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
 buildah run $container bash /root/setup/repos.sh "astra"
+
 buildah commit --format docker $container astra:latest
diff --git a/build/buildah-base.sh b/build/buildah-base.sh
new file mode 100644
index 0000000..b656515
--- /dev/null
+++ b/build/buildah-base.sh
@@ -0,0 +1,42 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
+
+buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
+
+buildah run $container ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime
+buildah run $container sh -c 'echo "Europe/London" > /etc/timezone'
+
+buildah run $container sh -c 'echo "ssh:x:101:" >> /etc/group'
+buildah run $container sh -c 'echo "sshd:x:101:101:sshd:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "messagebus:x:102:" >> /etc/group'
+buildah run $container sh -c 'echo "messagebus:x:102:102:messagebus:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "rdma:x:103:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-journal:x:104:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-network:x:105:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-network:x:105:105:systemd-network:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "systemd-resolve:x:106:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/dev/null:/sbin/nologin" >> /etc/passwd'
+
+buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
+buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
+buildah run $container sh -c 'apt-get install -y cmake make autoconf automake libtool gcc g++ libboost-all-dev'
+buildah run $container sh -c 'apt-get install -y ca-certificates pkg-config gobject-introspection liblapack-dev libjpeg-dev libtiff-dev libglib2.0-dev libjson-glib-dev libopenmpi-dev libhdf5-dev libclfft-dev libgsl-dev libgirepository1.0-dev fftw3-dev zlib1g-dev'
+buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy  python-gobject python-tifffile'
+buildah run $container sh -c 'apt-get install -y ocl-icd-opencl-dev clinfo'
+buildah run $container sh -c 'apt-get install -y jupyter-notebook'
+
+buildah run $container sh -c 'update-alternatives --remove python /usr/bin/python2'
+buildah run $container sh -c 'update-alternatives --install /usr/bin/python python /usr/bin/python3 10'
+
+buildah run $container mkdir -p /etc/OpenCL/vendors
+buildah run $container sh -c 'echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd'
+
+buildah copy $container pre-setup /root/pre-setup
+buildah run $container pip3 install -r /root/pre-setup/requirements.txt
+
+buildah copy $container setup /root/setup
+buildah commit --format docker $container pycuda:latest
diff --git a/build/buildah-ccpi.sh b/build/buildah-ccpi.sh
index 1bc6b10..3d006fc 100644
--- a/build/buildah-ccpi.sh
+++ b/build/buildah-ccpi.sh
@@ -4,18 +4,7 @@ set -o errexit
 
 container=$(buildah from astra:latest)
 
-buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=19.10 --env HOME=/ccpi/data $container
-buildah config --port 2222 $container
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
+buildah run $container bash /root/setup/repos.sh "ccpi"
 
-buildah run $container bash /root/setup/repos.sh "all"
-
-buildah run $container sh -c "echo '$(id -ng):x:$(id -g):' >> /etc/group"
-#buildah run $container sh -c "echo '$(id -nu):x:$(id -u):$(id -g):$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
-buildah run $container sh -c "echo '$(id -nu):x:0:0:$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
-
-buildah run $container sh -c 'service ssh start'
-buildah run $container sh -c "echo '$(id -nu):test' | chpasswd"
 buildah commit --format docker $container ccpi:latest
-
-echo $container
-
diff --git a/build/buildah-remote.sh b/build/buildah-remote.sh
new file mode 100644
index 0000000..1016d19
--- /dev/null
+++ b/build/buildah-remote.sh
@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+
+container=$(buildah from ${image}:latest)
+buildah config --port 2222/tcp --port 8888/tcp $container
+
+buildah run $container sh -c "echo '$(id -ng):x:$(id -g):' >> /etc/group"
+#buildah run $container sh -c "echo '$(id -nu):x:$(id -u):$(id -g):$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
+buildah run $container sh -c "echo '$(id -nu):x:0:0:$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
+
+buildah run $container sh -c 'service ssh start'
+buildah run $container sh -c "echo '$(id -nu):test' | chpasswd"
+
+buildah commit --format docker $container ${image}-remote:latest
+
+echo "$container build from ${image} and saved as ${image}-remote"
diff --git a/build/buildah-tomo.sh b/build/buildah-tomo.sh
new file mode 100644
index 0000000..ed9287e
--- /dev/null
+++ b/build/buildah-tomo.sh
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from ccpi:latest)
+
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
+
+buildah run $container bash /root/setup/repos.sh "ufo"
+
+buildah commit --format docker $container tomo:latest
diff --git a/build/buildah-ufo.sh b/build/buildah-ufo.sh
new file mode 100644
index 0000000..ee8e328
--- /dev/null
+++ b/build/buildah-ufo.sh
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from pycuda:latest)
+
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
+
+buildah run $container bash /root/setup/repos.sh "ufo"
+
+buildah commit --format docker $container ufo:latest
diff --git a/build/buildah.sh b/build/buildah.sh
index 2f4f283..8e84457 100644
--- a/build/buildah.sh
+++ b/build/buildah.sh
@@ -1,4 +1,7 @@
 set -o errexit
 
+bash buildah-base.sh
 bash buildah-astra.sh
 bash buildah-ccpi.sh
+bash buildah-tomo.sh
+bash buildah-remote.sh tomo
diff --git a/build/extract.sh b/build/extract.sh
index 5321db8..17c6e3f 100644
--- a/build/extract.sh
+++ b/build/extract.sh
@@ -2,7 +2,11 @@
 
 set -o errexit
 
-container=$(buildah from localhost/ccpi)
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+
+
+container=$(buildah from localhost/${image})
 path=$(buildah mount $container)
 mkdir -p ../repos/
 cp -ra "$path/ccpi/repos"/* ../repos/
diff --git a/build/pre-setup/requirements.txt b/build/pre-setup/requirements.txt
index 87e3dca..e7b8d3b 100644
--- a/build/pre-setup/requirements.txt
+++ b/build/pre-setup/requirements.txt
@@ -4,3 +4,4 @@ scipy
 matplotlib
 h5py
 pillow
+pycuda
diff --git a/build/setup/build/cmake.sh b/build/setup/build/cmake.sh
index 09beccf..1065907 100644
--- a/build/setup/build/cmake.sh
+++ b/build/setup/build/cmake.sh
@@ -4,6 +4,6 @@ set -o errexit
 
 pypath=$(python3 -c 'import site; print(site.getsitepackages()[0])')
 #rm CMakeCache.txt
-cmake -D "PYTHON_DEST_DIR:PATH=$pypath" .
+cmake -DCMAKE_INSTALL_PREFIX=/usr -D "PYTHON_DEST_DIR:PATH=$pypath" .
 make
 make install
diff --git a/build/setup/build/python.sh b/build/setup/build/python.sh
index de08a2c..3694a98 100644
--- a/build/setup/build/python.sh
+++ b/build/setup/build/python.sh
@@ -3,7 +3,7 @@
 set -o errexit
 
 (
-    cd Wrappers/Python
+    [ -d "Wrappers/Python" ] &&  cd "Wrappers/Python"
     python3 setup.py install
 )
 
diff --git a/build/setup/repos.sh b/build/setup/repos.sh
index 92c1a2f..0c7481a 100644
--- a/build/setup/repos.sh
+++ b/build/setup/repos.sh
@@ -20,6 +20,12 @@ repos_ccpi=( \
     "https://github.com/vais-ral/CCPi-Regularisation-Toolkit.git master ccpi.sh" \
 )
 
+repos_ufo=( \
+    "https://github.com/ufo-kit/ufo-core.git master cmake.sh" \
+    "https://github.com/ufo-kit/ufo-filters.git master cmake.sh" \
+    "https://github.com/ufo-kit/tofu master python.sh" \
+)
+
 repos_all=("${repos_astra[@]}" "${repos_ccpi[@]}")
 declare -n repos="repos_$repo_group"
 
diff --git a/build/update.sh b/build/update.sh
index 0201a4d..a3dc9ff 100644
--- a/build/update.sh
+++ b/build/update.sh
@@ -2,15 +2,18 @@
 
 set -o errexit
 
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ufo|ccpi|ccpi-remote|tomo|tomo-remote|imageX>"; exit 1; }
+
 #container=cuda-working-container-15
-container=$(buildah from localhost/ccpi)
+container=$(buildah from localhost/${image})
 
 #buildah run $container bash
 #exit
 
-buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data  $container
+buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
 buildah copy $container setup /root/setup
 buildah run $container bash /root/setup/repos.sh "all" "/root/setup"
 
 #buildah commit --format docker $container ccpi:latest
-echo "buildah commit --format docker $container ccpi:latest"
+echo "buildah commit --format docker $container ${image}:latest"