Update containers to ubuntu 22.04, gcc-12; include perf/gdb; add python libraries for opencv, machine learning, etc.

10 files changed, 74 insertions, 26 deletions

diff --git a/build/buildah-astra.sh b/build/buildah-astra.sh
index 1282e28..6f6e7b1 100644
--- a/build/buildah-astra.sh
+++ b/build/buildah-astra.sh
@@ -2,9 +2,12 @@
 
 set -o errexit
 
-container=$(buildah from pycuda:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from pycuda:$tag)
 
 buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
 buildah run $container bash /root/setup/repos.sh "astra"
 
-buildah commit --format docker $container astra:latest
+buildah commit --format docker $container astra:$tag
diff --git a/build/buildah-base.sh b/build/buildah-base.sh
index 319ed6b..fcb3200 100644
--- a/build/buildah-base.sh
+++ b/build/buildah-base.sh
@@ -2,8 +2,13 @@
 
 set -o errexit
 
+tag=latest
+[ -n "$1" ] && tag="$1"
+
 #container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
-container=$(buildah from nvidia/cuda:11.3.0-devel-ubuntu20.04)
+[ $tag == "20.04" ] && container=$(buildah from nvidia/cuda:11.3.0-devel-ubuntu20.04)
+[ $tag == "22.04" ] && container=$(buildah from nvidia/cuda:11.7.1-devel-ubuntu22.04)
+[ -n "$container" ] && container=$(buildah from nvidia/cuda:11.7.1-devel-ubuntu$tag)
 
 buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
 buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
@@ -24,12 +29,23 @@ buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/d
 buildah run $container sh -c 'echo "systemd-timesync:x:107:" >> /etc/group'
 buildah run $container sh -c 'echo "systemd-timesync:x:107:107:systemd-timesync:/dev/null:/sbin/nologin" >> /etc/passwd'
 
+# https://stackoverflow.com/questions/71941032/why-i-cannot-run-apt-update-inside-a-fresh-ubuntu22-04
+buildah run $container sh -c 'sed -i -e "s/^APT/# APT/" -e "s/^DPkg/# DPkg/" /etc/apt/apt.conf.d/docker-clean'
 
-buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
+buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc rar ca-certificates git openssh-server'
 buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
 buildah run $container sh -c 'apt-get install -y cmake make autoconf automake libtool gcc g++ libboost-all-dev'
+
+[ $tag == "22.04" ] && buildah run $container sh -c 'apt-get install -y gcc-12 g++-12'					# First gcc version supporting _Float16 data type
+[ $tag != "20.04" ] && buildah run $container sh -c 'apt-get install -y screen perf-tools-unstable gdb htop'		# Debugging/Profilling tools
 buildah run $container sh -c 'apt-get install -y ca-certificates pkg-config gobject-introspection liblapack-dev libjpeg-dev libtiff-dev libglib2.0-dev libjson-glib-dev libopenmpi-dev libhdf5-dev libclfft-dev libgsl-dev libgirepository1.0-dev fftw3-dev zlib1g-dev'
-buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy  python-gobject tifffile'
+
+# Problems with python3-tifffile (this should be ran after installing pkgconfig/gobject stuff)
+buildah run $container sh -c 'sed -i -e "s/^post-invoke/# post-invoke/" /etc/dpkg/dpkg.cfg.d/pkg-config-hook-config'
+
+buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy  python3-gi python3-gi-cairo python3-tifffile'
+
+[ $tag != "20.04" ] && buildah run $container sh -c 'apt-get install -y python3-astropy python3-opencv python3-willow python3-pypillowfight python3-scipy python3-sklearn python3-skorch python3-tomopy'	# Not needed for ccpi/ufo, but rather additional image-processing code for python
 buildah run $container sh -c 'apt-get install -y ocl-icd-opencl-dev clinfo'
 buildah run $container sh -c 'apt-get install -y jupyter-notebook'
 
@@ -39,8 +55,10 @@ buildah run $container sh -c 'update-alternatives --install /usr/bin/python pyth
 buildah run $container mkdir -p /etc/OpenCL/vendors
 buildah run $container sh -c 'echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd'
 
+[ $tag != "20.04" ] && buildah run $container sh -c 'rm /usr/bin/perf; ln -s /usr/lib/linux-tools/*/perf /usr/bin/perf'
+
 buildah copy $container pre-setup /root/pre-setup
 buildah run $container pip3 install -r /root/pre-setup/requirements.txt
 
 buildah copy $container setup /root/setup
-buildah commit --format docker $container pycuda:latest
+buildah commit --format docker $container pycuda:$tag
diff --git a/build/buildah-ccpi.sh b/build/buildah-ccpi.sh
index 661e0ef..6912476 100644
--- a/build/buildah-ccpi.sh
+++ b/build/buildah-ccpi.sh
@@ -2,9 +2,12 @@
 
 set -o errexit
 
-container=$(buildah from astra:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from astra:$tag)
 
 buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=21.0.0 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
 buildah run $container bash /root/setup/repos.sh "ccpi"
 
-buildah commit --format docker $container ccpi:latest
+buildah commit --format docker $container ccpi:$tag
diff --git a/build/buildah-remote.sh b/build/buildah-remote.sh
index 5e9d038..7cb0eeb 100644
--- a/build/buildah-remote.sh
+++ b/build/buildah-remote.sh
@@ -3,9 +3,12 @@
 set -o errexit
 
 image="$1"
-[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX> [tag]"; exit 1; }
 
-container=$(buildah from ${image}:latest)
+tag=latest
+[ -n "$2" ] && tag="$2"
+
+container=$(buildah from ${image}:${tag})
 buildah config --port 2222/tcp --port 8888/tcp $container
 
 buildah run $container sh -c 'apt-get install -y supervisor'
@@ -36,6 +39,6 @@ exitcodes=0
 EOF
 '
 
-buildah commit --format docker $container ${image}-remote:latest
+buildah commit --format docker $container ${image}-remote:${tag}
 
-echo "$container build from ${image} and saved as ${image}-remote"
+echo "$container build from ${image}:${tag} and saved as ${image}-remote:${tag}"
diff --git a/build/buildah-tomo.sh b/build/buildah-tomo.sh
index ed9287e..486f182 100644
--- a/build/buildah-tomo.sh
+++ b/build/buildah-tomo.sh
@@ -2,10 +2,13 @@
 
 set -o errexit
 
-container=$(buildah from ccpi:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from ccpi:$tag)
 
 buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
 
 buildah run $container bash /root/setup/repos.sh "ufo"
 
-buildah commit --format docker $container tomo:latest
+buildah commit --format docker $container tomo:$tag
diff --git a/build/buildah-ufo.sh b/build/buildah-ufo.sh
index ee8e328..bdc6331 100644
--- a/build/buildah-ufo.sh
+++ b/build/buildah-ufo.sh
@@ -2,10 +2,13 @@
 
 set -o errexit
 
-container=$(buildah from pycuda:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from pycuda:$tag)
 
 buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
 
 buildah run $container bash /root/setup/repos.sh "ufo"
 
-buildah commit --format docker $container ufo:latest
+buildah commit --format docker $container ufo:$tag
diff --git a/build/buildah.sh b/build/buildah.sh
index 8e84457..2ba99e5 100644
--- a/build/buildah.sh
+++ b/build/buildah.sh
@@ -1,7 +1,9 @@
 set -o errexit
 
-bash buildah-base.sh
-bash buildah-astra.sh
-bash buildah-ccpi.sh
-bash buildah-tomo.sh
-bash buildah-remote.sh tomo
+tag=22.04
+
+bash buildah-base.sh $tag
+bash buildah-astra.sh $tag
+bash buildah-ccpi.sh $tag
+bash buildah-tomo.sh $tag
+bash buildah-remote.sh tomo $tag
diff --git a/build/update.sh b/build/update.sh
index a3dc9ff..6a8ac7a 100644
--- a/build/update.sh
+++ b/build/update.sh
@@ -3,10 +3,14 @@
 set -o errexit
 
 image="$1"
-[ -n "$image" ] || { echo "Usage: $0 <astra|ufo|ccpi|ccpi-remote|tomo|tomo-remote|imageX>"; exit 1; }
+[ -n "$image" ] || { echo "Usage: $0 <astra|ufo|ccpi|ccpi-remote|tomo|tomo-remote|imageX> [tag]"; exit 1; }
+
+tag=latest
+[ -n "$2" ] && tag="$2"
+
 
 #container=cuda-working-container-15
-container=$(buildah from localhost/${image})
+container=$(buildah from localhost/${image}:${tag})
 
 #buildah run $container bash
 #exit
@@ -15,5 +19,5 @@ buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PA
 buildah copy $container setup /root/setup
 buildah run $container bash /root/setup/repos.sh "all" "/root/setup"
 
-#buildah commit --format docker $container ccpi:latest
-echo "buildah commit --format docker $container ${image}:latest"
+#buildah commit --format docker $container ccpi:${tag}
+echo "buildah commit --format docker $container ${image}:${tag}"
diff --git a/patches/ccpi-readme.txt b/patches/ccpi-readme.txt
new file mode 100644
index 0000000..4399a22
--- /dev/null
+++ b/patches/ccpi-readme.txt
@@ -0,0 +1,7 @@
+ - There is also extra memory consumed in FISTA implementation, here:
+	/ccpi/repos/CCPi-Framework/Wrappers/Python/cil/optimisation/algorithms
+    We might reduce further 1-2 volumes if rewritten in C and temporarily variables
+    are properly eleminated. We further might introduce simplified memory manager
+    which will allow to give up further temporarily in FISTA and allocated them in
+    regularize and, then, vice-versa.
+    
diff --git a/run/config.sh b/run/config.sh
index bc1d6c5..d2b01dd 100644
--- a/run/config.sh
+++ b/run/config.sh
@@ -6,7 +6,9 @@ web_port="${port}80"
 
 
 image="$1"
-tag="latest"
+#tag="latest"
+tag="22.04"
+
 [ -n "$image" ] || image="localhost/tomo-remote:$tag"		# tomo-remote:latest tomo-remote:devel tomo-remote:evelina (also ccpi, ufo, tomo)
 
 options="--cap-add SYS_ADMIN --cap-add=SYS_PTRACE --hooks-dir /usr/share/containers/oci/hooks.d"