Initial import

22 files changed, 442 insertions, 0 deletions

diff --git a/.env b/.env
new file mode 100644
index 0000000..06783cb
--- /dev/null
+++ b/.env
@@ -0,0 +1 @@
+BIOMEDISA_HOST_SRC_PATH=/home/csa/biomedisa
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..60e89bb
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,43 @@
+# It seems start/stop does not respect '-p' option. We either need to avoid using it or skip -p
+
+.PHONY: run cpu start stop rebuild rebuild-cpu bootstrap volumes attach bash clean
+
+run: 
+	docker-compose -p biomedisa up
+
+cpu: 
+	docker-compose -p biomedisa -f docker-compose.yml -f docker-compose.cpu.yml up
+
+start:
+	docker-compose -p biomedisa up -d
+#	docker-compose -p biomedisa up --no-start
+#	docker-compose -p biomedisa start
+
+stop:
+	docker-compose -p biomedisa down
+#	docker-compose -p biomedisa stop
+
+rebuild:
+	docker-compose -p biomedisa build
+
+rebuild-cpu: 
+	docker-compose -p biomedisa -f docker-compose.yml -f docker-compose.cpu.yml build
+
+bootstrap:
+	docker-compose -p biomedisa run --rm django /run-bootstrap.sh
+
+#volumes:
+#	for name in /var/lib/docker/volumes/biomedisa_*; do ln -s $name/_data/ `basename $name | cut -d '_' -f 2`; done
+
+attach:
+	docker attach `docker ps | grep "biomedisa_biomedisa " | cut -f 1 -d ' '`
+
+bash:
+	docker exec -it `docker ps | grep "biomedisa_biomedisa " | cut -f 1 -d ' '` bash
+
+clean: stop
+	c=`docker ps -a | grep "biomedisa_mysql " | cut -f 1 -d ' '`; \
+	if [ -n "$$c" ]; then docker rm -f "$$c"; fi
+	v=`docker volume ls | grep "biomedisa_mysql"`; \
+	if [ -n "$$v" ]; then docker volume rm biomedisa_mysql; fi
+	make bootstrap
diff --git a/biomedisa/Dockerfile b/biomedisa/Dockerfile
new file mode 100644
index 0000000..6666be7
--- /dev/null
+++ b/biomedisa/Dockerfile
@@ -0,0 +1,49 @@
+#FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
+FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+MAINTAINER Suren A. Chilingaryan <csa@suren.me>
+
+# Install Devel & Python
+RUN apt-get update -qq \
+ && apt-get install --no-install-recommends -y \
+    git pkg-config netcat supervisor \
+    telnet iputils-ping dnsutils \
+    python python-dev python-pip python-setuptools python-virtualenv python-wheel python-numpy python-scipy python-h5py python-yaml python-pydot python-matplotlib libopenblas-base
+
+# Install Keras + Tensor Flow
+ARG TENSORFLOW_VERSION=1.12.0
+ARG TENSORFLOW_DEVICE=gpu
+ARG TENSORFLOW_APPEND=_gpu
+
+ARG KERAS_VERSION=2.2.4
+ENV KERAS_BACKEND=tensorflow
+
+RUN pip --no-cache-dir install \
+    https://storage.googleapis.com/tensorflow/linux/${TENSORFLOW_DEVICE}/tensorflow${TENSORFLOW_APPEND}-${TENSORFLOW_VERSION}-cp27-none-linux_x86_64.whl
+
+RUN pip --no-cache-dir install --no-dependencies \
+    git+https://github.com/fchollet/keras.git@${KERAS_VERSION}
+
+ADD tests/ /root/tests
+
+# install dependencies from debian packages
+RUN apt-get install --no-install-recommends -y \
+    mc \
+    libsm6 libxrender-dev \
+    openmpi-bin openmpi-doc libopenmpi-dev \
+    python-gdcm python-mysqldb
+
+# upgrade pip
+RUN pip --no-cache-dir install --upgrade pip
+
+# install dependencies from python packages
+RUN pip --no-cache-dir install \
+    setuptools==39.1.0 numpy==1.14.5 tifffile==0.14.0 SimpleParse==2.1.1 redis==2.10.6 rq==0.12 \
+    scipy weave scikit-image opencv-python h5py pydicom Pillow django mpi4py pycuda
+
+COPY supervisor/*.conf /etc/
+COPY *-entrypoint.sh /
+COPY run-*.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+ENV BIOMEDISA_DEVICE=${TENSORFLOW_DEVICE}
+CMD /usr/bin/supervisord -c /etc/supervisord-${BIOMEDISA_DEVICE}.conf
diff --git a/biomedisa/Makefile b/biomedisa/Makefile
new file mode 100644
index 0000000..c14b89e
--- /dev/null
+++ b/biomedisa/Makefile
@@ -0,0 +1,48 @@
+all: build
+#install: push
+
+.PHONY: rebuild build push start stop restart bash
+
+build: Dockerfile
+	docker build --tag chsa/biomedisa:latest .
+
+rebuild: Dockerfile
+	docker build --no-cache --tag chsa/biomedisa:latest .
+
+#push: build
+#	docker push chsa/biomedisa:latest
+
+start: build
+	@if [ `docker ps | grep adei-mysql | wc -l` -eq 0 ]; then \
+	    if [ `docker ps -a | grep adei-mysql | wc -l` -eq 0 ]; then \
+		docker run --name adei-mysql -p 53306:3306/tcp  -e MYSQL_ROOT_PASSWORD="adei" -e MYSQL_DATABASE="adei" -e MYSQL_USER="adei" -e MYSQL_PASSWORD="adei" -d mysql:5.7 ;\
+	    else \
+		docker start adei-mysql ;\
+	    fi ;\
+	fi ;\
+	if [ `docker ps | grep chsa/adei | wc -l` -eq 0 ]; then \
+	    if [ `docker ps -a | grep adei-full | wc -l` -gt 0 ]; then \
+		echo "Removing the stalled copy..." ;\
+		docker rm adei-full ;\
+	    fi ;\
+	    docker run -p 8000:80 --name adei-full --link adei-mysql:adei-mysql -e ADEI_PORTS=8000 -t -d chsa/adei:latest ;\
+	else \
+	    echo "Already running..." ;\
+	fi
+
+stop:
+	@if [ `docker ps | grep adei-full | wc -l` -gt 0 ]; then \
+	    docker stop adei-full ;\
+	    docker rm adei-full ;\
+	fi
+
+stop_mysql: stop
+	docker stop adei-mysql
+
+restart:
+	make stop
+	make start
+
+bash: build
+	docker run --runtime=nvidia -it chsa/biomedisa /bin/bash
+#	docker exec -it chsa/biomedisa /bin/bash
diff --git a/biomedisa/docker-entrypoint.sh b/biomedisa/docker-entrypoint.sh
new file mode 100755
index 0000000..a0f9cba
--- /dev/null
+++ b/biomedisa/docker-entrypoint.sh
@@ -0,0 +1,16 @@
+#! /bin/bash
+
+set -e
+
+export PYTHONPATH=/biomedisa:${PYTHONPATH}
+export DJANGO_SETTINGS_MODULE=biomedisa.settings
+
+if ! whoami &> /dev/null; then
+  if [ -w /etc/passwd ]; then
+      echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd
+  fi
+fi
+
+cd "/biomedisa"
+echo "Running: $@"
+exec "$@"
diff --git a/biomedisa/run-queues.sh b/biomedisa/run-queues.sh
new file mode 100755
index 0000000..a531b3b
--- /dev/null
+++ b/biomedisa/run-queues.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# start background workers
+rq worker -u http://redis rq worker first_queue &
+rq worker -u http://redis rq worker slices &
+rq worker -u http://redis rq worker acwe &
+rq worker -u http://redis rq worker cleanup &
+rq worker -u http://redis rq worker share_notification &
+rq worker -u http://redis rq worker stop_job &
diff --git a/biomedisa/supervisor/supervisord-cpu.conf b/biomedisa/supervisor/supervisord-cpu.conf
new file mode 100644
index 0000000..c942d7d
--- /dev/null
+++ b/biomedisa/supervisor/supervisord-cpu.conf
@@ -0,0 +1,27 @@
+[supervisord]
+nodaemon=true
+
+[program:slices]
+command=rq worker -u http://redis rq worker slices
+autorestart=true
+exitcodes=0
+
+[program:acwe]
+command=rq worker -u http://redis rq worker acwe
+autorestart=true
+exitcodes=0
+
+[program:cleanup]
+command=rq worker -u http://redis rq worker cleanup
+autorestart=true
+exitcodes=0
+
+[program:share_notification]
+command=rq worker -u http://redis rq worker share_notification
+autorestart=true
+exitcodes=0
+
+[program:stop_job]
+command=rq worker -u http://redis rq worker stop_job
+autorestart=true
+exitcodes=0
diff --git a/biomedisa/supervisor/supervisord-gpu.conf b/biomedisa/supervisor/supervisord-gpu.conf
new file mode 100644
index 0000000..94323d5
--- /dev/null
+++ b/biomedisa/supervisor/supervisord-gpu.conf
@@ -0,0 +1,32 @@
+[supervisord]
+nodaemon=true
+
+[program:first]
+command=rq worker -u http://redis rq worker first_queue
+autorestart=true
+exitcodes=0
+
+[program:slices]
+command=rq worker -u http://redis rq worker slices
+autorestart=true
+exitcodes=0
+
+[program:acwe]
+command=rq worker -u http://redis rq worker acwe
+autorestart=true
+exitcodes=0
+
+[program:cleanup]
+command=rq worker -u http://redis rq worker cleanup
+autorestart=true
+exitcodes=0
+
+[program:share_notification]
+command=rq worker -u http://redis rq worker share_notification
+autorestart=true
+exitcodes=0
+
+[program:stop_job]
+command=rq worker -u http://redis rq worker stop_job
+autorestart=true
+exitcodes=0
diff --git a/biomedisa/tests/test.sh b/biomedisa/tests/test.sh
new file mode 100755
index 0000000..cf16ce1
--- /dev/null
+++ b/biomedisa/tests/test.sh
@@ -0,0 +1,4 @@
+#! /bin/bash
+
+python -c 'import tensorflow; print(tensorflow.__version__)'
+python -c 'import keras; print(keras.__version__)'
diff --git a/django/Dockerfile b/django/Dockerfile
new file mode 100644
index 0000000..0f1e044
--- /dev/null
+++ b/django/Dockerfile
@@ -0,0 +1,23 @@
+FROM ubuntu:16.04
+MAINTAINER Suren A. Chilingaryan <csa@suren.me>
+
+# Install Devel & Python
+RUN apt-get update -qq \
+ && apt-get install --no-install-recommends -y \
+    gcc git pkg-config netcat \
+    telnet iputils-ping dnsutils \
+    python python-dev python-pip python-setuptools python-virtualenv python-wheel python-numpy python-scipy python-h5py python-yaml python-pydot python-matplotlib libopenblas-base \
+    mysql-client libmysqlclient-dev
+
+# upgrade pip
+RUN pip --no-cache-dir install --upgrade pip
+
+# install dependencies from python packages
+RUN pip --no-cache-dir install \
+    setuptools==39.1.0 numpy==1.14.5 tifffile==0.14.0 SimpleParse==2.1.1 redis==2.10.6 rq==0.12 \
+    scipy weave scikit-image opencv-python h5py pydicom Pillow django MySQL
+
+COPY *-entrypoint.sh /
+COPY run*.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["/run.sh"]
diff --git a/django/Makefile b/django/Makefile
new file mode 100644
index 0000000..78b94ac
--- /dev/null
+++ b/django/Makefile
@@ -0,0 +1,16 @@
+all: build
+#install: push
+
+.PHONY: rebuild build push start stop restart bash
+
+build: Dockerfile
+	docker build --tag chsa/biomedisa-django:latest .
+
+rebuild: Dockerfile
+	docker build --no-cache --tag chsa/biomedisa-django:latest .
+
+#push: build
+#	docker push chsa/biomedisa:latest
+
+bash: build
+	docker run -it chsa/biomedisa-django /bin/bash
diff --git a/django/docker-entrypoint.sh b/django/docker-entrypoint.sh
new file mode 100755
index 0000000..3a132a3
--- /dev/null
+++ b/django/docker-entrypoint.sh
@@ -0,0 +1,21 @@
+#! /bin/bash
+
+set -e
+
+export PYTHONPATH=/biomedisa:${PYTHONPATH}
+export DJANGO_SETTINGS_MODULE=biomedisa.settings
+
+if ! whoami &> /dev/null; then
+  if [ -w /etc/passwd ]; then
+      echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd
+  fi
+fi
+
+until nc -z -v -w30 $MYSQL_HOST 3306; do
+  echo "Waiting for database connection..."
+  sleep 1
+done
+
+cd /biomedisa
+echo "Running: $@"
+exec "$@"
diff --git a/django/run-bootstrap.sh b/django/run-bootstrap.sh
new file mode 100755
index 0000000..c57325b
--- /dev/null
+++ b/django/run-bootstrap.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+python manage.py makemigrations
+python manage.py migrate
+python manage.py createsuperuser
diff --git a/django/run.sh b/django/run.sh
new file mode 100755
index 0000000..f473012
--- /dev/null
+++ b/django/run.sh
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+python manage.py runserver 0.0.0.0:8000
diff --git a/docker-compose.cpu.yml b/docker-compose.cpu.yml
new file mode 100644
index 0000000..99d530f
--- /dev/null
+++ b/docker-compose.cpu.yml
@@ -0,0 +1,15 @@
+version: '2.4'
+
+services:
+    redis:
+        ports:
+            - 127.0.0.1:6379:6379
+    mysql:
+        ports:
+            - 127.0.0.1:3306:3306
+    biomedisa:
+        build:
+            context: ./biomedisa
+            args:
+                TENSORFLOW_DEVICE: cpu
+                TENSORFLOW_APPEND:
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..1351b2a
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,66 @@
+version: '2.4'
+
+services:
+    mysql:
+        build: ./mysql
+        restart: always
+        volumes:
+            - mysql:/var/lib/mysql
+        environment:
+            MYSQL_ROOT_PASSWORD: biomedisa
+            MYSQL_DATABASE: biomedisa_database
+            MYSQL_USER: biomedisa
+            MYSQL_PASSWORD: biomedisa
+    redis:
+        image: redis:2
+        restart: always
+    django:
+        build: ./django
+        restart: always
+        tty: true
+        stdin_open: true
+        depends_on:
+            - mysql
+            - redis
+        links:
+            - mysql:mysql
+            - redis:redis
+        volumes:
+            - src:/biomedisa
+            - src:${BIOMEDISA_HOST_SRC_PATH}
+        environment:
+            MYSQL_HOST: mysql
+            REDIS_HOST: redis
+            BIOMEDISA_SRC_PATH: /biomedisa
+        ports:
+            - 8000:8000
+    biomedisa:
+        build: ./biomedisa
+        runtime: nvidia
+        restart: always
+        tty: true
+        stdin_open: true
+        depends_on:
+            - mysql
+            - redis
+        links:
+            - mysql:mysql
+            - redis:redis
+        volumes:
+            - src:/biomedisa
+            - src:${BIOMEDISA_HOST_SRC_PATH}
+        environment:
+            MYSQL_HOST: mysql
+            REDIS_HOST: redis
+            BIOMEDISA_SRC_PATH: /biomedisa
+            NVIDIA_VISIBLE_DEVICES: all
+
+volumes:
+    src:
+        driver: local
+        driver_opts:
+            type: none
+            o: bind
+            device: ${BIOMEDISA_HOST_SRC_PATH}
+    mysql:
+#    media:
diff --git a/external/install.sh b/external/install.sh
new file mode 100644
index 0000000..3ad20f6
--- /dev/null
+++ b/external/install.sh
@@ -0,0 +1,22 @@
+#! /bin/bash
+
+yum install -y libtiff-devel
+yum install -y MySQL-python python-devel python2-pip python2-setuptools python2-virtualenv numpy scipy h5py python2-pyyaml python2-pydot python2-matplotlib mysql-connector-python openblas
+yum install -y openmpi openmpi-devel
+
+# upgrade pip
+pip --no-cache-dir install --upgrade pip
+
+# extra pre-requisites
+#pip --no-cache-dir install \
+#    scipy decorator
+
+# conflicting packages
+rpm -e scipy pyparsing python2-pydot python-matplotlib python2-matplotlib
+
+# install dependencies from python packages
+export CC=/usr/lib64/openmpi/bin/mpicc
+
+pip --no-cache-dir install \
+    setuptools==39.1.0 numpy==1.14.5 tifffile==0.14.0 SimpleParse==2.1.1 redis==2.10.6 rq==0.12 \
+    weave scikit-image opencv-python h5py pydicom Pillow django mpi4py pycuda
diff --git a/external/run.sh b/external/run.sh
new file mode 100755
index 0000000..94490d7
--- /dev/null
+++ b/external/run.sh
@@ -0,0 +1,14 @@
+path=$(cat ../.env | grep -oP 'BIOMEDISA_HOST_SRC_PATH=\K.*')
+(
+    cd "$path" || exit -1
+
+    export PYTHONPATH=$path:${PYTHONPATH}
+    export DJANGO_SETTINGS_MODULE=biomedisa.settings
+
+    export BIOMEDISA_SRC_PATH=$path
+    export MYSQL_HOST="127.0.0.1"			# localhost is treated as socket
+    export REDIS_HOST="localhost"
+    export PATH="$PATH:/usr/lib64/openmpi/bin/"
+
+    LD_PRELOAD=/usr/lib64/openmpi/lib/libmpi.so rq worker -u http://localhost rq worker first_queue
+)
diff --git a/external/tests/test.py b/external/tests/test.py
new file mode 100644
index 0000000..da9ff38
--- /dev/null
+++ b/external/tests/test.py
@@ -0,0 +1,12 @@
+from time import sleep
+from redis import Redis
+from rq import Queue
+
+from testmod import test
+
+q = Queue('slices', connection=Redis(host='localhost'))
+j = q.enqueue(test)
+print(j.result)
+while j.result == None:
+    sleep(1)
+    print(j.result)
diff --git a/external/tests/testmod.py b/external/tests/testmod.py
new file mode 100644
index 0000000..e833d21
--- /dev/null
+++ b/external/tests/testmod.py
@@ -0,0 +1,11 @@
+import subprocess 
+
+def test():
+#    proc = subprocess.Popen(['env'], stdout=subprocess.PIPE)
+    proc = subprocess.Popen(['nvidia-smi'], stdout=subprocess.PIPE)
+#    proc = subprocess.Popen(['nvidia-smi', '-q', '-i', '0'], stdout=subprocess.PIPE)
+#    proc = subprocess.Popen(['ls', '/usr/bin/'], stdout=subprocess.PIPE)
+#    proc = subprocess.Popen('ls', stdout=subprocess.PIPE)
+    return proc.stdout.read()
+
+
diff --git a/mysql/Dockerfile b/mysql/Dockerfile
new file mode 100644
index 0000000..8a89e26
--- /dev/null
+++ b/mysql/Dockerfile
@@ -0,0 +1,4 @@
+FROM mysql:5.7
+MAINTAINER Suren A. Chilingaryan <csa@suren.me>
+
+COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
diff --git a/mysql/docker-entrypoint-initdb.d/biomedisa.sql b/mysql/docker-entrypoint-initdb.d/biomedisa.sql
new file mode 100644
index 0000000..c437f0d
--- /dev/null
+++ b/mysql/docker-entrypoint-initdb.d/biomedisa.sql
@@ -0,0 +1 @@
+GRANT ALL ON `biomedisa_database`.* TO 'biomedisa'@'%';