From 0ce4e8d52fd491268a56c10dbb32fd5c996e2589 Mon Sep 17 00:00:00 2001
From: "Suren A. Chilingaryan" <csa@suren.me>
Date: Sat, 1 Feb 2020 13:07:46 +0100
Subject: Initial Python infrastructure to build more complex processing
pipelines and the corresponding changes in ROOF filters
---
.gitignore | 1 +
docs/hardware.txt | 5 +
docs/schemes/roof_graph.odg | Bin 0 -> 20566 bytes
docs/schemes/roof_graph.pdf | Bin 0 -> 29905 bytes
docs/todo.txt | 8 +-
src/CMakeLists.txt | 2 +-
src/meson.build | 4 +-
src/ufo-roof-build-task.c | 26 ++++-
src/ufo-roof-config.c | 14 ++-
src/ufo-roof-config.h | 10 +-
src/ufo-roof-filter-task.c | 249 ++++++++++++++++++++++++++++++++++++++++++++
src/ufo-roof-filter-task.h | 53 ++++++++++
src/ufo-roof-plane-task.c | 233 -----------------------------------------
src/ufo-roof-plane-task.h | 53 ----------
src/ufo-roof-read-file.c | 6 +-
src/ufo-roof-read-file.h | 2 +-
src/ufo-roof-read-task.c | 62 ++++++++++-
tests/config.sh | 2 +
tests/roof-net.sh | 19 ----
tests/roof-sim.sh | 14 ---
tests/roof-vma.sh | 1 +
tests/roof.json | 18 +++-
tests/roof.py | 81 ++------------
tests/roof.sh | 32 ++++++
tests/roof.yaml | 30 ++++--
tests/roof/__init__.py | 1 +
tests/roof/arguments.py | 32 ++++++
tests/roof/config.py | 67 ++++++++++++
tests/roof/defaults.py | 42 ++++++++
tests/roof/graph.py | 203 ++++++++++++++++++++++++++++++++++++
tests/roof/utils.py | 13 +++
31 files changed, 855 insertions(+), 428 deletions(-)
create mode 100644 docs/schemes/roof_graph.odg
create mode 100644 docs/schemes/roof_graph.pdf
create mode 100644 src/ufo-roof-filter-task.c
create mode 100644 src/ufo-roof-filter-task.h
delete mode 100644 src/ufo-roof-plane-task.c
delete mode 100644 src/ufo-roof-plane-task.h
delete mode 100755 tests/roof-net.sh
delete mode 100755 tests/roof-sim.sh
create mode 120000 tests/roof-vma.sh
create mode 100755 tests/roof.sh
create mode 100644 tests/roof/__init__.py
create mode 100644 tests/roof/arguments.py
create mode 100644 tests/roof/config.py
create mode 100644 tests/roof/defaults.py
create mode 100644 tests/roof/graph.py
create mode 100644 tests/roof/utils.py
diff --git a/.gitignore b/.gitignore
index c8831d9..79897a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ Makefile
/tests/venv
/build/
/_build/
+__pycache__
diff --git a/docs/hardware.txt b/docs/hardware.txt
index a293887..50c3a0c 100644
--- a/docs/hardware.txt
+++ b/docs/hardware.txt
@@ -4,3 +4,8 @@
* With 46 packets, however, we can't split a full rotation in a whole number of packets.
So, we need to find maximal number m, so that
(m <= n) and (samples_per_rotation % m = 0) i.e. 40
+
+
+Questions
+=========
+ - Do we need to compute 'flats' and 'darks' for each plane separately? Or just one set will work for all?
diff --git a/docs/schemes/roof_graph.odg b/docs/schemes/roof_graph.odg
new file mode 100644
index 0000000..1fb4643
Binary files /dev/null and b/docs/schemes/roof_graph.odg differ
diff --git a/docs/schemes/roof_graph.pdf b/docs/schemes/roof_graph.pdf
new file mode 100644
index 0000000..8ba6b7f
Binary files /dev/null and b/docs/schemes/roof_graph.pdf differ
diff --git a/docs/todo.txt b/docs/todo.txt
index 8497a69..88b518c 100644
--- a/docs/todo.txt
+++ b/docs/todo.txt
@@ -3,13 +3,17 @@ Main
+ Add plane/frame-number/broken metadata in the UFO buffers. Check propagation trough standard ufo filters. [ propogates trough processors, but not reductors ]
+ Plane selector filter
- Handle packets with data from multiple datasets
- - Filter to ingest zero-padded broken frames.
- Try UFO 'flat-field' correction filter
- Cone-beam to parallel-beam resampling ?
- Full reconstruction chain
- Try UFO visualization filter
- - "Reconstructed data storage" and "Visualization + raw data storage" modes
+ - "Reconstructed data storage" and "Visualization + raw data storage" modes. Implement stand-alone 'roof-converter' filter.
+If necesary
+===========
+ - Task 'roof-ingest-missing' to ingest zero-padded broken frames (and include get_writer())
+ - Add ROOF metadata (plane, etc.) if reading from sinograms
+
Optional
========
- Try online compression of the ROOF data @Gregoire
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0967ffb..116b400 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 2.6)
set(ufofilter_SRCS
ufo-roof-read-task.c
ufo-roof-build-task.c
- ufo-roof-plane-task.c
+ ufo-roof-filter-task.c
)
set(common_SRCS
diff --git a/src/meson.build b/src/meson.build
index 8a03a57..8f8930a 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -1,7 +1,7 @@
plugins = [
'roof-read',
'roof-build',
- 'roof-plane',
+ 'roof-filter',
]
roof_common_src = [
@@ -16,7 +16,7 @@ roof_plugin_src = {
'roof-build': [
'ufo-roof-buffer.c',
],
- 'roof-plane': [
+ 'roof-filter': [
],
}
diff --git a/src/ufo-roof-build-task.c b/src/ufo-roof-build-task.c
index 93cb133..9d55d38 100644
--- a/src/ufo-roof-build-task.c
+++ b/src/ufo-roof-build-task.c
@@ -44,6 +44,7 @@ struct _UfoRoofBuildTaskPrivate {
BuildType build; // What dataset do we build: ROOF sinogram or raw network data
guint number; // Number of datasets to read
gboolean stop; // Stop flag
+ gboolean simulate; // Indicates if we are running in network or simulation modes
guint announced; // For debugging
@@ -71,6 +72,7 @@ static GEnumValue build_values[] = {
enum {
PROP_0,
PROP_STOP,
+ PROP_SIMULATE,
PROP_NUMBER,
PROP_BUILD,
PROP_CONFIG,
@@ -97,7 +99,7 @@ ufo_roof_build_task_setup (UfoTask *task,
if (!priv->config)
roof_setup_error(error, "ROOF configuration is not specified");
- priv->cfg = ufo_roof_config_new(priv->config, &gerr);
+ priv->cfg = ufo_roof_config_new(priv->config, priv->simulate?UFO_ROOF_CONFIG_SIMULATION:UFO_ROOF_CONFIG_DEFAULT, &gerr);
if (!priv->cfg)
roof_propagate_error(error, gerr, "roof-build-setup: ");
@@ -281,15 +283,23 @@ ufo_roof_build_task_generate (UfoTask *task,
case 32:
ufo_buffer_convert(output, UFO_BUFFER_DEPTH_32U);
break;
+ default:
+ printf("Usupported bit-depth %u\n", cfg->bit_depth);
}
}
// Metadata: plane and sequential number within the plane
g_value_init (&ival, G_TYPE_UINT);
g_value_init (&lval, G_TYPE_ULONG);
+ if (priv->build != BUILD_UFO) {
+ g_value_set_uint (&ival, cfg->bit_depth);
+ ufo_buffer_set_metadata (output, "bpp", &ival);
+ }
g_value_set_uint (&ival, 1 + seqid % cfg->n_planes);
ufo_buffer_set_metadata (output, "plane", &ival);
g_value_set_ulong (&lval, seqid / cfg->n_planes);
+ ufo_buffer_set_metadata (output, "plane_id", &lval);
+ g_value_set_ulong (&lval, seqid);
ufo_buffer_set_metadata (output, "seqid", &lval);
g_value_unset(&lval);
g_value_unset(&ival);
@@ -325,6 +335,9 @@ ufo_roof_build_task_set_property (GObject *object,
case PROP_STOP:
priv->stop = g_value_get_boolean (value);
break;
+ case PROP_SIMULATE:
+ priv->simulate = g_value_get_boolean (value);
+ break;
case PROP_NUMBER:
priv->number = g_value_get_uint (value);
break;
@@ -356,6 +369,9 @@ ufo_roof_build_task_get_property (GObject *object,
case PROP_STOP:
g_value_set_boolean (value, priv->stop);
break;
+ case PROP_SIMULATE:
+ g_value_set_boolean (value, priv->simulate);
+ break;
case PROP_NUMBER:
g_value_set_uint (value, priv->number);
break;
@@ -403,6 +419,14 @@ ufo_roof_build_task_class_init (UfoRoofBuildTaskClass *klass)
FALSE,
G_PARAM_READWRITE);
+
+ properties[PROP_SIMULATE] =
+ g_param_spec_boolean ("simulate",
+ "Simulation mode",
+ "Read data from the specified files instead of network",
+ FALSE,
+ G_PARAM_READWRITE);
+
properties[PROP_NUMBER] =
g_param_spec_uint("number",
"Number of datasets to receive",
diff --git a/src/ufo-roof-config.c b/src/ufo-roof-config.c
index 4788a2a..17f4b30 100644
--- a/src/ufo-roof-config.c
+++ b/src/ufo-roof-config.c
@@ -43,9 +43,6 @@ void ufo_roof_config_free(UfoRoofConfig *cfg) {
if (cfg) {
UfoRoofConfigPrivate *priv = (UfoRoofConfigPrivate*)cfg;
- if (cfg->path)
- g_free(cfg->path);
-
if (priv->parser)
g_object_unref (priv->parser);
@@ -53,7 +50,7 @@ void ufo_roof_config_free(UfoRoofConfig *cfg) {
}
}
-UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
+UfoRoofConfig *ufo_roof_config_new(const char *config, UfoRoofConfigFlags flags, GError **error) {
UfoRoofConfigPrivate *priv;
UfoRoofConfig *cfg;
@@ -66,6 +63,7 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
JsonObject *performance = NULL;
JsonObject *simulation = NULL;
JsonObject *reconstruction = NULL;
+ JsonObject *data = NULL;
GError *gerr = NULL;
@@ -97,7 +95,6 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
cfg->dataset_size = 0;
cfg->buffer_size = 2;
cfg->drop_buffers = 0;
- cfg->path = NULL;
// Read configuration
@@ -118,8 +115,11 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
roof_config_node_get(optics, root, object, "optics");
roof_config_node_get(network, root, object, "network");
roof_config_node_get(reconstruction, root, object, "reconstruction");
- roof_config_node_get(simulation, root, object, "simulation");
roof_config_node_get(performance, root, object, "performance");
+ roof_config_node_get(data, root, object, "data");
+
+ if (flags&UFO_ROOF_CONFIG_SIMULATION)
+ roof_config_node_get(simulation, root, object, "simulation");
}
if (hardware) {
@@ -183,8 +183,6 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
}
if (simulation) {
- roof_config_node_get_string(cfg->path, simulation, "path");
- roof_config_node_get(cfg->first_file_number, simulation, int, "first_file_number");
roof_config_node_get(cfg->header_size, simulation, int, "header_size");
if (!cfg->payload_size)
diff --git a/src/ufo-roof-config.h b/src/ufo-roof-config.h
index b6ee748..34bef0b 100644
--- a/src/ufo-roof-config.h
+++ b/src/ufo-roof-config.h
@@ -25,8 +25,6 @@ typedef struct {
// Network Server / Reader
- gchar *path; // Location of data files for simmulation purposes (i.e. reading a sequence of files instead listening on the corresponding ports)
- guint first_file_number; // Indicates if the numbering of files starts at 0 or 1
gchar *protocol; // Protocols: tcp, udp, tcp6, udp6, ...
guint port; // First port
guint n_streams; // Number of independent data streams (expected on sequential ports), by default equal to number of ROOF modules
@@ -53,7 +51,13 @@ typedef struct {
} UfoRoofConfig;
-UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error);
+typedef enum {
+ UFO_ROOF_CONFIG_DEFAULT = 0,
+ UFO_ROOF_CONFIG_SIMULATION = 1
+} UfoRoofConfigFlags;
+
+
+UfoRoofConfig *ufo_roof_config_new(const char *config, UfoRoofConfigFlags flags, GError **error);
void ufo_roof_config_free(UfoRoofConfig *cfg);
#endif /* __UFO_ROOF_CONFIG_H */
diff --git a/src/ufo-roof-filter-task.c b/src/ufo-roof-filter-task.c
new file mode 100644
index 0000000..01bc742
--- /dev/null
+++ b/src/ufo-roof-filter-task.c
@@ -0,0 +1,249 @@
+/*
+ * Copyright (C) 2011-2015 Karlsruhe Institute of Technology
+ *
+ * This file is part of Ufo.
+ *
+ * This library is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation, either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#include "ufo-roof-filter-task.h"
+
+
+struct _UfoRoofFilterTaskPrivate {
+ gboolean ready; // Indicates if data is ready for generation
+ gboolean block; // Block output alltogether
+ guint plane; // Selected plane (0 - pass everything trough)
+ gdouble fps; // Limit maximum frame rate
+};
+
+static void ufo_task_interface_init (UfoTaskIface *iface);
+
+G_DEFINE_TYPE_WITH_CODE (UfoRoofFilterTask, ufo_roof_filter_task, UFO_TYPE_TASK_NODE,
+ G_IMPLEMENT_INTERFACE (UFO_TYPE_TASK,
+ ufo_task_interface_init))
+
+#define UFO_ROOF_FILTER_TASK_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE((obj), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTaskPrivate))
+
+enum {
+ PROP_0,
+ PROP_PLANE,
+ PROP_FPS,
+ PROP_BLOCK,
+ N_PROPERTIES
+};
+
+static GParamSpec *properties[N_PROPERTIES] = { NULL, };
+
+UfoNode *
+ufo_roof_filter_task_new (void)
+{
+ return UFO_NODE (g_object_new (UFO_TYPE_ROOF_FILTER_TASK, NULL));
+}
+
+static void
+ufo_roof_filter_task_setup (UfoTask *task,
+ UfoResources *resources,
+ GError **error)
+{
+}
+
+static void
+ufo_roof_filter_task_get_requisition (UfoTask *task,
+ UfoBuffer **inputs,
+ UfoRequisition *requisition,
+ GError **error)
+{
+ ufo_buffer_get_requisition (inputs[0], requisition);
+}
+
+static guint
+ufo_roof_filter_task_get_num_inputs (UfoTask *task)
+{
+ return 1;
+}
+
+static guint
+ufo_roof_filter_task_get_num_dimensions (UfoTask *task,
+ guint input)
+{
+ return 2;
+}
+
+static UfoTaskMode
+ufo_roof_filter_task_get_mode (UfoTask *task)
+{
+ return UFO_TASK_MODE_CPU|UFO_TASK_MODE_REDUCTOR;
+}
+
+static gboolean
+ufo_roof_filter_task_process (UfoTask *task,
+ UfoBuffer **inputs,
+ UfoBuffer *output,
+ UfoRequisition *requisition)
+{
+ UfoRoofFilterTaskPrivate *priv;
+
+ priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (task);
+
+ if (priv->plane) {
+ int buf_plane;
+ GValue *value;
+
+ value = ufo_buffer_get_metadata(inputs[0], "plane");
+ buf_plane = g_value_get_uint(value);
+ if (buf_plane != priv->plane)
+ return TRUE;
+ }
+
+ ufo_buffer_copy(inputs[0], output);
+ ufo_buffer_copy_metadata (inputs[0], output);
+ priv->ready = TRUE;
+
+ return FALSE;
+}
+
+static gboolean
+ufo_roof_filter_task_generate (UfoTask *task,
+ UfoBuffer *output,
+ UfoRequisition *requisition)
+{
+ UfoRoofFilterTaskPrivate *priv;
+
+ priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (task);
+ if (!priv->ready) return FALSE;
+
+/*
+ GValue *value = ufo_buffer_get_metadata(output, "plane");
+ guint buf_plane = g_value_get_uint(value);
+ printf("Passing buffer for plane %u\n", buf_plane);
+*/
+
+ priv->ready = FALSE;
+
+ return TRUE;
+}
+
+static void
+ufo_roof_filter_task_set_property (GObject *object,
+ guint property_id,
+ const GValue *value,
+ GParamSpec *pspec)
+{
+ UfoRoofFilterTaskPrivate *priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (object);
+
+ switch (property_id) {
+ case PROP_BLOCK:
+ priv->block = g_value_get_boolean(value);
+ break;
+ case PROP_PLANE:
+ priv->plane = g_value_get_uint (value);
+ break;
+ case PROP_FPS:
+ priv->fps = g_value_get_double (value);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
+ break;
+ }
+}
+
+static void
+ufo_roof_filter_task_get_property (GObject *object,
+ guint property_id,
+ GValue *value,
+ GParamSpec *pspec)
+{
+ UfoRoofFilterTaskPrivate *priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (object);
+
+ switch (property_id) {
+ case PROP_BLOCK:
+ g_value_set_boolean (value, priv->block);
+ break;
+ case PROP_PLANE:
+ g_value_set_uint (value, priv->plane);
+ break;
+ case PROP_FPS:
+ g_value_set_double (value, priv->fps);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
+ break;
+ }
+}
+
+static void
+ufo_roof_filter_task_finalize (GObject *object)
+{
+ G_OBJECT_CLASS (ufo_roof_filter_task_parent_class)->finalize (object);
+}
+
+static void
+ufo_task_interface_init (UfoTaskIface *iface)
+{
+ iface->setup = ufo_roof_filter_task_setup;
+ iface->get_num_inputs = ufo_roof_filter_task_get_num_inputs;
+ iface->get_num_dimensions = ufo_roof_filter_task_get_num_dimensions;
+ iface->get_mode = ufo_roof_filter_task_get_mode;
+ iface->get_requisition = ufo_roof_filter_task_get_requisition;
+ iface->process = ufo_roof_filter_task_process;
+ iface->generate = ufo_roof_filter_task_generate;
+}
+
+static void
+ufo_roof_filter_task_class_init (UfoRoofFilterTaskClass *klass)
+{
+ GObjectClass *oclass = G_OBJECT_CLASS (klass);
+
+ oclass->set_property = ufo_roof_filter_task_set_property;
+ oclass->get_property = ufo_roof_filter_task_get_property;
+ oclass->finalize = ufo_roof_filter_task_finalize;
+
+ properties[PROP_BLOCK] =
+ g_param_spec_boolean ("block",
+ "Block/Unblock data stream",
+ "Blocks all data output from the filter",
+ FALSE,
+ G_PARAM_READWRITE);
+
+ properties[PROP_PLANE] =
+ g_param_spec_uint ("plane",
+ "Detector planes",
+ "Only passes trough the data for the selecte plane",
+ 0, G_MAXUINT, 0,
+ G_PARAM_READWRITE);
+
+ properties[PROP_FPS] =
+ g_param_spec_double ("fps",
+ "Maximum frame rate",
+ "Limits maximum frame rate",
+ 0, G_MAXDOUBLE, 0,
+ G_PARAM_READWRITE);
+
+ for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++)
+ g_object_class_install_property (oclass, i, properties[i]);
+
+ g_type_class_add_private (oclass, sizeof(UfoRoofFilterTaskPrivate));
+}
+
+static void
+ufo_roof_filter_task_init(UfoRoofFilterTask *self)
+{
+ self->priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE(self);
+}
diff --git a/src/ufo-roof-filter-task.h b/src/ufo-roof-filter-task.h
new file mode 100644
index 0000000..920ba45
--- /dev/null
+++ b/src/ufo-roof-filter-task.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2011-2013 Karlsruhe Institute of Technology
+ *
+ * This file is part of Ufo.
+ *
+ * This library is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation, either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __UFO_ROOF_FILTER_TASK_H
+#define __UFO_ROOF_FILTER_TASK_H
+
+#include <ufo/ufo.h>
+
+G_BEGIN_DECLS
+
+#define UFO_TYPE_ROOF_FILTER_TASK (ufo_roof_filter_task_get_type())
+#define UFO_ROOF_FILTER_TASK(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTask))
+#define UFO_IS_ROOF_FILTER_TASK(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), UFO_TYPE_ROOF_FILTER_TASK))
+#define UFO_ROOF_FILTER_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTaskClass))
+#define UFO_IS_ROOF_FILTER_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), UFO_TYPE_ROOF_FILTER_TASK))
+#define UFO_ROOF_FILTER_TASK_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTaskClass))
+
+typedef struct _UfoRoofFilterTask UfoRoofFilterTask;
+typedef struct _UfoRoofFilterTaskClass UfoRoofFilterTaskClass;
+typedef struct _UfoRoofFilterTaskPrivate UfoRoofFilterTaskPrivate;
+
+struct _UfoRoofFilterTask {
+ UfoTaskNode parent_instance;
+
+ UfoRoofFilterTaskPrivate *priv;
+};
+
+struct _UfoRoofFilterTaskClass {
+ UfoTaskNodeClass parent_class;
+};
+
+UfoNode *ufo_roof_filter_task_new (void);
+GType ufo_roof_filter_task_get_type (void);
+
+G_END_DECLS
+
+#endif
diff --git a/src/ufo-roof-plane-task.c b/src/ufo-roof-plane-task.c
deleted file mode 100644
index 25c2bfa..0000000
--- a/src/ufo-roof-plane-task.c
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Copyright (C) 2011-2015 Karlsruhe Institute of Technology
- *
- * This file is part of Ufo.
- *
- * This library is free software: you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation, either
- * version 3 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <stdio.h>
-
-#ifdef __APPLE__
-#include <OpenCL/cl.h>
-#else
-#include <CL/cl.h>
-#endif
-
-#include "ufo-roof-plane-task.h"
-
-
-struct _UfoRoofPlaneTaskPrivate {
- gboolean ready; // Indicates if data is ready for generation
- guint plane; // Selected plane (0 - pass everything trough)
-};
-
-static void ufo_task_interface_init (UfoTaskIface *iface);
-
-G_DEFINE_TYPE_WITH_CODE (UfoRoofPlaneTask, ufo_roof_plane_task, UFO_TYPE_TASK_NODE,
- G_IMPLEMENT_INTERFACE (UFO_TYPE_TASK,
- ufo_task_interface_init))
-
-#define UFO_ROOF_PLANE_TASK_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE((obj), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTaskPrivate))
-
-enum {
- PROP_0,
- PROP_PLANE,
- N_PROPERTIES
-};
-
-static GParamSpec *properties[N_PROPERTIES] = { NULL, };
-
-UfoNode *
-ufo_roof_plane_task_new (void)
-{
- return UFO_NODE (g_object_new (UFO_TYPE_ROOF_PLANE_TASK, NULL));
-}
-
-static void
-ufo_roof_plane_task_setup (UfoTask *task,
- UfoResources *resources,
- GError **error)
-{
-}
-
-static void
-ufo_roof_plane_task_get_requisition (UfoTask *task,
- UfoBuffer **inputs,
- UfoRequisition *requisition,
- GError **error)
-{
- ufo_buffer_get_requisition (inputs[0], requisition);
-/* int i;
-// UfoRoofPlaneTaskPrivate *priv;
- UfoRequisition in_req;
-
-// priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (task);
- ufo_buffer_get_requisition (inputs[0], &in_req);
-
- requisition->n_dims = in_req.n_dims;
- for (i = 0; i < in_req.n_dims; i++)
- requisition->dims[i] = in_req.dims[i];*/
-}
-
-static guint
-ufo_roof_plane_task_get_num_inputs (UfoTask *task)
-{
- return 1;
-}
-
-static guint
-ufo_roof_plane_task_get_num_dimensions (UfoTask *task,
- guint input)
-{
- return 2;
-}
-
-static UfoTaskMode
-ufo_roof_plane_task_get_mode (UfoTask *task)
-{
- return UFO_TASK_MODE_CPU|UFO_TASK_MODE_REDUCTOR;
-}
-
-static gboolean
-ufo_roof_plane_task_process (UfoTask *task,
- UfoBuffer **inputs,
- UfoBuffer *output,
- UfoRequisition *requisition)
-{
- UfoRoofPlaneTaskPrivate *priv;
-
- priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (task);
-
- if (priv->plane) {
- int buf_plane;
- GValue *value;
-
- value = ufo_buffer_get_metadata(inputs[0], "plane");
- buf_plane = g_value_get_uint(value);
- if (buf_plane != priv->plane)
- return TRUE;
- }
-
- ufo_buffer_copy(inputs[0], output);
- ufo_buffer_copy_metadata (inputs[0], output);
- priv->ready = TRUE;
-
- return FALSE;
-}
-
-static gboolean
-ufo_roof_plane_task_generate (UfoTask *task,
- UfoBuffer *output,
- UfoRequisition *requisition)
-{
- UfoRoofPlaneTaskPrivate *priv;
-
- priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (task);
- if (!priv->ready) return FALSE;
-
-/*
- GValue *value = ufo_buffer_get_metadata(output, "plane");
- guint buf_plane = g_value_get_uint(value);
- printf("Passing buffer for plane %u\n", buf_plane);
-*/
-
- priv->ready = FALSE;
-
- return TRUE;
-}
-
-
-static void
-ufo_roof_plane_task_set_property (GObject *object,
- guint property_id,
- const GValue *value,
- GParamSpec *pspec)
-{
- UfoRoofPlaneTaskPrivate *priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (object);
-
- switch (property_id) {
- case PROP_PLANE:
- priv->plane = g_value_get_uint (value);
- break;
- default:
- G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
- break;
- }
-}
-
-static void
-ufo_roof_plane_task_get_property (GObject *object,
- guint property_id,
- GValue *value,
- GParamSpec *pspec)
-{
- UfoRoofPlaneTaskPrivate *priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (object);
-
- switch (property_id) {
- case PROP_PLANE:
- g_value_set_uint (value, priv->plane);
- break;
- default:
- G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
- break;
- }
-}
-
-static void
-ufo_roof_plane_task_finalize (GObject *object)
-{
- G_OBJECT_CLASS (ufo_roof_plane_task_parent_class)->finalize (object);
-}
-
-static void
-ufo_task_interface_init (UfoTaskIface *iface)
-{
- iface->setup = ufo_roof_plane_task_setup;
- iface->get_num_inputs = ufo_roof_plane_task_get_num_inputs;
- iface->get_num_dimensions = ufo_roof_plane_task_get_num_dimensions;
- iface->get_mode = ufo_roof_plane_task_get_mode;
- iface->get_requisition = ufo_roof_plane_task_get_requisition;
- iface->process = ufo_roof_plane_task_process;
- iface->generate = ufo_roof_plane_task_generate;
-
-}
-
-static void
-ufo_roof_plane_task_class_init (UfoRoofPlaneTaskClass *klass)
-{
- GObjectClass *oclass = G_OBJECT_CLASS (klass);
-
- oclass->set_property = ufo_roof_plane_task_set_property;
- oclass->get_property = ufo_roof_plane_task_get_property;
- oclass->finalize = ufo_roof_plane_task_finalize;
-
- properties[PROP_PLANE] =
- g_param_spec_uint ("plane",
- "Detector planes",
- "Only passes trough the data for the selecte plane",
- 0, G_MAXUINT, 0,
- G_PARAM_READWRITE);
-
- for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++)
- g_object_class_install_property (oclass, i, properties[i]);
-
- g_type_class_add_private (oclass, sizeof(UfoRoofPlaneTaskPrivate));
-}
-
-static void
-ufo_roof_plane_task_init(UfoRoofPlaneTask *self)
-{
- self->priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE(self);
-}
diff --git a/src/ufo-roof-plane-task.h b/src/ufo-roof-plane-task.h
deleted file mode 100644
index 7794065..0000000
--- a/src/ufo-roof-plane-task.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright (C) 2011-2013 Karlsruhe Institute of Technology
- *
- * This file is part of Ufo.
- *
- * This library is free software: you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation, either
- * version 3 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __UFO_ROOF_PLANE_TASK_H
-#define __UFO_ROOF_PLANE_TASK_H
-
-#include <ufo/ufo.h>
-
-G_BEGIN_DECLS
-
-#define UFO_TYPE_ROOF_PLANE_TASK (ufo_roof_plane_task_get_type())
-#define UFO_ROOF_PLANE_TASK(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTask))
-#define UFO_IS_ROOF_PLANE_TASK(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), UFO_TYPE_ROOF_PLANE_TASK))
-#define UFO_ROOF_PLANE_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTaskClass))
-#define UFO_IS_ROOF_PLANE_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), UFO_TYPE_ROOF_PLANE_TASK))
-#define UFO_ROOF_PLANE_TASK_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTaskClass))
-
-typedef struct _UfoRoofPlaneTask UfoRoofPlaneTask;
-typedef struct _UfoRoofPlaneTaskClass UfoRoofPlaneTaskClass;
-typedef struct _UfoRoofPlaneTaskPrivate UfoRoofPlaneTaskPrivate;
-
-struct _UfoRoofPlaneTask {
- UfoTaskNode parent_instance;
-
- UfoRoofPlaneTaskPrivate *priv;
-};
-
-struct _UfoRoofPlaneTaskClass {
- UfoTaskNodeClass parent_class;
-};
-
-UfoNode *ufo_roof_plane_task_new (void);
-GType ufo_roof_plane_task_get_type (void);
-
-G_END_DECLS
-
-#endif
diff --git a/src/ufo-roof-read-file.c b/src/ufo-roof-read-file.c
index a5eb69b..4ee11c6 100644
--- a/src/ufo-roof-read-file.c
+++ b/src/ufo-roof-read-file.c
@@ -55,7 +55,7 @@ static guint ufo_roof_read_file(UfoRoofReadInterface *iface, uint8_t *buffers, G
}
-UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GError **error) {
+UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, const char *path, guint file_id, GError **error) {
UfoRoofReadFile *reader = (UfoRoofReadFile*)calloc(1, sizeof(UfoRoofReadFile));
if (!reader) roof_new_error(error, "Can't allocate UfoRoofReadFile");
@@ -67,7 +67,7 @@ UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GErro
reader->iface.close = ufo_roof_read_file_free;
reader->iface.read =ufo_roof_read_file;
- reader->fname = g_strdup_printf(cfg->path, id + cfg->first_file_number);
+ reader->fname = g_strdup_printf(path, file_id);
if (!reader->fname) {
free(reader);
roof_new_error(error, "Can't build file name");
@@ -77,7 +77,7 @@ UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GErro
if (!reader->fd) {
g_free(reader->fname);
g_free(reader);
- roof_new_error(error, "Can't open file %s", reader->fname);
+ roof_new_error(error, "Can't open file %i at path %s", file_id, path);
}
return (UfoRoofReadInterface*)reader;
diff --git a/src/ufo-roof-read-file.h b/src/ufo-roof-read-file.h
index 54bcf49..787b441 100644
--- a/src/ufo-roof-read-file.h
+++ b/src/ufo-roof-read-file.h
@@ -3,6 +3,6 @@
#include "ufo-roof-read.h"
-UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GError **error);
+UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, const char *path, guint file_id, GError **error);
#endif
diff --git a/src/ufo-roof-read-task.c b/src/ufo-roof-read-task.c
index 1582437..a8ddded 100644
--- a/src/ufo-roof-read-task.c
+++ b/src/ufo-roof-read-task.c
@@ -39,6 +39,9 @@ struct _UfoRoofReadTaskPrivate {
guint id; // Reader ID (defince sequential port number)
gboolean stop; // Flag requiring termination
+ gboolean simulate; // Indicates if we are running in network or simulation modes
+ gchar *path; // UFO file path for simulation mode
+ guint first_file_number; // Number of a first simulated file (0 or 1)
};
static void ufo_task_interface_init (UfoTaskIface *iface);
@@ -54,6 +57,9 @@ enum {
PROP_ID,
PROP_STOP,
PROP_CONFIG,
+ PROP_SIMULATE,
+ PROP_PATH,
+ PROP_FIRST,
N_PROPERTIES
};
@@ -77,7 +83,7 @@ ufo_roof_read_task_setup (UfoTask *task,
if (!priv->config)
roof_setup_error(error, "ROOF configuration is not specified");
- priv->cfg = ufo_roof_config_new(priv->config, &gerr);
+ priv->cfg = ufo_roof_config_new(priv->config, priv->simulate?UFO_ROOF_CONFIG_SIMULATION:UFO_ROOF_CONFIG_DEFAULT, &gerr);
if (!priv->cfg) roof_propagate_error(error, gerr, "roof_config_new: ");
// Consistency checks
@@ -85,9 +91,12 @@ ufo_roof_read_task_setup (UfoTask *task,
roof_setup_error(error, "Specified Stream ID is %u, but only %u data streams is configured", priv->id, priv->cfg->n_streams);
// Start actual reader
- if (priv->cfg->path)
- priv->reader = ufo_roof_read_file_new(priv->cfg, priv->id, &gerr);
- else
+ if (priv->simulate) {
+ if (!priv->path)
+ roof_setup_error(error, "Path to simulated data should be specified");
+
+ priv->reader = ufo_roof_read_file_new(priv->cfg, priv->path, priv->id + priv->first_file_number, &gerr);
+ } else
priv->reader = ufo_roof_read_socket_new(priv->cfg, priv->id, &gerr);
if (!priv->reader)
@@ -115,6 +124,11 @@ ufo_roof_read_task_finalize (GObject *object)
priv->config = NULL;
}
+ if (priv->path) {
+ g_free(priv->path);
+ priv->path = NULL;
+ }
+
G_OBJECT_CLASS (ufo_roof_read_task_parent_class)->finalize (object);
}
@@ -206,6 +220,16 @@ ufo_roof_read_task_set_property (GObject *object,
case PROP_STOP:
priv->stop = g_value_get_boolean (value);
break;
+ case PROP_SIMULATE:
+ priv->simulate = g_value_get_boolean (value);
+ break;
+ case PROP_PATH:
+ if (priv->path) g_free(priv->path);
+ priv->path = g_value_dup_string(value);
+ break;
+ case PROP_FIRST:
+ priv->first_file_number = g_value_get_uint (value);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
@@ -230,6 +254,15 @@ ufo_roof_read_task_get_property (GObject *object,
case PROP_STOP:
g_value_set_boolean (value, priv->stop);
break;
+ case PROP_SIMULATE:
+ g_value_set_boolean (value, priv->simulate);
+ break;
+ case PROP_PATH:
+ g_value_set_string(value, priv->path?priv->path:"");
+ break;
+ case PROP_FIRST:
+ g_value_set_uint (value, priv->first_file_number);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
@@ -277,6 +310,27 @@ ufo_roof_read_task_class_init (UfoRoofReadTaskClass *klass)
FALSE,
G_PARAM_READWRITE);
+ properties[PROP_SIMULATE] =
+ g_param_spec_boolean ("simulate",
+ "Simulation mode",
+ "Read data from the specified files instead of network",
+ FALSE,
+ G_PARAM_READWRITE);
+
+ properties[PROP_PATH] =
+ g_param_spec_string ("path",
+ "Input files for simulation mode",
+ "Optional path to input files for simulation mode (parameter from configuration file is used if not specified)",
+ "",
+ G_PARAM_READWRITE);
+
+ properties[PROP_FIRST] =
+ g_param_spec_uint ("first_file_number",
+ "Offset to the first read file",
+ "Offset to the first read file",
+ 0, G_MAXUINT, 0,
+ G_PARAM_READWRITE);
+
for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++)
g_object_class_install_property (oclass, i, properties[i]);
diff --git a/tests/config.sh b/tests/config.sh
index 3d5dbba..7b9d7d8 100644
--- a/tests/config.sh
+++ b/tests/config.sh
@@ -3,6 +3,8 @@ el7=$(($? == 0))
arch=""
[ $el7 -ne 0 ] && arch="64"
+[ -f /etc/gentoo-release ] && arch="64"
+
ods_path=/mnt/ands/ods/bin-fedora/
vma_path=/mnt/ands/
diff --git a/tests/roof-net.sh b/tests/roof-net.sh
deleted file mode 100755
index 66faa43..0000000
--- a/tests/roof-net.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#! /bin/bash
-
-. config.sh
-
-bufs=800000
-bufs=$((bufs * 4))
-
-#cat roof.yaml | sed '/simulation/,$d' | yq . > roof.json
-
-ulimit -l unlimited
-echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399
-echo 8000 > /proc/sys/vm/nr_hugepages # 0
-
-#VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=$bufs LD_PRELOAD=$vma_lib \
- LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib64" GI_TYPELIB_PATH="/usr/local/lib64/girepository-1.0/" \
- python3 roof.py -c roofhw.json "$@"
-
-
-# python3 roof.py "$@"
diff --git a/tests/roof-sim.sh b/tests/roof-sim.sh
deleted file mode 100755
index 4374221..0000000
--- a/tests/roof-sim.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#! /bin/bash
-
-. config.sh
-
-#cat roof.yaml | yq r - -j | jq '' | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json
-cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json
-
-LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib$arch" GI_TYPELIB_PATH="/usr/local/lib$arch/girepository-1.0/" \
- python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino-%03i.tif" -n 1 "$@"
-# python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino-%03i.raw" -n 1 "$@"
-
-# python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino%i.tif" -n 1 "$@"
-
-# python3 roof.py -c roof.json "$@"
diff --git a/tests/roof-vma.sh b/tests/roof-vma.sh
new file mode 120000
index 0000000..2faff84
--- /dev/null
+++ b/tests/roof-vma.sh
@@ -0,0 +1 @@
+roof.sh
\ No newline at end of file
diff --git a/tests/roof.json b/tests/roof.json
index d39a90f..1848bc6 100644
--- a/tests/roof.json
+++ b/tests/roof.json
@@ -14,10 +14,6 @@
"delta_x": 500,
"delta_z": 1200
},
- "reconstruction": {
- "parallel_projections": 512,
- "parallel_bins": 256
- },
"network": {
"protocol": "udp",
"port": 52067,
@@ -29,9 +25,21 @@
"buffer_size": 10,
"packets_at_once": 100
},
+ "data": {
+ "base_path": "/home/csa/roof2_data/test_data"
+ },
"simulation": {
- "path": "/home/csa/roof2_data/test_data/data_%02u.dat",
"first_file_number": 1,
+ "data": "sim/data_%02u.dat",
+ "flat_fields": "sim/flat_%02u.dat",
+ "dark_fields": "sim/dark_%02u.dat",
"header_size": 0
+ },
+ "correction": {
+ "aggregation": "median"
+ },
+ "reconstruction": {
+ "parallel_projections": 512,
+ "parallel_bins": 256
}
}
diff --git a/tests/roof.py b/tests/roof.py
index 1941aa8..71b4465 100644
--- a/tests/roof.py
+++ b/tests/roof.py
@@ -1,79 +1,14 @@
-import gi
-import re
+#import gi
import sys
-import json
-import argparse
+from roof.graph import RoofGraph
-gi.require_version('Ufo', '0.0')
-from gi.repository import Ufo
-from gi.repository import GObject
+#gi.require_version('Ufo', '0.0')
+#from gi.repository import Ufo
+#from gi.repository import GObject
-class RoofConfig:
- def __init__(self, args, config="roof.json"):
- self.streams = 1
- self.bit_depth = 8
- self.convert = False if ((not args.output) or (re.compile('\.raw$').search(args.output))) else True
- self.build = "raw" if args.noroof else "ufo" if self.convert else "sino"
- with open(config) as json_file:
- cfg = json.load(json_file)
- if cfg.get("network", {}).get("streams") != None:
- self.streams = cfg["network"]["streams"]
- elif cfg.get("hardware", {}).get("modules") != None:
- self.streams = cfg["setup"]["modules"]
+roof = RoofGraph()
+graph = roof.get()
- if cfg.get("hardware", {}).get("bit_depth") != None:
- self.bit_depth = cfg["hardware"]["bit_depth"]
-
-parser = argparse.ArgumentParser()
-parser.add_argument('-c', '--config', dest="config", default="roof.json", help="ROOF configuration (JSON)")
-parser.add_argument('-o', '--output', dest="output", default=None, help="Output file")
-parser.add_argument('-n', '--number', dest="number", default=None, type=int, help="Specify number of frames to capture (limits number of captured frames irrespective of further filtering)")
-parser.add_argument('-p', '--plane', dest="plane", default=None, type=int, help="Only process the specified detector plane (indexed from 1)")
-parser.add_argument( '--no-roof', dest="noroof", default=False, type=bool, help="Disable ROOF, only network testing (no sinogram building, store linearly)")
-#parser.add_argument('-r', '--raw', dest="raw", default=False, type=bool, help="Store raw data, ignore processed")
-#parser.add_argument('-v', '--visualize', dest='visualize', default=False, type=bool, help="Visualize data")
-args = parser.parse_args()
-
-
-cfg = RoofConfig(args, args.config)
-
-pm = Ufo.PluginManager()
-graph = Ufo.TaskGraph()
-scheduler = Ufo.Scheduler()
-
-if args.output is None:
- print ("Starting ROOF using NULL writter")
- write = pm.get_task('null')
- if args.number is None: args.number = 0
-else:
- print ("Starting ROOF streaming to {}".format(args.output))
- write = pm.get_task('write')
- write.set_properties(filename=args.output)
- if args.number is None: args.number = 5
-
-build = pm.get_task('roof-build')
-build.set_properties(config=args.config, number=args.number, build=cfg.build)
-
-plane = pm.get_task('roof-plane') if args.plane else None
-if plane: plane.set_properties(plane=args.plane)
-
-for id in range(cfg.streams):
- read = pm.get_task('roof-read')
- read.set_properties(config=args.config, id=id)
- graph.connect_nodes(read, build)
- build.bind_property('stop', read, 'stop', GObject.BindingFlags.DEFAULT)
-
-#read_task.set_properties(path='/home/data/*.tif', start=10, number=100)
-#graph.connect_nodes_full(read, write, 0)
-
-if plane:
- graph.connect_nodes(build, plane)
- graph.connect_nodes(plane, write)
-else:
- graph.connect_nodes(build, write)
-
-
-
-scheduler.run(graph)
+roof.run()
diff --git a/tests/roof.sh b/tests/roof.sh
new file mode 100755
index 0000000..a8fa89e
--- /dev/null
+++ b/tests/roof.sh
@@ -0,0 +1,32 @@
+#! /bin/bash
+
+. config.sh
+
+function pyroof {
+ LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib$arch" GI_TYPELIB_PATH="/usr/local/lib$arch/girepository-1.0/" \
+ python3 roof.py "$@"
+}
+
+if [[ "$0" =~ roof-vma ]]; then
+ function roof {
+ bufs=800000
+ bufs=$((bufs * 4))
+
+ ulimit -l unlimited
+ echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399
+ echo 8000 > /proc/sys/vm/nr_hugepages # 0
+
+ VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=$bufs LD_PRELOAD=$vma_lib \
+ pyroof "$@"
+ }
+else
+ function roof {
+ pyroof "$@"
+ }
+fi
+
+#cat roof.yaml | sed '/simulation/,$d' | yq . > roof.json
+#cat roof.yaml | yq r - -j | jq '' | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json
+cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json
+
+roof "$@"
diff --git a/tests/roof.yaml b/tests/roof.yaml
index d8a1c92..0a0ce1d 100644
--- a/tests/roof.yaml
+++ b/tests/roof.yaml
@@ -14,11 +14,6 @@ geometry:
# source_angle_config: "path.xxx"
delta_x: 500
delta_z: 1200
-#optics:
-# flat_field_config: "path.xxx"
-reconstruction:
- parallel_projections: 512
- parallel_bins: 256
network:
protocol: udp
port: 52067
@@ -32,9 +27,32 @@ performance:
buffer_size: 10
# drop_buffers: 0
packets_at_once: 100
+data:
+ base_path: "/home/csa/roof2_data/test_data"
+# first_file_number: 1
+# flat_fields: "flats/flat_%04u.tif"
+# dark_fields: "darks/dark_%04u.tif"
+# raw_sinograms: "raw/sino_%04u.raw"
+# fan_sinograms: "fan/sino_%04u.raw"
+# parallel_sinograms: "par/sino_%04u.raw"
+# filtered_sinograms: "flt/sino_%04u.raw"
+# slices: "slices/slice_%04u.raw"
+ #rings: ...
simulation:
- path: "/home/csa/roof2_data/test_data/data_%02u.dat"
first_file_number: 1
+# base_path: "/home/csa/roof2_data/test_data"
+ data: "sim/data_%02u.dat"
+ flat_fields: "sim/flat_%02u.dat"
+ dark_fields: "sim/dark_%02u.dat"
header_size: 0
# max_packet_size: 1284
# dataset_size: 1024000
+correction:
+ aggregation: "median"
+reconstruction:
+ parallel_projections: 512
+ parallel_bins: 256
+# filters: [ "roof-fan2par", "fft", "filter", "ifft", "backproject" ]
+# backproject-options:
+#visualization:
+#control:
diff --git a/tests/roof/__init__.py b/tests/roof/__init__.py
new file mode 100644
index 0000000..b8023d8
--- /dev/null
+++ b/tests/roof/__init__.py
@@ -0,0 +1 @@
+__version__ = '0.0.1'
diff --git a/tests/roof/arguments.py b/tests/roof/arguments.py
new file mode 100644
index 0000000..22ea42b
--- /dev/null
+++ b/tests/roof/arguments.py
@@ -0,0 +1,32 @@
+import argparse
+from roof.defaults import roof_data_types
+
+def roof_get_args():
+ data_types = []
+ for stage in roof_data_types:
+ data_types += roof_data_types[stage].keys()
+ data_types = set(data_types)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-c', '--config', dest="config", default="roof.json", help="ROOF configuration (JSON)")
+
+ # Modes
+ parser.add_argument('-s', '--simulate', dest="simulate", default=None, action="store_true", help="Simulation mode, read data from files instead of network")
+ parser.add_argument('-b', '--benchmark', dest="benchmark", default=None, action="store_true", help="Bencmarking mode, writes to /dev/null")
+ parser.add_argument('-g', '--gui', dest='gui', default=False, action="store_true", help="Visualize data")
+ parser.add_argument('-t', '--track', dest='track', default=False, action="store_true", help="Track & control experiment")
+
+ parser.add_argument( '--no-roof', dest="noroof", default=False, action="store_true", help="Disable ROOF, only network testing (no sinogram building, store linearly)")
+
+ # I/O
+ #parser.add_argument('-i', '--input', dest="input", default=None, help="Reconstruct from sinograms")
+ parser.add_argument('-o', '--output', dest="output", default=None, help="Output file(s)")
+ parser.add_argument('-r', '--read', dest="read", default=None, choices=data_types, nargs='?', const="raw_sinograms", help="Read recorded sinograms instead of listening on the network")
+ parser.add_argument('-w', '--write', dest="write", default=None, choices=data_types, nargs='?', const="slices", help="Only generate flat-fields, dark-fields, or sinograms (default)")
+ parser.add_argument( '--format', dest="format", default=None, help="Override default storage format")
+
+ # Limits & Filtering
+ parser.add_argument('-n', '--number', dest="number", default=None, type=int, help="Specify number of frames to capture (limits number of captured frames irrespective of further filtering)")
+ parser.add_argument('-p', '--plane', dest="plane", default=None, type=int, help="Only process the specified detector plane (indexed from 1)")
+
+ return parser.parse_args()
diff --git a/tests/roof/config.py b/tests/roof/config.py
new file mode 100644
index 0000000..e085ed8
--- /dev/null
+++ b/tests/roof/config.py
@@ -0,0 +1,67 @@
+import re
+import json
+
+from roof.arguments import roof_get_args
+from roof.defaults import roof_default_paths, roof_raw_data_types
+
+class RoofConfig:
+ def __init__(self, config=None):
+ self.args = roof_get_args()
+ self.config_file = self.get_arg('config', 'roof.json') if config is None else config
+ with open(self.config_file) as json_file:
+ self.cfg = json.load(json_file)
+
+ self.path = self.get_opt('data', 'base_path', './')
+ self.planes = self.get_opt('hardware', 'planes', 1)
+ self.modules = self.get_opt('hardware', 'modules', None)
+ self.streams = self.get_opt('network', 'streams', 1 if self.modules is None else self.modules)
+ self.bit_depth = self.get_opt('hardware', 'bit_depth', 8)
+
+ if self.args.number is None: self.args.number = 0 if self.args.benchmark else self.planes
+
+ # Consistency and default mode
+ if (self.args.plane is not None) and (self.args.plane > self.planes):
+ raise ValueError("Only {} planes in configuration, but the plane {} is requested".format(self.planes, self.args.plane))
+
+ n_modes = (int(self.args.gui) + int(self.args.track) + int(0 if self.args.write is None else 1))
+ if n_modes > 1:
+ raise ValueError("GUI, Control, and Write modes are mutualy incompatible")
+ elif n_modes == 0:
+ self.args.write = "raw_sinograms"
+
+
+ def get_arg(self, arg, default = None):
+ ret = getattr(self.args, arg)
+ return ret if ret is not None else default
+
+ def get_opt(self, group, item, default = None):
+ if self.cfg.get(group, {}).get(item) != None:
+ return self.cfg[group][item]
+ else:
+ return default
+
+ def get_roof_path(self, data_type):
+ subpath = self.get_opt('data', data_type)
+ if subpath is None: subpath = roof_default_paths[data_type]
+ if subpath is None: raise "Unknown data type %s is requested" % subpath
+ return subpath if subpath.startswith('/') else self.path + '/' + subpath
+
+ def get_writer_type(self):
+ return None if self.args.benchmark else self.args.write if self.args.write else 'raw_sinograms'
+
+ def get_writer_path(self):
+ data_type = self.get_writer_type()
+ if data_type is not None:
+ path = self.args.output if self.args.output is not None else self.get_roof_path(data_type)
+ if self.args.format: path = re.sub('\.([^.]+)$', '.' + self.args.format, path)
+ return path
+ return None
+
+ def check_writer_type_is_raw_or_none(self):
+ data_type = self.get_writer_type()
+ data_path = self.get_writer_path()
+ return (data_type is None) or ((data_type in roof_raw_data_types) and re.search('\.raw$', data_path))
+
+ def check_writer_type_is_raw(self):
+ data_type = self.get_writer_type()
+ return (data_type is not None) and self.check_writer_type_is_raw_or_none()
diff --git a/tests/roof/defaults.py b/tests/roof/defaults.py
new file mode 100644
index 0000000..eed3fe5
--- /dev/null
+++ b/tests/roof/defaults.py
@@ -0,0 +1,42 @@
+roof_default_paths = {
+ 'flat_fields': "flats/flat_%04u.raw",
+ 'dark_fields': "darks/dark_%04u.raw",
+ 'raw_sinograms': "raw/sino_%04u.raw",
+ 'fan_sinograms': "fan/sino_%04u.tif",
+ 'parallel_sinograms': "par/sino_%04u.tif",
+ 'filtered_sinograms': "flt/sino_%04u.tif",
+ 'slices': "slices/slice_%04u.raw"
+}
+
+#roof_default_simulation_paths = {
+# 'data': "sim/data_%02u.dat",
+# 'flat_fields': "sim/flat_%02u.dat",
+# 'dark_fields': "sim/dark_%02u.dat"
+#}
+
+roof_filters = {
+ 'correction': [ "flat-field-correct" ],
+ 'reconstruction': [ "roof-fan2par", "fft", "filter", "ifft", "backproject" ],
+ 'control': [ ],
+ 'visualization': [ ]
+}
+
+roof_data_types = {
+ 'correction': {
+ 'flat_fields': 'flat-field-correct',
+ 'dark_fields': 'flat-field-correct',
+ 'raw_sinograms': 'flat-field-correct',
+ 'fan_sinograms': None
+ },
+ 'reconstruction': {
+ 'fan_sinograms': 'roof-fan2par',
+ 'parallel_sinograms': 'fft',
+ 'filtered_sinograms': 'backproject',
+ 'slices': None
+ },
+ 'control': {
+ }
+}
+
+roof_raw_data_types = [k for k, v in roof_data_types['correction'].items() if v is not None ]
+roof_aux_data_types = [v for v in roof_raw_data_types if 'sino' not in v ]
diff --git a/tests/roof/graph.py b/tests/roof/graph.py
new file mode 100644
index 0000000..c34a3ed
--- /dev/null
+++ b/tests/roof/graph.py
@@ -0,0 +1,203 @@
+import re
+import gi
+
+gi.require_version('Ufo', '0.0')
+from gi.repository import Ufo
+from gi.repository import GObject
+
+from roof.config import RoofConfig
+from roof.defaults import roof_filters, roof_data_types, roof_raw_data_types, roof_aux_data_types
+from roof.utils import get_filenames
+
+class RoofGraph(RoofConfig):
+ def __init__(self, config=None):
+ self.pm = Ufo.PluginManager()
+ self.graph = Ufo.TaskGraph()
+ self.scheduler = Ufo.Scheduler()
+ self.tasks = {}
+
+ super(RoofGraph, self).__init__()
+
+ def get_task(self, name, **kwargs):
+ task = self.pm.get_task(name)
+ task.set_properties(name, **kwargs)
+ return task
+
+ def save_task(self, stage, alias, task):
+ if stage is None: stage = "general"
+ if stage not in self.tasks: self.tasks[stage] = {}
+ self.tasks[stage][alias if alias is not None else name] = task
+ return task
+
+ def get_roof_task(self, name, **kwargs):
+ kwargs.update(config = self.config_file)
+ return self.get_task(name, **kwargs)
+
+ def get_processor_task(self, stage, name, **kwargs):
+ extra_args = self.get_opt(stage, name + '-options')
+ if extra_args is not None: kwargs.update(extra_args)
+ if (re.compile('roof').match(name)): kwargs.update(config = self.config_file)
+ return self.save_task(stage, name, self.get_task(name, **kwargs))
+
+ def get_reader(self):
+ first = self.get_opt('data', 'first_file_number', 1)
+ if self.args.read:
+ # Reconstruction from standard UFO files
+ path = self.get_roof_path(self.args.read)
+ step = 1
+ if (self.args.plane is not None) and (self.args.plane > 0):
+ first += self.args.plane - 1;
+ step = self.planes
+
+ params = { 'path': path, 'first': first, 'step': step }
+ if self.args.number:
+ params['number'] = self.args.number
+
+ print ("Reading {} data from {}".format(self.args.read,path))
+ return self.get_task('read', **params)
+ else:
+ path = None
+ if self.args.simulate:
+ first = self.get_opt('simulation', 'first_file_number', first)
+ base_path = self.get_opt('simulation', 'base_path', self.path)
+ read_path = self.get_opt('simulation', self.args.write if self.args.write and self.args.write in roof_aux_data_types else 'data')
+ path = read_path if read_path.startswith('/') else base_path + '/' + read_path
+ print ("Simulating packets from {}".format(path))
+
+ # Reconstruction from network or simulated data (also generation of flat/dark-fields)
+ build_type = "raw" if self.args.noroof else "sino" if self.check_writer_type_is_raw() else "ufo"
+ build = self.get_roof_task('roof-build', simulate = self.args.simulate, number = self.args.number, build = build_type)
+ for id in range(self.streams):
+ read = self.get_roof_task('roof-read', id = id, simulate = self.args.simulate, path = path, first_file_number = first)
+ self.graph.connect_nodes(read, build)
+ build.bind_property('stop', read, 'stop', GObject.BindingFlags.DEFAULT)
+
+ return build
+
+ def get_writer(self):
+ path = self.get_writer_path()
+ if path is None:
+ print ("Starting ROOF using NULL writter")
+ write = self.get_task('null')
+ else:
+ # FIXME: If writting non raw data, we may need to generate all-0-frames if something broken/corrupted.
+ print ("Starting ROOF streaming to {}".format(path))
+ write = self.get_task('write', filename=path)
+ return write
+
+ def get_correction_flat_field_correct(self, head):
+ # Standard UFO reconstruction stack distinguish flat/dark-fields recorded before and after experiment. We only do 'before experiment' part.
+ darks = self.get_roof_path('dark_fields')
+ n_darks = len(get_filenames(darks))
+ if n_darks == 0: raise FileNotFoundError("Dark fields are not found in {}".format(darks))
+ flats = self.get_roof_path('falt_fields')
+ n_flats = len(get_filenames(flats))
+ if n_flats == 0: raise FileNotFoundError("Flat fields are not found in {}".format(flats))
+ dark_reader = self.get_task('read', path = darks)
+ flat_reader = self.get_task('read', path = flats)
+
+ # We are using standard get_task here because this is too generic plugin to allow config-based customization
+ mode = self.get_opt('correction', 'aggregation', 'average')
+ if mode == 'median':
+ dark_stack = self.get_task('stack', number = n_darks)
+ dark_reduced = self.get_task('flatten', mode = 'median')
+ flat_stack = self.get_task('stack', number = n_flats)
+ flat_reduced = self.get_task('flatten', mode = 'median')
+
+ self.graph.connect_nodes(dark_reader, dark_stack)
+ self.graph.connect_nodes(dark_stack, dark_reduced)
+ self.graph.connect_nodes(flat_reader, flat_stack)
+ self.graph.connect_nodes(flat_stack, flat_reduced)
+ elif mode == 'average':
+ dark_reduced = self.get_task('average')
+ flat_reduced = self.get_task('average')
+ self.graph.connect_nodes(dark_reader, dark_reduced)
+ self.graph.connect_nodes(flat_reader, flat_reduced)
+ else:
+ raise ValueError('Invalid reduction mode')
+
+ ffc = self.get_task('flat-field-correct') # dark_scale=args.dark_scale, absorption_correct=args.absorptivity, fix_nan_and_inf=args.fix_nan_and_inf)
+ self.graph.connect_nodes_full(head, ffc, 0)
+ self.graph.connect_nodes_full(dark_reduced, ffc, 1)
+ self.graph.connect_nodes_full(flat_reduced, ffc, 2)
+ return ffc
+
+ def get_processor(self, head, stage, writer = None):
+ # skip (but not if not already skipped in previous processor)
+ # how to connect readers to ffc?
+
+ filters = self.get_opt(stage, 'filters', roof_filters[stage])
+ read_here = self.args.read and self.args.read in roof_data_types[stage].keys()
+ write_here = self.args.write and self.args.write in roof_data_types[stage].keys()
+
+ start_pos = 0
+ if read_here:
+ start_filter = roof_data_types[stage][self.args.read]
+ start_pos = filters.index(start_filter)
+
+ last_pos = len(filters)
+ if write_here:
+ stop_filter = roof_data_types[stage][self.args.write]
+ if stop_filter: last_pos = filters.index(stop_filter)
+
+ # Will just execute empty range if we start reading from the end (e.g. 'fan-sinograms' in correction)
+ for i in range(start_pos, last_pos):
+ method = 'get_' + stage + '_' + filters[i].replace('-','_')
+ if method in dir(self):
+ f = getattr(self, method)(head)
+ else:
+ f = self.get_processor_task(stage, filters[pos])
+ graph.connect_nodes(head, f)
+ head = f
+
+ if write_here and writer:
+ self.graph.connect_nodes(head, writer)
+
+ return None if write_here else head
+
+
+ def get(self):
+ reader = self.get_reader()
+ writer = self.get_writer()
+
+ # We support following operation modes (defined by modifiers -w -c -g ]
+ # - Record mode: Writting raw data (raw-sinograms, flat-fields, dark-fields) [ no modified or -w <...> ]
+ # - Write mode: The reconstruction is performed and data is written after the specified step (default) [ -w <all other data types> ]
+ # - Control mode: Control branch and raw data writting [ -c ]
+ # - GUI mode: Visualization in GUI + raw_sinograms are written when enabled in GUI + some control tasks (also when enabled) [ -g ]
+
+ head = reader
+ # Check if we are branching here
+ if (self.args.track or self.args.gui) and (self.get_data_type() is not None):
+ # FIXME: In GUI mode we can add here a 'write filter' to pause/resume writting. Alternative is to pass gobject flaga to fastwriter (this will be limited to fastwriter, then, which is likely OK)
+ # FIXME: we may need to convert in the end if we are writing raw data and the data is comming from net/simulation
+ # In other case (non branch), either we have already converted (in reader) or we don't need to convert (writing raw data). Small performance penalty if we convert before filter, but ....
+ copy = Ufo.CopyTask()
+ self.graph.connect_nodes(reader, copy)
+ self.graph.connect_nodes(copy, writer)
+ head = copy
+
+ # Sinograms are already filtered in the reader
+ if not self.args.read:
+ main_filter = self.get_task('roof-filter', plane = self.args.plane) if self.args.plane else None
+ if main_filter:
+ self.graph.connect_nodes(head, main_filter)
+ head = main_filter
+
+ class finish(Exception): pass
+ try:
+ if not self.args.read or self.args.read in roof_data_types['correction'].keys():
+ head = self.get_processor(head, 'correction', writer)
+ if not head: raise finish()
+
+ if head != reader or self.args.read in roof_data_types['reconstruction'].keys():
+ head = self.get_processor(head, 'reconstruction', writer)
+ if not head: raise finish()
+
+ # if head split to 3 branches.... Otherwise, continue with control branch...
+ except finish:
+ pass
+
+ def run(self):
+ self.scheduler.run(self.graph)
+
\ No newline at end of file
diff --git a/tests/roof/utils.py b/tests/roof/utils.py
new file mode 100644
index 0000000..eb389ed
--- /dev/null
+++ b/tests/roof/utils.py
@@ -0,0 +1,13 @@
+import glob
+import logging
+import math
+import os
+
+def get_filenames(path):
+ """Get all filenams from *path*, which could be a directory or a pattern
+ for matching files in a directory.
+ """
+ if os.path.isdir(path):
+ path = os.path.join(path, '*')
+
+ return sorted(glob.glob(path))
--
cgit v1.2.3