diff options
30 files changed, 673 insertions, 246 deletions
@@ -13,3 +13,4 @@ Makefile /tests/venv /build/ /_build/ +__pycache__ diff --git a/docs/hardware.txt b/docs/hardware.txt index a293887..50c3a0c 100644 --- a/docs/hardware.txt +++ b/docs/hardware.txt @@ -4,3 +4,8 @@ * With 46 packets, however, we can't split a full rotation in a whole number of packets. So, we need to find maximal number m, so that (m <= n) and (samples_per_rotation % m = 0) i.e. 40 + + +Questions +========= + - Do we need to compute 'flats' and 'darks' for each plane separately? Or just one set will work for all? diff --git a/docs/schemes/roof_graph.odg b/docs/schemes/roof_graph.odg Binary files differnew file mode 100644 index 0000000..1fb4643 --- /dev/null +++ b/docs/schemes/roof_graph.odg diff --git a/docs/schemes/roof_graph.pdf b/docs/schemes/roof_graph.pdf Binary files differnew file mode 100644 index 0000000..8ba6b7f --- /dev/null +++ b/docs/schemes/roof_graph.pdf diff --git a/docs/todo.txt b/docs/todo.txt index 8497a69..88b518c 100644 --- a/docs/todo.txt +++ b/docs/todo.txt @@ -3,13 +3,17 @@ Main + Add plane/frame-number/broken metadata in the UFO buffers. Check propagation trough standard ufo filters. [ propogates trough processors, but not reductors ] + Plane selector filter - Handle packets with data from multiple datasets - - Filter to ingest zero-padded broken frames. - Try UFO 'flat-field' correction filter - Cone-beam to parallel-beam resampling ? - Full reconstruction chain - Try UFO visualization filter - - "Reconstructed data storage" and "Visualization + raw data storage" modes + - "Reconstructed data storage" and "Visualization + raw data storage" modes. Implement stand-alone 'roof-converter' filter. +If necesary +=========== + - Task 'roof-ingest-missing' to ingest zero-padded broken frames (and include get_writer()) + - Add ROOF metadata (plane, etc.) if reading from sinograms + Optional ======== - Try online compression of the ROOF data @Gregoire diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0967ffb..116b400 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 2.6) set(ufofilter_SRCS ufo-roof-read-task.c ufo-roof-build-task.c - ufo-roof-plane-task.c + ufo-roof-filter-task.c ) set(common_SRCS diff --git a/src/meson.build b/src/meson.build index 8a03a57..8f8930a 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,7 +1,7 @@ plugins = [ 'roof-read', 'roof-build', - 'roof-plane', + 'roof-filter', ] roof_common_src = [ @@ -16,7 +16,7 @@ roof_plugin_src = { 'roof-build': [ 'ufo-roof-buffer.c', ], - 'roof-plane': [ + 'roof-filter': [ ], } diff --git a/src/ufo-roof-build-task.c b/src/ufo-roof-build-task.c index 93cb133..9d55d38 100644 --- a/src/ufo-roof-build-task.c +++ b/src/ufo-roof-build-task.c @@ -44,6 +44,7 @@ struct _UfoRoofBuildTaskPrivate { BuildType build; // What dataset do we build: ROOF sinogram or raw network data guint number; // Number of datasets to read gboolean stop; // Stop flag + gboolean simulate; // Indicates if we are running in network or simulation modes guint announced; // For debugging @@ -71,6 +72,7 @@ static GEnumValue build_values[] = { enum { PROP_0, PROP_STOP, + PROP_SIMULATE, PROP_NUMBER, PROP_BUILD, PROP_CONFIG, @@ -97,7 +99,7 @@ ufo_roof_build_task_setup (UfoTask *task, if (!priv->config) roof_setup_error(error, "ROOF configuration is not specified"); - priv->cfg = ufo_roof_config_new(priv->config, &gerr); + priv->cfg = ufo_roof_config_new(priv->config, priv->simulate?UFO_ROOF_CONFIG_SIMULATION:UFO_ROOF_CONFIG_DEFAULT, &gerr); if (!priv->cfg) roof_propagate_error(error, gerr, "roof-build-setup: "); @@ -281,15 +283,23 @@ ufo_roof_build_task_generate (UfoTask *task, case 32: ufo_buffer_convert(output, UFO_BUFFER_DEPTH_32U); break; + default: + printf("Usupported bit-depth %u\n", cfg->bit_depth); } } // Metadata: plane and sequential number within the plane g_value_init (&ival, G_TYPE_UINT); g_value_init (&lval, G_TYPE_ULONG); + if (priv->build != BUILD_UFO) { + g_value_set_uint (&ival, cfg->bit_depth); + ufo_buffer_set_metadata (output, "bpp", &ival); + } g_value_set_uint (&ival, 1 + seqid % cfg->n_planes); ufo_buffer_set_metadata (output, "plane", &ival); g_value_set_ulong (&lval, seqid / cfg->n_planes); + ufo_buffer_set_metadata (output, "plane_id", &lval); + g_value_set_ulong (&lval, seqid); ufo_buffer_set_metadata (output, "seqid", &lval); g_value_unset(&lval); g_value_unset(&ival); @@ -325,6 +335,9 @@ ufo_roof_build_task_set_property (GObject *object, case PROP_STOP: priv->stop = g_value_get_boolean (value); break; + case PROP_SIMULATE: + priv->simulate = g_value_get_boolean (value); + break; case PROP_NUMBER: priv->number = g_value_get_uint (value); break; @@ -356,6 +369,9 @@ ufo_roof_build_task_get_property (GObject *object, case PROP_STOP: g_value_set_boolean (value, priv->stop); break; + case PROP_SIMULATE: + g_value_set_boolean (value, priv->simulate); + break; case PROP_NUMBER: g_value_set_uint (value, priv->number); break; @@ -403,6 +419,14 @@ ufo_roof_build_task_class_init (UfoRoofBuildTaskClass *klass) FALSE, G_PARAM_READWRITE); + + properties[PROP_SIMULATE] = + g_param_spec_boolean ("simulate", + "Simulation mode", + "Read data from the specified files instead of network", + FALSE, + G_PARAM_READWRITE); + properties[PROP_NUMBER] = g_param_spec_uint("number", "Number of datasets to receive", diff --git a/src/ufo-roof-config.c b/src/ufo-roof-config.c index 4788a2a..17f4b30 100644 --- a/src/ufo-roof-config.c +++ b/src/ufo-roof-config.c @@ -43,9 +43,6 @@ void ufo_roof_config_free(UfoRoofConfig *cfg) { if (cfg) { UfoRoofConfigPrivate *priv = (UfoRoofConfigPrivate*)cfg; - if (cfg->path) - g_free(cfg->path); - if (priv->parser) g_object_unref (priv->parser); @@ -53,7 +50,7 @@ void ufo_roof_config_free(UfoRoofConfig *cfg) { } } -UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) { +UfoRoofConfig *ufo_roof_config_new(const char *config, UfoRoofConfigFlags flags, GError **error) { UfoRoofConfigPrivate *priv; UfoRoofConfig *cfg; @@ -66,6 +63,7 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) { JsonObject *performance = NULL; JsonObject *simulation = NULL; JsonObject *reconstruction = NULL; + JsonObject *data = NULL; GError *gerr = NULL; @@ -97,7 +95,6 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) { cfg->dataset_size = 0; cfg->buffer_size = 2; cfg->drop_buffers = 0; - cfg->path = NULL; // Read configuration @@ -118,8 +115,11 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) { roof_config_node_get(optics, root, object, "optics"); roof_config_node_get(network, root, object, "network"); roof_config_node_get(reconstruction, root, object, "reconstruction"); - roof_config_node_get(simulation, root, object, "simulation"); roof_config_node_get(performance, root, object, "performance"); + roof_config_node_get(data, root, object, "data"); + + if (flags&UFO_ROOF_CONFIG_SIMULATION) + roof_config_node_get(simulation, root, object, "simulation"); } if (hardware) { @@ -183,8 +183,6 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) { } if (simulation) { - roof_config_node_get_string(cfg->path, simulation, "path"); - roof_config_node_get(cfg->first_file_number, simulation, int, "first_file_number"); roof_config_node_get(cfg->header_size, simulation, int, "header_size"); if (!cfg->payload_size) diff --git a/src/ufo-roof-config.h b/src/ufo-roof-config.h index b6ee748..34bef0b 100644 --- a/src/ufo-roof-config.h +++ b/src/ufo-roof-config.h @@ -25,8 +25,6 @@ typedef struct { // Network Server / Reader - gchar *path; // Location of data files for simmulation purposes (i.e. reading a sequence of files instead listening on the corresponding ports) - guint first_file_number; // Indicates if the numbering of files starts at 0 or 1 gchar *protocol; // Protocols: tcp, udp, tcp6, udp6, ... guint port; // First port guint n_streams; // Number of independent data streams (expected on sequential ports), by default equal to number of ROOF modules @@ -53,7 +51,13 @@ typedef struct { } UfoRoofConfig; -UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error); +typedef enum { + UFO_ROOF_CONFIG_DEFAULT = 0, + UFO_ROOF_CONFIG_SIMULATION = 1 +} UfoRoofConfigFlags; + + +UfoRoofConfig *ufo_roof_config_new(const char *config, UfoRoofConfigFlags flags, GError **error); void ufo_roof_config_free(UfoRoofConfig *cfg); #endif /* __UFO_ROOF_CONFIG_H */ diff --git a/src/ufo-roof-plane-task.c b/src/ufo-roof-filter-task.c index 25c2bfa..01bc742 100644 --- a/src/ufo-roof-plane-task.c +++ b/src/ufo-roof-filter-task.c @@ -17,98 +17,90 @@ * License along with this library. If not, see <http://www.gnu.org/licenses/>. */ -#include <stdio.h> - #ifdef __APPLE__ #include <OpenCL/cl.h> #else #include <CL/cl.h> #endif -#include "ufo-roof-plane-task.h" +#include "ufo-roof-filter-task.h" -struct _UfoRoofPlaneTaskPrivate { +struct _UfoRoofFilterTaskPrivate { gboolean ready; // Indicates if data is ready for generation + gboolean block; // Block output alltogether guint plane; // Selected plane (0 - pass everything trough) + gdouble fps; // Limit maximum frame rate }; static void ufo_task_interface_init (UfoTaskIface *iface); -G_DEFINE_TYPE_WITH_CODE (UfoRoofPlaneTask, ufo_roof_plane_task, UFO_TYPE_TASK_NODE, +G_DEFINE_TYPE_WITH_CODE (UfoRoofFilterTask, ufo_roof_filter_task, UFO_TYPE_TASK_NODE, G_IMPLEMENT_INTERFACE (UFO_TYPE_TASK, ufo_task_interface_init)) -#define UFO_ROOF_PLANE_TASK_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE((obj), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTaskPrivate)) +#define UFO_ROOF_FILTER_TASK_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE((obj), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTaskPrivate)) enum { PROP_0, PROP_PLANE, + PROP_FPS, + PROP_BLOCK, N_PROPERTIES }; static GParamSpec *properties[N_PROPERTIES] = { NULL, }; UfoNode * -ufo_roof_plane_task_new (void) +ufo_roof_filter_task_new (void) { - return UFO_NODE (g_object_new (UFO_TYPE_ROOF_PLANE_TASK, NULL)); + return UFO_NODE (g_object_new (UFO_TYPE_ROOF_FILTER_TASK, NULL)); } static void -ufo_roof_plane_task_setup (UfoTask *task, +ufo_roof_filter_task_setup (UfoTask *task, UfoResources *resources, GError **error) { } static void -ufo_roof_plane_task_get_requisition (UfoTask *task, +ufo_roof_filter_task_get_requisition (UfoTask *task, UfoBuffer **inputs, UfoRequisition *requisition, GError **error) { ufo_buffer_get_requisition (inputs[0], requisition); -/* int i; -// UfoRoofPlaneTaskPrivate *priv; - UfoRequisition in_req; - -// priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (task); - ufo_buffer_get_requisition (inputs[0], &in_req); - - requisition->n_dims = in_req.n_dims; - for (i = 0; i < in_req.n_dims; i++) - requisition->dims[i] = in_req.dims[i];*/ } static guint -ufo_roof_plane_task_get_num_inputs (UfoTask *task) +ufo_roof_filter_task_get_num_inputs (UfoTask *task) { return 1; } static guint -ufo_roof_plane_task_get_num_dimensions (UfoTask *task, +ufo_roof_filter_task_get_num_dimensions (UfoTask *task, guint input) { return 2; } static UfoTaskMode -ufo_roof_plane_task_get_mode (UfoTask *task) +ufo_roof_filter_task_get_mode (UfoTask *task) { return UFO_TASK_MODE_CPU|UFO_TASK_MODE_REDUCTOR; } static gboolean -ufo_roof_plane_task_process (UfoTask *task, +ufo_roof_filter_task_process (UfoTask *task, UfoBuffer **inputs, UfoBuffer *output, UfoRequisition *requisition) { - UfoRoofPlaneTaskPrivate *priv; + UfoRoofFilterTaskPrivate *priv; - priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (task); + priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (task); if (priv->plane) { int buf_plane; @@ -128,13 +120,13 @@ ufo_roof_plane_task_process (UfoTask *task, } static gboolean -ufo_roof_plane_task_generate (UfoTask *task, +ufo_roof_filter_task_generate (UfoTask *task, UfoBuffer *output, UfoRequisition *requisition) { - UfoRoofPlaneTaskPrivate *priv; + UfoRoofFilterTaskPrivate *priv; - priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (task); + priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (task); if (!priv->ready) return FALSE; /* @@ -148,19 +140,24 @@ ufo_roof_plane_task_generate (UfoTask *task, return TRUE; } - static void -ufo_roof_plane_task_set_property (GObject *object, +ufo_roof_filter_task_set_property (GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { - UfoRoofPlaneTaskPrivate *priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (object); + UfoRoofFilterTaskPrivate *priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (object); switch (property_id) { + case PROP_BLOCK: + priv->block = g_value_get_boolean(value); + break; case PROP_PLANE: priv->plane = g_value_get_uint (value); break; + case PROP_FPS: + priv->fps = g_value_get_double (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -168,17 +165,23 @@ ufo_roof_plane_task_set_property (GObject *object, } static void -ufo_roof_plane_task_get_property (GObject *object, +ufo_roof_filter_task_get_property (GObject *object, guint property_id, GValue *value, GParamSpec *pspec) { - UfoRoofPlaneTaskPrivate *priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE (object); + UfoRoofFilterTaskPrivate *priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE (object); switch (property_id) { + case PROP_BLOCK: + g_value_set_boolean (value, priv->block); + break; case PROP_PLANE: g_value_set_uint (value, priv->plane); break; + case PROP_FPS: + g_value_set_double (value, priv->fps); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -186,32 +189,38 @@ ufo_roof_plane_task_get_property (GObject *object, } static void -ufo_roof_plane_task_finalize (GObject *object) +ufo_roof_filter_task_finalize (GObject *object) { - G_OBJECT_CLASS (ufo_roof_plane_task_parent_class)->finalize (object); + G_OBJECT_CLASS (ufo_roof_filter_task_parent_class)->finalize (object); } static void ufo_task_interface_init (UfoTaskIface *iface) { - iface->setup = ufo_roof_plane_task_setup; - iface->get_num_inputs = ufo_roof_plane_task_get_num_inputs; - iface->get_num_dimensions = ufo_roof_plane_task_get_num_dimensions; - iface->get_mode = ufo_roof_plane_task_get_mode; - iface->get_requisition = ufo_roof_plane_task_get_requisition; - iface->process = ufo_roof_plane_task_process; - iface->generate = ufo_roof_plane_task_generate; - + iface->setup = ufo_roof_filter_task_setup; + iface->get_num_inputs = ufo_roof_filter_task_get_num_inputs; + iface->get_num_dimensions = ufo_roof_filter_task_get_num_dimensions; + iface->get_mode = ufo_roof_filter_task_get_mode; + iface->get_requisition = ufo_roof_filter_task_get_requisition; + iface->process = ufo_roof_filter_task_process; + iface->generate = ufo_roof_filter_task_generate; } static void -ufo_roof_plane_task_class_init (UfoRoofPlaneTaskClass *klass) +ufo_roof_filter_task_class_init (UfoRoofFilterTaskClass *klass) { GObjectClass *oclass = G_OBJECT_CLASS (klass); - oclass->set_property = ufo_roof_plane_task_set_property; - oclass->get_property = ufo_roof_plane_task_get_property; - oclass->finalize = ufo_roof_plane_task_finalize; + oclass->set_property = ufo_roof_filter_task_set_property; + oclass->get_property = ufo_roof_filter_task_get_property; + oclass->finalize = ufo_roof_filter_task_finalize; + + properties[PROP_BLOCK] = + g_param_spec_boolean ("block", + "Block/Unblock data stream", + "Blocks all data output from the filter", + FALSE, + G_PARAM_READWRITE); properties[PROP_PLANE] = g_param_spec_uint ("plane", @@ -220,14 +229,21 @@ ufo_roof_plane_task_class_init (UfoRoofPlaneTaskClass *klass) 0, G_MAXUINT, 0, G_PARAM_READWRITE); + properties[PROP_FPS] = + g_param_spec_double ("fps", + "Maximum frame rate", + "Limits maximum frame rate", + 0, G_MAXDOUBLE, 0, + G_PARAM_READWRITE); + for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++) g_object_class_install_property (oclass, i, properties[i]); - g_type_class_add_private (oclass, sizeof(UfoRoofPlaneTaskPrivate)); + g_type_class_add_private (oclass, sizeof(UfoRoofFilterTaskPrivate)); } static void -ufo_roof_plane_task_init(UfoRoofPlaneTask *self) +ufo_roof_filter_task_init(UfoRoofFilterTask *self) { - self->priv = UFO_ROOF_PLANE_TASK_GET_PRIVATE(self); + self->priv = UFO_ROOF_FILTER_TASK_GET_PRIVATE(self); } diff --git a/src/ufo-roof-filter-task.h b/src/ufo-roof-filter-task.h new file mode 100644 index 0000000..920ba45 --- /dev/null +++ b/src/ufo-roof-filter-task.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2011-2013 Karlsruhe Institute of Technology + * + * This file is part of Ufo. + * + * This library is free software: you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef __UFO_ROOF_FILTER_TASK_H +#define __UFO_ROOF_FILTER_TASK_H + +#include <ufo/ufo.h> + +G_BEGIN_DECLS + +#define UFO_TYPE_ROOF_FILTER_TASK (ufo_roof_filter_task_get_type()) +#define UFO_ROOF_FILTER_TASK(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTask)) +#define UFO_IS_ROOF_FILTER_TASK(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), UFO_TYPE_ROOF_FILTER_TASK)) +#define UFO_ROOF_FILTER_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTaskClass)) +#define UFO_IS_ROOF_FILTER_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), UFO_TYPE_ROOF_FILTER_TASK)) +#define UFO_ROOF_FILTER_TASK_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), UFO_TYPE_ROOF_FILTER_TASK, UfoRoofFilterTaskClass)) + +typedef struct _UfoRoofFilterTask UfoRoofFilterTask; +typedef struct _UfoRoofFilterTaskClass UfoRoofFilterTaskClass; +typedef struct _UfoRoofFilterTaskPrivate UfoRoofFilterTaskPrivate; + +struct _UfoRoofFilterTask { + UfoTaskNode parent_instance; + + UfoRoofFilterTaskPrivate *priv; +}; + +struct _UfoRoofFilterTaskClass { + UfoTaskNodeClass parent_class; +}; + +UfoNode *ufo_roof_filter_task_new (void); +GType ufo_roof_filter_task_get_type (void); + +G_END_DECLS + +#endif diff --git a/src/ufo-roof-plane-task.h b/src/ufo-roof-plane-task.h deleted file mode 100644 index 7794065..0000000 --- a/src/ufo-roof-plane-task.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2011-2013 Karlsruhe Institute of Technology - * - * This file is part of Ufo. - * - * This library is free software: you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation, either - * version 3 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library. If not, see <http://www.gnu.org/licenses/>. - */ - -#ifndef __UFO_ROOF_PLANE_TASK_H -#define __UFO_ROOF_PLANE_TASK_H - -#include <ufo/ufo.h> - -G_BEGIN_DECLS - -#define UFO_TYPE_ROOF_PLANE_TASK (ufo_roof_plane_task_get_type()) -#define UFO_ROOF_PLANE_TASK(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTask)) -#define UFO_IS_ROOF_PLANE_TASK(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), UFO_TYPE_ROOF_PLANE_TASK)) -#define UFO_ROOF_PLANE_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTaskClass)) -#define UFO_IS_ROOF_PLANE_TASK_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), UFO_TYPE_ROOF_PLANE_TASK)) -#define UFO_ROOF_PLANE_TASK_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), UFO_TYPE_ROOF_PLANE_TASK, UfoRoofPlaneTaskClass)) - -typedef struct _UfoRoofPlaneTask UfoRoofPlaneTask; -typedef struct _UfoRoofPlaneTaskClass UfoRoofPlaneTaskClass; -typedef struct _UfoRoofPlaneTaskPrivate UfoRoofPlaneTaskPrivate; - -struct _UfoRoofPlaneTask { - UfoTaskNode parent_instance; - - UfoRoofPlaneTaskPrivate *priv; -}; - -struct _UfoRoofPlaneTaskClass { - UfoTaskNodeClass parent_class; -}; - -UfoNode *ufo_roof_plane_task_new (void); -GType ufo_roof_plane_task_get_type (void); - -G_END_DECLS - -#endif diff --git a/src/ufo-roof-read-file.c b/src/ufo-roof-read-file.c index a5eb69b..4ee11c6 100644 --- a/src/ufo-roof-read-file.c +++ b/src/ufo-roof-read-file.c @@ -55,7 +55,7 @@ static guint ufo_roof_read_file(UfoRoofReadInterface *iface, uint8_t *buffers, G } -UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GError **error) { +UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, const char *path, guint file_id, GError **error) { UfoRoofReadFile *reader = (UfoRoofReadFile*)calloc(1, sizeof(UfoRoofReadFile)); if (!reader) roof_new_error(error, "Can't allocate UfoRoofReadFile"); @@ -67,7 +67,7 @@ UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GErro reader->iface.close = ufo_roof_read_file_free; reader->iface.read =ufo_roof_read_file; - reader->fname = g_strdup_printf(cfg->path, id + cfg->first_file_number); + reader->fname = g_strdup_printf(path, file_id); if (!reader->fname) { free(reader); roof_new_error(error, "Can't build file name"); @@ -77,7 +77,7 @@ UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GErro if (!reader->fd) { g_free(reader->fname); g_free(reader); - roof_new_error(error, "Can't open file %s", reader->fname); + roof_new_error(error, "Can't open file %i at path %s", file_id, path); } return (UfoRoofReadInterface*)reader; diff --git a/src/ufo-roof-read-file.h b/src/ufo-roof-read-file.h index 54bcf49..787b441 100644 --- a/src/ufo-roof-read-file.h +++ b/src/ufo-roof-read-file.h @@ -3,6 +3,6 @@ #include "ufo-roof-read.h" -UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, guint id, GError **error); +UfoRoofReadInterface *ufo_roof_read_file_new(UfoRoofConfig *cfg, const char *path, guint file_id, GError **error); #endif diff --git a/src/ufo-roof-read-task.c b/src/ufo-roof-read-task.c index 1582437..a8ddded 100644 --- a/src/ufo-roof-read-task.c +++ b/src/ufo-roof-read-task.c @@ -39,6 +39,9 @@ struct _UfoRoofReadTaskPrivate { guint id; // Reader ID (defince sequential port number) gboolean stop; // Flag requiring termination + gboolean simulate; // Indicates if we are running in network or simulation modes + gchar *path; // UFO file path for simulation mode + guint first_file_number; // Number of a first simulated file (0 or 1) }; static void ufo_task_interface_init (UfoTaskIface *iface); @@ -54,6 +57,9 @@ enum { PROP_ID, PROP_STOP, PROP_CONFIG, + PROP_SIMULATE, + PROP_PATH, + PROP_FIRST, N_PROPERTIES }; @@ -77,7 +83,7 @@ ufo_roof_read_task_setup (UfoTask *task, if (!priv->config) roof_setup_error(error, "ROOF configuration is not specified"); - priv->cfg = ufo_roof_config_new(priv->config, &gerr); + priv->cfg = ufo_roof_config_new(priv->config, priv->simulate?UFO_ROOF_CONFIG_SIMULATION:UFO_ROOF_CONFIG_DEFAULT, &gerr); if (!priv->cfg) roof_propagate_error(error, gerr, "roof_config_new: "); // Consistency checks @@ -85,9 +91,12 @@ ufo_roof_read_task_setup (UfoTask *task, roof_setup_error(error, "Specified Stream ID is %u, but only %u data streams is configured", priv->id, priv->cfg->n_streams); // Start actual reader - if (priv->cfg->path) - priv->reader = ufo_roof_read_file_new(priv->cfg, priv->id, &gerr); - else + if (priv->simulate) { + if (!priv->path) + roof_setup_error(error, "Path to simulated data should be specified"); + + priv->reader = ufo_roof_read_file_new(priv->cfg, priv->path, priv->id + priv->first_file_number, &gerr); + } else priv->reader = ufo_roof_read_socket_new(priv->cfg, priv->id, &gerr); if (!priv->reader) @@ -115,6 +124,11 @@ ufo_roof_read_task_finalize (GObject *object) priv->config = NULL; } + if (priv->path) { + g_free(priv->path); + priv->path = NULL; + } + G_OBJECT_CLASS (ufo_roof_read_task_parent_class)->finalize (object); } @@ -206,6 +220,16 @@ ufo_roof_read_task_set_property (GObject *object, case PROP_STOP: priv->stop = g_value_get_boolean (value); break; + case PROP_SIMULATE: + priv->simulate = g_value_get_boolean (value); + break; + case PROP_PATH: + if (priv->path) g_free(priv->path); + priv->path = g_value_dup_string(value); + break; + case PROP_FIRST: + priv->first_file_number = g_value_get_uint (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -230,6 +254,15 @@ ufo_roof_read_task_get_property (GObject *object, case PROP_STOP: g_value_set_boolean (value, priv->stop); break; + case PROP_SIMULATE: + g_value_set_boolean (value, priv->simulate); + break; + case PROP_PATH: + g_value_set_string(value, priv->path?priv->path:""); + break; + case PROP_FIRST: + g_value_set_uint (value, priv->first_file_number); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -277,6 +310,27 @@ ufo_roof_read_task_class_init (UfoRoofReadTaskClass *klass) FALSE, G_PARAM_READWRITE); + properties[PROP_SIMULATE] = + g_param_spec_boolean ("simulate", + "Simulation mode", + "Read data from the specified files instead of network", + FALSE, + G_PARAM_READWRITE); + + properties[PROP_PATH] = + g_param_spec_string ("path", + "Input files for simulation mode", + "Optional path to input files for simulation mode (parameter from configuration file is used if not specified)", + "", + G_PARAM_READWRITE); + + properties[PROP_FIRST] = + g_param_spec_uint ("first_file_number", + "Offset to the first read file", + "Offset to the first read file", + 0, G_MAXUINT, 0, + G_PARAM_READWRITE); + for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++) g_object_class_install_property (oclass, i, properties[i]); diff --git a/tests/config.sh b/tests/config.sh index 3d5dbba..7b9d7d8 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -3,6 +3,8 @@ el7=$(($? == 0)) arch="" [ $el7 -ne 0 ] && arch="64" +[ -f /etc/gentoo-release ] && arch="64" + ods_path=/mnt/ands/ods/bin-fedora/ vma_path=/mnt/ands/ diff --git a/tests/roof-net.sh b/tests/roof-net.sh deleted file mode 100755 index 66faa43..0000000 --- a/tests/roof-net.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /bin/bash - -. config.sh - -bufs=800000 -bufs=$((bufs * 4)) - -#cat roof.yaml | sed '/simulation/,$d' | yq . > roof.json - -ulimit -l unlimited -echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399 -echo 8000 > /proc/sys/vm/nr_hugepages # 0 - -#VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=$bufs LD_PRELOAD=$vma_lib \ - LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib64" GI_TYPELIB_PATH="/usr/local/lib64/girepository-1.0/" \ - python3 roof.py -c roofhw.json "$@" - - -# python3 roof.py "$@" diff --git a/tests/roof-sim.sh b/tests/roof-sim.sh deleted file mode 100755 index 4374221..0000000 --- a/tests/roof-sim.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -. config.sh - -#cat roof.yaml | yq r - -j | jq '' | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json -cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json - -LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib$arch" GI_TYPELIB_PATH="/usr/local/lib$arch/girepository-1.0/" \ - python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino-%03i.tif" -n 1 "$@" -# python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino-%03i.raw" -n 1 "$@" - -# python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino%i.tif" -n 1 "$@" - -# python3 roof.py -c roof.json "$@" diff --git a/tests/roof-vma.sh b/tests/roof-vma.sh new file mode 120000 index 0000000..2faff84 --- /dev/null +++ b/tests/roof-vma.sh @@ -0,0 +1 @@ +roof.sh
\ No newline at end of file diff --git a/tests/roof.json b/tests/roof.json index d39a90f..1848bc6 100644 --- a/tests/roof.json +++ b/tests/roof.json @@ -14,10 +14,6 @@ "delta_x": 500, "delta_z": 1200 }, - "reconstruction": { - "parallel_projections": 512, - "parallel_bins": 256 - }, "network": { "protocol": "udp", "port": 52067, @@ -29,9 +25,21 @@ "buffer_size": 10, "packets_at_once": 100 }, + "data": { + "base_path": "/home/csa/roof2_data/test_data" + }, "simulation": { - "path": "/home/csa/roof2_data/test_data/data_%02u.dat", "first_file_number": 1, + "data": "sim/data_%02u.dat", + "flat_fields": "sim/flat_%02u.dat", + "dark_fields": "sim/dark_%02u.dat", "header_size": 0 + }, + "correction": { + "aggregation": "median" + }, + "reconstruction": { + "parallel_projections": 512, + "parallel_bins": 256 } } diff --git a/tests/roof.py b/tests/roof.py index 1941aa8..71b4465 100644 --- a/tests/roof.py +++ b/tests/roof.py @@ -1,79 +1,14 @@ -import gi -import re +#import gi import sys -import json -import argparse +from roof.graph import RoofGraph -gi.require_version('Ufo', '0.0') -from gi.repository import Ufo -from gi.repository import GObject +#gi.require_version('Ufo', '0.0') +#from gi.repository import Ufo +#from gi.repository import GObject -class RoofConfig: - def __init__(self, args, config="roof.json"): - self.streams = 1 - self.bit_depth = 8 - self.convert = False if ((not args.output) or (re.compile('\.raw$').search(args.output))) else True - self.build = "raw" if args.noroof else "ufo" if self.convert else "sino" - with open(config) as json_file: - cfg = json.load(json_file) - if cfg.get("network", {}).get("streams") != None: - self.streams = cfg["network"]["streams"] - elif cfg.get("hardware", {}).get("modules") != None: - self.streams = cfg["setup"]["modules"] +roof = RoofGraph() +graph = roof.get() - if cfg.get("hardware", {}).get("bit_depth") != None: - self.bit_depth = cfg["hardware"]["bit_depth"] - -parser = argparse.ArgumentParser() -parser.add_argument('-c', '--config', dest="config", default="roof.json", help="ROOF configuration (JSON)") -parser.add_argument('-o', '--output', dest="output", default=None, help="Output file") -parser.add_argument('-n', '--number', dest="number", default=None, type=int, help="Specify number of frames to capture (limits number of captured frames irrespective of further filtering)") -parser.add_argument('-p', '--plane', dest="plane", default=None, type=int, help="Only process the specified detector plane (indexed from 1)") -parser.add_argument( '--no-roof', dest="noroof", default=False, type=bool, help="Disable ROOF, only network testing (no sinogram building, store linearly)") -#parser.add_argument('-r', '--raw', dest="raw", default=False, type=bool, help="Store raw data, ignore processed") -#parser.add_argument('-v', '--visualize', dest='visualize', default=False, type=bool, help="Visualize data") -args = parser.parse_args() - - -cfg = RoofConfig(args, args.config) - -pm = Ufo.PluginManager() -graph = Ufo.TaskGraph() -scheduler = Ufo.Scheduler() - -if args.output is None: - print ("Starting ROOF using NULL writter") - write = pm.get_task('null') - if args.number is None: args.number = 0 -else: - print ("Starting ROOF streaming to {}".format(args.output)) - write = pm.get_task('write') - write.set_properties(filename=args.output) - if args.number is None: args.number = 5 - -build = pm.get_task('roof-build') -build.set_properties(config=args.config, number=args.number, build=cfg.build) - -plane = pm.get_task('roof-plane') if args.plane else None -if plane: plane.set_properties(plane=args.plane) - -for id in range(cfg.streams): - read = pm.get_task('roof-read') - read.set_properties(config=args.config, id=id) - graph.connect_nodes(read, build) - build.bind_property('stop', read, 'stop', GObject.BindingFlags.DEFAULT) - -#read_task.set_properties(path='/home/data/*.tif', start=10, number=100) -#graph.connect_nodes_full(read, write, 0) - -if plane: - graph.connect_nodes(build, plane) - graph.connect_nodes(plane, write) -else: - graph.connect_nodes(build, write) - - - -scheduler.run(graph) +roof.run() diff --git a/tests/roof.sh b/tests/roof.sh new file mode 100755 index 0000000..a8fa89e --- /dev/null +++ b/tests/roof.sh @@ -0,0 +1,32 @@ +#! /bin/bash + +. config.sh + +function pyroof { + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib$arch" GI_TYPELIB_PATH="/usr/local/lib$arch/girepository-1.0/" \ + python3 roof.py "$@" +} + +if [[ "$0" =~ roof-vma ]]; then + function roof { + bufs=800000 + bufs=$((bufs * 4)) + + ulimit -l unlimited + echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399 + echo 8000 > /proc/sys/vm/nr_hugepages # 0 + + VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=$bufs LD_PRELOAD=$vma_lib \ + pyroof "$@" + } +else + function roof { + pyroof "$@" + } +fi + +#cat roof.yaml | sed '/simulation/,$d' | yq . > roof.json +#cat roof.yaml | yq r - -j | jq '' | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json +cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json + +roof "$@" diff --git a/tests/roof.yaml b/tests/roof.yaml index d8a1c92..0a0ce1d 100644 --- a/tests/roof.yaml +++ b/tests/roof.yaml @@ -14,11 +14,6 @@ geometry: # source_angle_config: "path.xxx" delta_x: 500 delta_z: 1200 -#optics: -# flat_field_config: "path.xxx" -reconstruction: - parallel_projections: 512 - parallel_bins: 256 network: protocol: udp port: 52067 @@ -32,9 +27,32 @@ performance: buffer_size: 10 # drop_buffers: 0 packets_at_once: 100 +data: + base_path: "/home/csa/roof2_data/test_data" +# first_file_number: 1 +# flat_fields: "flats/flat_%04u.tif" +# dark_fields: "darks/dark_%04u.tif" +# raw_sinograms: "raw/sino_%04u.raw" +# fan_sinograms: "fan/sino_%04u.raw" +# parallel_sinograms: "par/sino_%04u.raw" +# filtered_sinograms: "flt/sino_%04u.raw" +# slices: "slices/slice_%04u.raw" + #rings: ... simulation: - path: "/home/csa/roof2_data/test_data/data_%02u.dat" first_file_number: 1 +# base_path: "/home/csa/roof2_data/test_data" + data: "sim/data_%02u.dat" + flat_fields: "sim/flat_%02u.dat" + dark_fields: "sim/dark_%02u.dat" header_size: 0 # max_packet_size: 1284 # dataset_size: 1024000 +correction: + aggregation: "median" +reconstruction: + parallel_projections: 512 + parallel_bins: 256 +# filters: [ "roof-fan2par", "fft", "filter", "ifft", "backproject" ] +# backproject-options: +#visualization: +#control: diff --git a/tests/roof/__init__.py b/tests/roof/__init__.py new file mode 100644 index 0000000..b8023d8 --- /dev/null +++ b/tests/roof/__init__.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/tests/roof/arguments.py b/tests/roof/arguments.py new file mode 100644 index 0000000..22ea42b --- /dev/null +++ b/tests/roof/arguments.py @@ -0,0 +1,32 @@ +import argparse +from roof.defaults import roof_data_types + +def roof_get_args(): + data_types = [] + for stage in roof_data_types: + data_types += roof_data_types[stage].keys() + data_types = set(data_types) + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', dest="config", default="roof.json", help="ROOF configuration (JSON)") + + # Modes + parser.add_argument('-s', '--simulate', dest="simulate", default=None, action="store_true", help="Simulation mode, read data from files instead of network") + parser.add_argument('-b', '--benchmark', dest="benchmark", default=None, action="store_true", help="Bencmarking mode, writes to /dev/null") + parser.add_argument('-g', '--gui', dest='gui', default=False, action="store_true", help="Visualize data") + parser.add_argument('-t', '--track', dest='track', default=False, action="store_true", help="Track & control experiment") + + parser.add_argument( '--no-roof', dest="noroof", default=False, action="store_true", help="Disable ROOF, only network testing (no sinogram building, store linearly)") + + # I/O + #parser.add_argument('-i', '--input', dest="input", default=None, help="Reconstruct from sinograms") + parser.add_argument('-o', '--output', dest="output", default=None, help="Output file(s)") + parser.add_argument('-r', '--read', dest="read", default=None, choices=data_types, nargs='?', const="raw_sinograms", help="Read recorded sinograms instead of listening on the network") + parser.add_argument('-w', '--write', dest="write", default=None, choices=data_types, nargs='?', const="slices", help="Only generate flat-fields, dark-fields, or sinograms (default)") + parser.add_argument( '--format', dest="format", default=None, help="Override default storage format") + + # Limits & Filtering + parser.add_argument('-n', '--number', dest="number", default=None, type=int, help="Specify number of frames to capture (limits number of captured frames irrespective of further filtering)") + parser.add_argument('-p', '--plane', dest="plane", default=None, type=int, help="Only process the specified detector plane (indexed from 1)") + + return parser.parse_args() diff --git a/tests/roof/config.py b/tests/roof/config.py new file mode 100644 index 0000000..e085ed8 --- /dev/null +++ b/tests/roof/config.py @@ -0,0 +1,67 @@ +import re +import json + +from roof.arguments import roof_get_args +from roof.defaults import roof_default_paths, roof_raw_data_types + +class RoofConfig: + def __init__(self, config=None): + self.args = roof_get_args() + self.config_file = self.get_arg('config', 'roof.json') if config is None else config + with open(self.config_file) as json_file: + self.cfg = json.load(json_file) + + self.path = self.get_opt('data', 'base_path', './') + self.planes = self.get_opt('hardware', 'planes', 1) + self.modules = self.get_opt('hardware', 'modules', None) + self.streams = self.get_opt('network', 'streams', 1 if self.modules is None else self.modules) + self.bit_depth = self.get_opt('hardware', 'bit_depth', 8) + + if self.args.number is None: self.args.number = 0 if self.args.benchmark else self.planes + + # Consistency and default mode + if (self.args.plane is not None) and (self.args.plane > self.planes): + raise ValueError("Only {} planes in configuration, but the plane {} is requested".format(self.planes, self.args.plane)) + + n_modes = (int(self.args.gui) + int(self.args.track) + int(0 if self.args.write is None else 1)) + if n_modes > 1: + raise ValueError("GUI, Control, and Write modes are mutualy incompatible") + elif n_modes == 0: + self.args.write = "raw_sinograms" + + + def get_arg(self, arg, default = None): + ret = getattr(self.args, arg) + return ret if ret is not None else default + + def get_opt(self, group, item, default = None): + if self.cfg.get(group, {}).get(item) != None: + return self.cfg[group][item] + else: + return default + + def get_roof_path(self, data_type): + subpath = self.get_opt('data', data_type) + if subpath is None: subpath = roof_default_paths[data_type] + if subpath is None: raise "Unknown data type %s is requested" % subpath + return subpath if subpath.startswith('/') else self.path + '/' + subpath + + def get_writer_type(self): + return None if self.args.benchmark else self.args.write if self.args.write else 'raw_sinograms' + + def get_writer_path(self): + data_type = self.get_writer_type() + if data_type is not None: + path = self.args.output if self.args.output is not None else self.get_roof_path(data_type) + if self.args.format: path = re.sub('\.([^.]+)$', '.' + self.args.format, path) + return path + return None + + def check_writer_type_is_raw_or_none(self): + data_type = self.get_writer_type() + data_path = self.get_writer_path() + return (data_type is None) or ((data_type in roof_raw_data_types) and re.search('\.raw$', data_path)) + + def check_writer_type_is_raw(self): + data_type = self.get_writer_type() + return (data_type is not None) and self.check_writer_type_is_raw_or_none() diff --git a/tests/roof/defaults.py b/tests/roof/defaults.py new file mode 100644 index 0000000..eed3fe5 --- /dev/null +++ b/tests/roof/defaults.py @@ -0,0 +1,42 @@ +roof_default_paths = { + 'flat_fields': "flats/flat_%04u.raw", + 'dark_fields': "darks/dark_%04u.raw", + 'raw_sinograms': "raw/sino_%04u.raw", + 'fan_sinograms': "fan/sino_%04u.tif", + 'parallel_sinograms': "par/sino_%04u.tif", + 'filtered_sinograms': "flt/sino_%04u.tif", + 'slices': "slices/slice_%04u.raw" +} + +#roof_default_simulation_paths = { +# 'data': "sim/data_%02u.dat", +# 'flat_fields': "sim/flat_%02u.dat", +# 'dark_fields': "sim/dark_%02u.dat" +#} + +roof_filters = { + 'correction': [ "flat-field-correct" ], + 'reconstruction': [ "roof-fan2par", "fft", "filter", "ifft", "backproject" ], + 'control': [ ], + 'visualization': [ ] +} + +roof_data_types = { + 'correction': { + 'flat_fields': 'flat-field-correct', + 'dark_fields': 'flat-field-correct', + 'raw_sinograms': 'flat-field-correct', + 'fan_sinograms': None + }, + 'reconstruction': { + 'fan_sinograms': 'roof-fan2par', + 'parallel_sinograms': 'fft', + 'filtered_sinograms': 'backproject', + 'slices': None + }, + 'control': { + } +} + +roof_raw_data_types = [k for k, v in roof_data_types['correction'].items() if v is not None ] +roof_aux_data_types = [v for v in roof_raw_data_types if 'sino' not in v ] diff --git a/tests/roof/graph.py b/tests/roof/graph.py new file mode 100644 index 0000000..c34a3ed --- /dev/null +++ b/tests/roof/graph.py @@ -0,0 +1,203 @@ +import re +import gi + +gi.require_version('Ufo', '0.0') +from gi.repository import Ufo +from gi.repository import GObject + +from roof.config import RoofConfig +from roof.defaults import roof_filters, roof_data_types, roof_raw_data_types, roof_aux_data_types +from roof.utils import get_filenames + +class RoofGraph(RoofConfig): + def __init__(self, config=None): + self.pm = Ufo.PluginManager() + self.graph = Ufo.TaskGraph() + self.scheduler = Ufo.Scheduler() + self.tasks = {} + + super(RoofGraph, self).__init__() + + def get_task(self, name, **kwargs): + task = self.pm.get_task(name) + task.set_properties(name, **kwargs) + return task + + def save_task(self, stage, alias, task): + if stage is None: stage = "general" + if stage not in self.tasks: self.tasks[stage] = {} + self.tasks[stage][alias if alias is not None else name] = task + return task + + def get_roof_task(self, name, **kwargs): + kwargs.update(config = self.config_file) + return self.get_task(name, **kwargs) + + def get_processor_task(self, stage, name, **kwargs): + extra_args = self.get_opt(stage, name + '-options') + if extra_args is not None: kwargs.update(extra_args) + if (re.compile('roof').match(name)): kwargs.update(config = self.config_file) + return self.save_task(stage, name, self.get_task(name, **kwargs)) + + def get_reader(self): + first = self.get_opt('data', 'first_file_number', 1) + if self.args.read: + # Reconstruction from standard UFO files + path = self.get_roof_path(self.args.read) + step = 1 + if (self.args.plane is not None) and (self.args.plane > 0): + first += self.args.plane - 1; + step = self.planes + + params = { 'path': path, 'first': first, 'step': step } + if self.args.number: + params['number'] = self.args.number + + print ("Reading {} data from {}".format(self.args.read,path)) + return self.get_task('read', **params) + else: + path = None + if self.args.simulate: + first = self.get_opt('simulation', 'first_file_number', first) + base_path = self.get_opt('simulation', 'base_path', self.path) + read_path = self.get_opt('simulation', self.args.write if self.args.write and self.args.write in roof_aux_data_types else 'data') + path = read_path if read_path.startswith('/') else base_path + '/' + read_path + print ("Simulating packets from {}".format(path)) + + # Reconstruction from network or simulated data (also generation of flat/dark-fields) + build_type = "raw" if self.args.noroof else "sino" if self.check_writer_type_is_raw() else "ufo" + build = self.get_roof_task('roof-build', simulate = self.args.simulate, number = self.args.number, build = build_type) + for id in range(self.streams): + read = self.get_roof_task('roof-read', id = id, simulate = self.args.simulate, path = path, first_file_number = first) + self.graph.connect_nodes(read, build) + build.bind_property('stop', read, 'stop', GObject.BindingFlags.DEFAULT) + + return build + + def get_writer(self): + path = self.get_writer_path() + if path is None: + print ("Starting ROOF using NULL writter") + write = self.get_task('null') + else: + # FIXME: If writting non raw data, we may need to generate all-0-frames if something broken/corrupted. + print ("Starting ROOF streaming to {}".format(path)) + write = self.get_task('write', filename=path) + return write + + def get_correction_flat_field_correct(self, head): + # Standard UFO reconstruction stack distinguish flat/dark-fields recorded before and after experiment. We only do 'before experiment' part. + darks = self.get_roof_path('dark_fields') + n_darks = len(get_filenames(darks)) + if n_darks == 0: raise FileNotFoundError("Dark fields are not found in {}".format(darks)) + flats = self.get_roof_path('falt_fields') + n_flats = len(get_filenames(flats)) + if n_flats == 0: raise FileNotFoundError("Flat fields are not found in {}".format(flats)) + dark_reader = self.get_task('read', path = darks) + flat_reader = self.get_task('read', path = flats) + + # We are using standard get_task here because this is too generic plugin to allow config-based customization + mode = self.get_opt('correction', 'aggregation', 'average') + if mode == 'median': + dark_stack = self.get_task('stack', number = n_darks) + dark_reduced = self.get_task('flatten', mode = 'median') + flat_stack = self.get_task('stack', number = n_flats) + flat_reduced = self.get_task('flatten', mode = 'median') + + self.graph.connect_nodes(dark_reader, dark_stack) + self.graph.connect_nodes(dark_stack, dark_reduced) + self.graph.connect_nodes(flat_reader, flat_stack) + self.graph.connect_nodes(flat_stack, flat_reduced) + elif mode == 'average': + dark_reduced = self.get_task('average') + flat_reduced = self.get_task('average') + self.graph.connect_nodes(dark_reader, dark_reduced) + self.graph.connect_nodes(flat_reader, flat_reduced) + else: + raise ValueError('Invalid reduction mode') + + ffc = self.get_task('flat-field-correct') # dark_scale=args.dark_scale, absorption_correct=args.absorptivity, fix_nan_and_inf=args.fix_nan_and_inf) + self.graph.connect_nodes_full(head, ffc, 0) + self.graph.connect_nodes_full(dark_reduced, ffc, 1) + self.graph.connect_nodes_full(flat_reduced, ffc, 2) + return ffc + + def get_processor(self, head, stage, writer = None): + # skip (but not if not already skipped in previous processor) + # how to connect readers to ffc? + + filters = self.get_opt(stage, 'filters', roof_filters[stage]) + read_here = self.args.read and self.args.read in roof_data_types[stage].keys() + write_here = self.args.write and self.args.write in roof_data_types[stage].keys() + + start_pos = 0 + if read_here: + start_filter = roof_data_types[stage][self.args.read] + start_pos = filters.index(start_filter) + + last_pos = len(filters) + if write_here: + stop_filter = roof_data_types[stage][self.args.write] + if stop_filter: last_pos = filters.index(stop_filter) + + # Will just execute empty range if we start reading from the end (e.g. 'fan-sinograms' in correction) + for i in range(start_pos, last_pos): + method = 'get_' + stage + '_' + filters[i].replace('-','_') + if method in dir(self): + f = getattr(self, method)(head) + else: + f = self.get_processor_task(stage, filters[pos]) + graph.connect_nodes(head, f) + head = f + + if write_here and writer: + self.graph.connect_nodes(head, writer) + + return None if write_here else head + + + def get(self): + reader = self.get_reader() + writer = self.get_writer() + + # We support following operation modes (defined by modifiers -w -c -g ] + # - Record mode: Writting raw data (raw-sinograms, flat-fields, dark-fields) [ no modified or -w <...> ] + # - Write mode: The reconstruction is performed and data is written after the specified step (default) [ -w <all other data types> ] + # - Control mode: Control branch and raw data writting [ -c ] + # - GUI mode: Visualization in GUI + raw_sinograms are written when enabled in GUI + some control tasks (also when enabled) [ -g ] + + head = reader + # Check if we are branching here + if (self.args.track or self.args.gui) and (self.get_data_type() is not None): + # FIXME: In GUI mode we can add here a 'write filter' to pause/resume writting. Alternative is to pass gobject flaga to fastwriter (this will be limited to fastwriter, then, which is likely OK) + # FIXME: we may need to convert in the end if we are writing raw data and the data is comming from net/simulation + # In other case (non branch), either we have already converted (in reader) or we don't need to convert (writing raw data). Small performance penalty if we convert before filter, but .... + copy = Ufo.CopyTask() + self.graph.connect_nodes(reader, copy) + self.graph.connect_nodes(copy, writer) + head = copy + + # Sinograms are already filtered in the reader + if not self.args.read: + main_filter = self.get_task('roof-filter', plane = self.args.plane) if self.args.plane else None + if main_filter: + self.graph.connect_nodes(head, main_filter) + head = main_filter + + class finish(Exception): pass + try: + if not self.args.read or self.args.read in roof_data_types['correction'].keys(): + head = self.get_processor(head, 'correction', writer) + if not head: raise finish() + + if head != reader or self.args.read in roof_data_types['reconstruction'].keys(): + head = self.get_processor(head, 'reconstruction', writer) + if not head: raise finish() + + # if head split to 3 branches.... Otherwise, continue with control branch... + except finish: + pass + + def run(self): + self.scheduler.run(self.graph) +
\ No newline at end of file diff --git a/tests/roof/utils.py b/tests/roof/utils.py new file mode 100644 index 0000000..eb389ed --- /dev/null +++ b/tests/roof/utils.py @@ -0,0 +1,13 @@ +import glob +import logging +import math +import os + +def get_filenames(path): + """Get all filenams from *path*, which could be a directory or a pattern + for matching files in a directory. + """ + if os.path.isdir(path): + path = os.path.join(path, '*') + + return sorted(glob.glob(path)) |