From 616c54df3a4c6126c32d1cb9a9e060f8064e6272 Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Thu, 25 Jun 2026 22:03:51 +0200 Subject: [PATCH] [Algorithm] Remove unused parser and TableView utilities The header-only utilities Parser.h (ForwardParser/ReverseParser), TableView.h, O2FormatParser.h and PageParser.h have no users: the only o2::algorithm:: symbol referenced anywhere outside the module (across AliceO2 and O2Physics) is flatten, from FlattenRestore.h. These four headers are exercised solely by their own unit tests, and the #include "Algorithm/Parser.h" lines in GPU/TPC sources reference none of their symbols. Remove the headers, their tests, the corresponding o2_add_test entries, and the stale includes. RangeTokenizer.h, FlattenRestore.h, HeaderStack.h and mpl_tools.h (the parts actually in use) are untouched. Co-Authored-By: Claude Opus 4.8 --- Algorithm/CMakeLists.txt | 24 - Algorithm/include/Algorithm/O2FormatParser.h | 93 ---- Algorithm/include/Algorithm/PageParser.h | 498 ------------------ Algorithm/include/Algorithm/Parser.h | 403 -------------- Algorithm/include/Algorithm/TableView.h | 350 ------------ Algorithm/test/StaticSequenceAllocator.h | 157 ------ Algorithm/test/o2formatparser.cxx | 82 --- Algorithm/test/pageparser.cxx | 246 --------- Algorithm/test/parser.cxx | 252 --------- Algorithm/test/tableview.cxx | 153 ------ .../Detectors/TPC/src/ClusterNativeHelper.cxx | 1 - Detectors/TPC/workflow/src/ZSSpec.cxx | 1 - .../include/GPUWorkflow/GPUWorkflowSpec.h | 1 - GPU/Workflow/src/GPUWorkflowSpec.cxx | 1 - GPU/Workflow/src/GPUWorkflowTPC.cxx | 1 - 15 files changed, 2263 deletions(-) delete mode 100644 Algorithm/include/Algorithm/O2FormatParser.h delete mode 100644 Algorithm/include/Algorithm/PageParser.h delete mode 100644 Algorithm/include/Algorithm/Parser.h delete mode 100644 Algorithm/include/Algorithm/TableView.h delete mode 100644 Algorithm/test/StaticSequenceAllocator.h delete mode 100644 Algorithm/test/o2formatparser.cxx delete mode 100644 Algorithm/test/pageparser.cxx delete mode 100644 Algorithm/test/parser.cxx delete mode 100644 Algorithm/test/tableview.cxx diff --git a/Algorithm/CMakeLists.txt b/Algorithm/CMakeLists.txt index ed7a42a96e528..6deed0fb8614b 100644 --- a/Algorithm/CMakeLists.txt +++ b/Algorithm/CMakeLists.txt @@ -11,36 +11,12 @@ o2_add_header_only_library(Algorithm INTERFACE_LINK_LIBRARIES O2::Headers) -o2_add_test(o2formatparser - SOURCES test/o2formatparser.cxx - COMPONENT_NAME Algorithm - PUBLIC_LINK_LIBRARIES O2::Algorithm - LABELS algorithm) - o2_add_test(headerstack SOURCES test/headerstack.cxx COMPONENT_NAME Algorithm PUBLIC_LINK_LIBRARIES O2::Algorithm LABELS algorithm) -o2_add_test(parser - SOURCES test/parser.cxx - COMPONENT_NAME Algorithm - PUBLIC_LINK_LIBRARIES O2::Algorithm - LABELS algorithm) - -o2_add_test(tableview - SOURCES test/tableview.cxx - COMPONENT_NAME Algorithm - PUBLIC_LINK_LIBRARIES O2::Algorithm - LABELS algorithm) - -o2_add_test(pageparser - SOURCES test/pageparser.cxx - COMPONENT_NAME Algorithm - PUBLIC_LINK_LIBRARIES O2::Algorithm - LABELS algorithm) - o2_add_test(mpl_tools SOURCES test/test_mpl_tools.cxx COMPONENT_NAME Algorithm diff --git a/Algorithm/include/Algorithm/O2FormatParser.h b/Algorithm/include/Algorithm/O2FormatParser.h deleted file mode 100644 index d0d820391a331..0000000000000 --- a/Algorithm/include/Algorithm/O2FormatParser.h +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef ALGORITHM_O2FORMATPARSER_H -#define ALGORITHM_O2FORMATPARSER_H - -/// @file O2FormatParser.h -/// @author Matthias Richter -/// @since 2017-10-18 -/// @brief Parser for the O2 data format - -#include "HeaderStack.h" - -namespace o2 -{ - -namespace algorithm -{ - -/** - * parse an input list and try to interpret in O2 data format - * O2 format consist of header-payload message pairs. The header message - * always starts with the DataHeader, optionally there can be more - * headers in the header stack. - * - * The following callbacks are mandadory to be provided, e.g. through lambdas - * - insert function with signature (const DataHeader&, ptr, size) - * auto insertFct = [&] (const auto & dataheader, - * auto ptr, - * auto size) { - * // do something with dataheader and buffer - * }; - * - getter for the message pointer, e.g. provided std::pair is used - * auto getPointerFct = [] (const auto & arg) {return arg.first;}; - * - getter for the message size, e.g. provided std::pair is used - * auto getSizeFct = [] (const auto & arg) {return arg.second;}; - * - * Optionally, also the header stack can be parsed by specifying further - * arguments. For every header supposed to be parsed, a pair of a dummy object - * and callback has to be specified, e.g. - * // handler callback for MyHeaderStruct - * auto onMyHeaderStruct = [&] (const auto & mystruct) { - * // do something with mystruct - * }; // end handler callback - * - * parseO2Format(list, insertFct, MyHeaderStruct(), onMyHeaderStruct); - * - */ -template < - typename InputListT, typename GetPointerFctT, typename GetSizeFctT, typename InsertFctT, // (const auto&, ptr, size) - typename... HeaderStackTypes // pairs of HeaderType and CallbackType - > -int parseO2Format(const InputListT& list, - GetPointerFctT getPointer, - GetSizeFctT getSize, - InsertFctT insert, - HeaderStackTypes&&... stackArgs) -{ - const o2::header::DataHeader* dh = nullptr; - for (auto& part : list) { - if (!dh) { - // new header - payload pair, read DataHeader - dh = o2::header::get(getPointer(part), getSize(part)); - if (!dh) { - return -ENOMSG; - } - o2::algorithm::dispatchHeaderStackCallback(getPointer(part), - getSize(part), - stackArgs...); - } else { - insert(*dh, getPointer(part), getSize(part)); - dh = nullptr; - } - } - if (dh) { - return -ENOMSG; - } - return list.size() / 2; -} - -} // namespace algorithm - -} // namespace o2 - -#endif // ALGORITHM_O2FORMATPARSER_H diff --git a/Algorithm/include/Algorithm/PageParser.h b/Algorithm/include/Algorithm/PageParser.h deleted file mode 100644 index 3ca01d87bcba3..0000000000000 --- a/Algorithm/include/Algorithm/PageParser.h +++ /dev/null @@ -1,498 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef ALGORITHM_PAGEPARSER_H -#define ALGORITHM_PAGEPARSER_H - -/// @file PageParser.h -/// @author Matthias Richter -/// @since 2017-09-27 -/// @brief Parser for a set of data objects in consecutive memory pages. - -#include -#include -#include -#include -#include - -namespace o2 -{ - -namespace algorithm -{ - -namespace pageparser -{ -// a function to extract the number of elements from the group type -// this is the version for all but integral types -template -typename std::enable_if::value, size_t>::type - extractNElements(T* v) -{ - return 0; -} - -// the specialization for integral types -template -typename std::enable_if::value, T>::type - extractNElements(T* v) -{ - return *v; -} - -template -using DefaultGetNElementsFctT = size_t (*)(const GroupT*); - -// the default function to extract the number of elements in a group -// where the group header is a single integral type holding number of -// elements -auto defaultGetNElementsFct = [](const auto* groupdata) { - using ReturnType = size_t; - using T = typename std::remove_pointer::type; - // this default function is only for integral types - static_assert(std::is_integral::value || std::is_void::value, - "A function for extracting the number of elements from the " - "group header must be specified for non-trivial types"); - // the default function for trivial integral types means there - // is exactly one number holding the size - return static_cast(extractNElements(groupdata)); -}; - -template -T* alloc() -{ - return new T; -} - -template <> -void* alloc() -{ - return nullptr; -} - -template -void free(T* ptr) -{ - if (ptr) { - delete ptr; - } -} - -template <> -void free(void*) -{ -} - -template -size_t sizeofGroupHeader() -{ - return sizeof(T); -} - -template <> -size_t sizeofGroupHeader() -{ - return 0; -} - -template -void set(T* h, size_t v) -{ - *h = v; -} - -template <> -void set(void*, size_t) -{ -} -} // namespace pageparser - -/** - * @class PageParser - * Parser for a set of data objects in consecutive memory pages. - * - * All memory pages have a fixed size and start with a page header. - * Depending on the page size and size of the data object, some - * objects can be split at the page boundary and have the page header - * embedded. - * - * The class iterator can be used to iterate over the data objects - * transparently. - * - * In addition data elements can be grouped. In that case a group - * header comes immediately after the first page header. The header - * has to store the number of elements which follow, a getter function - * has to be provided to retrieve the number from the header. - * - * In the most simple case, the group header consists of just one - * element of arbitrary integral type, the parser implements a default - * getter function to retrieve that number. The parser can be invoked - * by simply specifying an integral type as GroupT template parameter. - * - * Multiple blocks of grouped data elements can be in the group, a block - * can wrap over page boundery. A new block of grouped elements can - * however only start in a new page right after the page header. - * - * Usage: ungrouped elements - * RawParser RawParser; - * RawParser parser(ptr, size); - * for (auto element : parser) { - * // do something with element - * } - * - * Usage: grouped elements - * RawParser RawParser; - * RawParser parser(ptr, size); - * for (auto element : parser) { - * // do something with element - * } - */ -template > -class PageParser -{ - public: - using PageHeaderType = PageHeaderT; - using BufferType = unsigned char; - using value_type = ElementT; - using GroupType = GroupT; - using GetNElements = GetNElementsFctT; - static const size_t page_size = PageSize; - - // at the moment an object can only be split among two pages - static_assert(PageSize >= sizeof(PageHeaderType) + sizeof(value_type), - "Page Header and at least one element have to fit into page"); - - // switches for the copy method, used to skip ill-formed expressions - using TargetInPageBuffer = std::true_type; - using SourceInPageBuffer = std::false_type; - - PageParser() = delete; - template - PageParser(T* buffer, size_t size, - GetNElements getNElementsFct = pageparser::defaultGetNElementsFct) - : mBuffer(nullptr), mBufferIsConst(std::is_const::value), mSize(size), mGetNElementsFct(getNElementsFct), mNPages(size > 0 ? ((size - 1) / page_size) + 1 : 0), mGroupHeader(pageparser::alloc()) - { - static_assert(sizeof(T) == sizeof(BufferType), - "buffer required to be byte-type"); - - // the buffer pointer is stored non-const, a runtime check ensures - // that iterator write works only for non-const buffers - mBuffer = const_cast(buffer); - } - ~PageParser() - { - pageparser::free(mGroupHeader); - } - - template - class Iterator - { - public: - using ParentType = PageParser; - using SelfType = Iterator; - using iterator_category = std::forward_iterator_tag; - using value_type = T; - using reference = T&; - using pointer = T*; - using difference_type = std::ptrdiff_t; - using ElementType = typename std::remove_const::type; - - Iterator() = delete; - - Iterator(ParentType const* parent, size_t position = 0) - : mParent(parent) - { - mPosition = position; - size_t argument = mPosition; - if (!mParent->getElement(argument, mElement)) { - // eof, both mPosition and mNextPosition point to buffer end - mPosition = argument; - } - mNextPosition = argument; - backup(); - } - ~Iterator() - { - sync(); - } - - // prefix increment - SelfType& operator++() - { - sync(); - mPosition = mNextPosition; - size_t argument = mPosition; - if (!mParent->getElement(argument, mElement)) { - // eof, both mPosition and mNextPosition point to buffer end - mPosition = argument; - } - mNextPosition = argument; - backup(); - return *this; - } - // postfix increment - SelfType operator++(int /*unused*/) - { - SelfType copy(*this); - operator++(); - return copy; - } - // return reference - reference operator*() - { - return mElement; - } - // comparison - bool operator==(const SelfType& rh) const - { - return mPosition == rh.mPosition; - } - // comparison - bool operator!=(const SelfType& rh) const - { - return mPosition != rh.mPosition; - } - - const GroupType* getGroupHeader() const - { - return mParent->getGroupHeader(); - } - - private: - // sync method for non-const iterator - template - typename std::enable_if::value, U>::type sync() - { - if (std::memcmp(&mElement, &mBackup, sizeof(value_type)) != 0) { - // mElement is changed, sync to buffer - mParent->setElement(mPosition, mElement); - } - } - - // overload for const_iterator, empty function body - template - typename std::enable_if::value, U>::type sync() - { - } - - // backup for non-const iterator - template - typename std::enable_if::value, U>::type backup() - { - mBackup = mElement; - } - - // overload for const_iterator, empty function body - template - typename std::enable_if::value, U>::type backup() - { - } - - int mPosition; - int mNextPosition; - ParentType const* mParent; - ElementType mElement; - ElementType mBackup; - }; - - /// set an object at position - size_t setElement(size_t position, const value_type& element) const - { - // write functionality not yet implemented for grouped elements - assert(std::is_void::value); - // check if we are at the end - if (position >= mSize) { - assert(position == mSize); - return mSize; - } - - // check if there is space for one element - if (position + sizeof(value_type) > mSize) { - // format error, probably throw exception - return mSize; - } - - auto source = reinterpret_cast(&element); - auto target = mBuffer + position; - return position + copy(source, target, page_size - (position % page_size)); - } - - template - size_t readGroupHeader(size_t position, T* groupHeader) const - { - assert((position % page_size) == sizeof(PageHeaderType)); - if (std::is_void::value) { - return 0; - } - - memcpy(groupHeader, mBuffer + position, pageparser::sizeofGroupHeader()); - return mGetNElementsFct(groupHeader); - } - - /// retrieve an object at position - bool getElement(size_t& position, value_type& element) const - { - // check if we are at the end - if (position >= mSize) { - assert(position == mSize); - position = mSize; - return false; - } - - // handle group if defined - if (!std::is_void::value) { - if (mNGroupElements == 0) { - // new group has to be read from the buffer - do { - if ((position % page_size) == 0) { - position += sizeof(PageHeaderType); - } - if ((position % page_size) != sizeof(PageHeaderType)) { - // forward to the next page - position += page_size - (position % page_size) + sizeof(PageHeaderType); - if (position > mSize) { - //this is probably a valid condition as the group header can just - //indicate zero clusters - //throw std::runtime_error(""); - position = mSize; - return false; - } - } - const_cast(this)->mNGroupElements = readGroupHeader(position, mGroupHeader); - position += pageparser::sizeofGroupHeader(); - } while (mNGroupElements == 0); - - size_t nPages = 0; - size_t required = pageparser::sizeofGroupHeader() + mNGroupElements * sizeof(value_type); - do { - // the block of elements can go beyond the current page, find out - // how many additional pages are required - required += sizeof(PageHeaderType); - ++nPages; - } while (required > nPages * page_size); - required -= sizeof(PageHeaderType) + pageparser::sizeofGroupHeader(); - if (position + required > mSize) { - throw std::runtime_error( - "format error: the number of group elements " - "does not fit into the remaining buffer"); - } - } - // now we will read one element - const_cast(this)->mNGroupElements -= 1; - ; - } - - // check if there is space for one element - if (position + sizeof(value_type) > mSize) { - // FIXME: not sure if this is considered an error condition if - // no groups are used, i.e. the buffer should have the correct size - // and no extra space after the last element - position = mSize; - return false; - } - - auto source = mBuffer + position; - auto target = reinterpret_cast(&element); - position += copy(source, target, page_size - (position % page_size)); - return true; - } - - // copy data, depending on compile time switch, either source or target - // pointer are treated as pointer in the raw page, i.e. can be additionally - // incremented by the page header - template - size_t copy(const BufferType* source, BufferType* target, size_t pageCapacity) const - { - size_t position = 0; - auto copySize = sizeof(value_type); - // choose which of the pointers needs additional PageHeader offsets - auto pageOffsetTarget = SwitchT::value ? &target : const_cast(&source); - if (pageCapacity == page_size) { - // skip the page header at beginning of page - position += sizeof(PageHeaderType); - pageCapacity -= sizeof(PageHeaderType); - *pageOffsetTarget += sizeof(PageHeaderType); - } - if (copySize > pageCapacity) { - // object is split at the page boundary, copy the part - // in the current page first - copySize = pageCapacity; - } - if (copySize > 0) { - memcpy(target, source, copySize); - position += copySize; - source += copySize; - target += copySize; - } - copySize = sizeof(value_type) - copySize; - if (copySize > 0) { - // skip page header at beginning of new page and copy - // remaining part of the element - position += sizeof(PageHeaderType); - *pageOffsetTarget += sizeof(PageHeaderType); - memcpy(target, source, copySize); - position += copySize; - } - return position; - } - - const GroupType getGroupHeader() const - { - return mGroupHeader; - } - - using iterator = Iterator; - using const_iterator = Iterator; - - const_iterator begin() const - { - return const_iterator(this, 0); - } - - const_iterator end() const - { - return const_iterator(this, mSize); - } - - iterator begin() - { - if (mBufferIsConst) { - // did not find a way to do this at compile time in the constructor, - // probably one needs to make the buffer type a template parameter - // to the class - throw std::runtime_error("the underlying buffer is not writeable"); - } - return iterator(this, 0); - } - - iterator end() - { - return iterator(this, mSize); - } - - private: - BufferType* mBuffer = nullptr; - bool mBufferIsConst = false; - size_t mSize = 0; - GetNElements mGetNElementsFct = nullptr; - size_t mNPages = 0; - GroupType* mGroupHeader = nullptr; - size_t mNGroupElements = 0; -}; - -} // namespace algorithm -} // namespace o2 - -#endif diff --git a/Algorithm/include/Algorithm/Parser.h b/Algorithm/include/Algorithm/Parser.h deleted file mode 100644 index a2a7468621b4c..0000000000000 --- a/Algorithm/include/Algorithm/Parser.h +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef ALGORITHM_PARSER_H -#define ALGORITHM_PARSER_H - -/// @file Parser.h -/// @author Matthias Richter -/// @since 2017-09-20 -/// @brief Utilities for parsing of data sequences - -#include -#include - -namespace o2 -{ - -namespace algorithm -{ - -/// helper function returning size of type with a specialization for -/// void returning 0 -template -struct typesize { - static const size_t size = sizeof(T); -}; -// specialization for void -template <> -struct typesize { - static const size_t size = 0; -}; - -/** - * @class ForwardParser - * Parser for a sequence of frames with header, trailer and variable payload. - * The size is expected to be part of the header. - * - * Trailer type can be void, which is also the default template parameter. That - * allows to define a frame consisting of only header and data. - * - * Usage: - *
- *   using SomeParser = ForwardParser;
- *   SomeParser parser;
- *   std::vector frames;
- *   parser.parse(ptr, size,
- *                [] (const typename SomeParser::HeaderType& h) {
- *                  // check the header
- *                  return true;
- *                },
- *                [] (const typename SomeParser::TrailerType& t) {
- *                  // check the trailer
- *                  return true;
- *                },
- *                [] (const typename SomeParser::HeaderType& h) {
- *                  // get the size of the frame including payload
- *                  // and header and trailer size, e.g. payload size
- *                  // from a header member
- *                  return h.payloadSize + SomeParser::totalOffset;
- *                },
- *                [&frames] (typename SomeParser::FrameInfo& info) {
- *                  frames.emplace_back(info);
- *                  return true;
- *                }
- *                )
- *
- *   // a reduced version without trailer check callback
- *   using SomeParser = ForwardParser;
- *   SomeParser parser;
- *   std::vector frames;
- *   parser.parse(ptr, size,
- *                [] (const typename SomeParser::HeaderType& h) {
- *                  // check the header
- *                  return true;
- *                },
- *                [] (const typename SomeParser::HeaderType& h) {
- *                  // get the size of the frame including payload
- *                  // and header and trailer size, e.g. payload size
- *                  // from a header member
- *                  return h.payloadSize + SomeParser::totalOffset;
- *                },
- *                [&frames] (typename SomeParser::FrameInfo& info) {
- *                  frames.emplace_back(info);
- *                  return true;
- *                }
- *                )
- * 
- */ -template -class ForwardParser -{ - public: - using HeaderType = HeaderT; - using TrailerType = TrailerT; - using PayloadType = unsigned char; - - /// @struct FrameInfo - /// a compound of header, data, and trailer - struct FrameInfo { - using PtrT = const PayloadType*; - - const HeaderType* header = nullptr; - const TrailerType* trailer = nullptr; - PtrT payload = nullptr; - size_t length = 0; - }; - - /// the length offset due to header - static const size_t headOffset = typesize::size; - /// the length offset due to trailer - static const size_t tailOffset = typesize::size; - /// total length offset due to header and trailer - static const size_t totalOffset = headOffset + tailOffset; - - /// alias for callback checking the header, return true if the object - /// is a valid header - using CheckHeaderFct = std::function; - - /// alias for the argument type to be used in the CheckTrailer function - /// have to forward to a valid type in case of void TrailerType in order - /// to allow passing by reference - using CheckTrailerFctArgumentT = typename std::conditional< - !std::is_void::value, TrailerType, int>::type; - - /// alias for callback checking the trailer, takes reference to trailer - /// object if TrailerType is a valid type, no argument otherwise - template - using CheckTrailerFct = typename std::conditional< - !std::is_void::value, - std::function, - std::function>::type; - - /// alias for callback to get the complete frame size including header, - /// trailer and the data - using GetFrameSizeFct = std::function; - - /// function callback to insert/handle one frame into, sequentially called - /// for all frames if the whole block has a valid format - using InsertFct = std::function; - - /// Parse buffer of size bufferSize, requires callbacks to check header - /// trailer, the frame size, and insert callback to handle a FrameInfo - /// object. - template - int parse(const InputType* buffer, size_t bufferSize, - CheckHeaderFct checkHeader, - CheckTrailerFct checkTrailer, - GetFrameSizeFct getFrameSize, - InsertFct insert) - { - static_assert(sizeof(InputType) == 1, - "ForwardParser currently only supports byte type buffer"); - if (buffer == nullptr || bufferSize == 0) { - return 0; - } - - size_t position = 0; - std::vector frames; - do { - FrameInfo entry; - - // check the header - if (sizeof(HeaderType) + position > bufferSize) { - break; - } - entry.header = reinterpret_cast(buffer + position); - if (!checkHeader(*entry.header)) { - break; - } - - // extract frame size from header, this is expected to be the - // total frome size including header, payload and optional trailer - auto frameSize = getFrameSize(*entry.header); - if (frameSize + position > bufferSize) { - break; - } - - // payload starts right after the header - entry.payload = reinterpret_cast(entry.header + 1); - entry.length = frameSize - totalOffset; - - // optionally extract and check trailer - if (tailOffset > 0) { - entry.trailer = nullptr; - } else { - auto trailerStart = buffer + position + frameSize - tailOffset; - entry.trailer = reinterpret_cast(trailerStart); - if (!CheckTrailer(entry, checkTrailer)) { - break; - } - } - - // store the extracted frame info and continue with remaining buffer - frames.emplace_back(entry); - position += frameSize; - } while (position < bufferSize); - - if (position == bufferSize) { - // frames found and format consistent, insert entries to target - // Note: the complete block must be consistent - for (auto entry : frames) { - if (!insert(entry)) { - break; - } - } - return frames.size(); - } else if (frames.size() == 0) { - // no frames found at all, the buffer does not contain any - return 0; - } - - // format error detected - // TODO: decide about error policy - return -1; - } - - /// Parse buffer of size bufferSize, specialization skipping the trailer - /// check, e.g. when its type is void, or when the integrity of the trailer - /// is not relevant. Requires callbacks to check header, frame size, and - /// insert callback to handle a FrameInfo object. - template - typename std::enable_if::value, int>::type - parse(const InputType* buffer, size_t bufferSize, - CheckHeaderFct checkHeader, - GetFrameSizeFct getFrameSize, - InsertFct insert) - { - auto checkTrailer = []() { return true; }; - return parse(buffer, bufferSize, checkHeader, checkTrailer, getFrameSize, insert); - } - - private: - /// internal function to check the trailer, distinguishes void and non-void - /// trailer type. - template - typename std::enable_if::value, bool>::type - CheckTrailer(const FrameInfo& entry, CheckTrailerFct& checkTrailer) const - { - return checkTrailer(*entry.trailer); - } - - template - typename std::enable_if::value, bool>::type - CheckTrailer(const FrameInfo&, CheckTrailerFct&) const - { - return true; - } -}; - -/** - * @class ReverseParser - * Parser for a sequence of frames with header, trailer and variable payload. - * The size is expected to be part of the trailer, the parsing is thus in - * reverse direction. Also the insert callback is called with the entries - * starting form the end of the buffer. - * TODO: an easy extension can be to reverse the order of the inserts, meaning - * that the entries are read from the beginning. - * - * Usage: - *
- *   using SomeParser = ReverseParser;
- *   SomeParser parser;
- *   std::vector frames;
- *   parser.parse(ptr, size,
- *                [] (const typename SomeParser::HeaderType& h) {
- *                  // check the header
- *                  return true;
- *                },
- *                [] (const typename SomeParser::TrailerType& t) {
- *                  // check the trailer
- *                  return true;
- *                },
- *                [] (const typename SomeParser::TrailerType& t) {
- *                  // get the size of the frame including payload
- *                  // and header and trailer size, e.g. payload size
- *                  // from a trailer member
- *                  return t.payloadSize + SomeParser::totalOffset;
- *                },
- *                [&frames] (typename SomeParser::FrameInfo& info) {
- *                  frames.emplace_back(info);
- *                  return true;
- *                }
- *                )
- * 
- */ -template -class ReverseParser -{ - public: - using HeaderType = HeaderT; - using TrailerType = TrailerT; - using PayloadType = unsigned char; - - /// @struct FrameInfo a compound of header, data, and trailer - struct FrameInfo { - using PtrT = const PayloadType*; - - const HeaderType* header = nullptr; - const TrailerType* trailer = nullptr; - PtrT payload = nullptr; - size_t length = 0; - }; - /// the length offset due to header - static const size_t headOffset = typesize::size; - /// the length offset due to trailer - static const size_t tailOffset = typesize::size; - /// total length offset due to header and trailer - static const size_t totalOffset = headOffset + tailOffset; - - /// alias for callback checking the header, return true if the object - /// is a valid header - using CheckHeaderFct = std::function; - /// alias for callback checking the trailer - using CheckTrailerFct = std::function; - /// alias for callback to get the complete frame size including header, - /// trailer and the data - using GetFrameSizeFct = std::function; - /// function callback to insert/handle one frame into, sequentially called - /// for all frames if the whole block has a valid format - using InsertFct = std::function; - - /// Parse buffer of size bufferSize, requires callbacks to check header - /// trailer, the frame size, and insert callback to handle a FrameInfo - /// object. - template - int parse(const InputType* buffer, size_t bufferSize, - CheckHeaderFct checkHeader, - CheckTrailerFct checkTrailer, - GetFrameSizeFct getFrameSize, - InsertFct insert) - { - static_assert(sizeof(InputType) == 1, - "ReverseParser currently only supports byte type buffer"); - if (buffer == nullptr || bufferSize == 0) { - return 0; - } - auto position = bufferSize; - std::vector frames; - do { - FrameInfo entry; - - // start from end, extract and check trailer - if (sizeof(TrailerType) > position) { - break; - } - entry.trailer = reinterpret_cast(buffer + position - sizeof(TrailerType)); - if (!checkTrailer(*entry.trailer)) { - break; - } - - // get the total frame size - auto frameSize = getFrameSize(*entry.trailer); - if (frameSize > position) { - break; - } - - // extract and check header - auto headerStart = buffer + position - frameSize; - entry.header = reinterpret_cast(headerStart); - if (!checkHeader(*entry.header)) { - break; - } - - // payload immediately after header - entry.payload = reinterpret_cast(entry.header + 1); - entry.length = frameSize - sizeof(HeaderType) - sizeof(TrailerType); - frames.emplace_back(entry); - position -= frameSize; - } while (position > 0); - - if (position == 0) { - // frames found and format consistent, the complete block must be consistent - for (auto entry : frames) { - if (!insert(entry)) { - break; - } - } - return frames.size(); - } else if (frames.size() == 0) { - // no frames found at all, the buffer does not contain any - return 0; - } - - // format error detected - // TODO: decide about error policy - return -1; - } -}; - -} // namespace algorithm - -} // namespace o2 - -#endif // ALGORITHM_PARSER_H diff --git a/Algorithm/include/Algorithm/TableView.h b/Algorithm/include/Algorithm/TableView.h deleted file mode 100644 index 36980e64d1bc9..0000000000000 --- a/Algorithm/include/Algorithm/TableView.h +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef ALGORITHM_TABLEVIEW_H -#define ALGORITHM_TABLEVIEW_H - -/// @file TableView.h -/// @author Matthias Richter -/// @since 2017-09-21 -/// @brief Container class for multiple sequences of data wrapped by markers - -#include -#include - -namespace o2 -{ - -namespace algorithm -{ - -/** - * @class TableView - * Container class for multiple sequences of data wrapped by markers. - * - * This is a container for data sequences of multiple frames consisting - * of a header marker struct, a payload, and an optional trailer marker - * struct. Each sequence forms a row in the TableView, the columns - * are provided by the markers/frames. - * - * A parser is used to step through the data sequence and extract - * headers, trailers and payload positions. - * - * Requirements: - * - both header and trailer type must provide an operator bool() method - * to check validity - * - the size of one frame needs to be extracted either from the header - * marker or the trailer marker. The first requires forward, the latter - * backward parsing. In the first case, the trailer is optional, while - * in the latter required - * - */ -template -class TableView -{ - public: - TableView() = default; - ~TableView() = default; - - using RowDescType = RowDescT; - using ColumnIndexType = ColumnDescT; - using ParserType = ParserT; - - /// FrameIndex is composed from column description and row number - struct FrameIndex { - ColumnIndexType columnIndex; - unsigned row; - - bool operator<(const FrameIndex& rh) const - { - if (rh.columnIndex < columnIndex) { - return false; - } - if (columnIndex < rh.columnIndex) { - return true; - } - return row < rh.row; - } - }; - - /// descriptor pointing to payload of one frame - struct FrameData { - const std::byte* buffer = nullptr; - size_t size = 0; - }; - - /** - * Add a new data sequence, the set is traversed according to parser - * - * TODO: functors to check header and trailer validity as well as retrieving - * the frame size could be passed as arguments. - * - * @param rowData Descriptive data struct for the sequence - * @param seqData Pointer to sequence - * @param seqSize Length of sequence - * @return number of inserted elements - */ - size_t addRow(RowDescType rowData, std::byte* seqData, size_t seqSize) - { - unsigned nFrames = mFrames.size(); - unsigned currentRow = mRowData.size(); - ParserType p; - p.parse( - seqData, seqSize, - [](const typename ParserT::HeaderType& h) { return (h); }, - [](const typename ParserT::TrailerType& t) { return (t); }, - [](const typename ParserT::TrailerType& t) { - return t.dataLength + ParserT::totalOffset; - }, - [this, currentRow](typename ParserT::FrameInfo entry) { - // insert the header as column index in ascending order - auto position = mColumns.begin(); - while (position != mColumns.end() && *position < *entry.header) { - position++; - } - if (position == mColumns.end() || *entry.header < *position) { - mColumns.emplace(position, *entry.header); - } - - // insert frame descriptor under key composed from header and row - auto result = mFrames.emplace(FrameIndex{*entry.header, currentRow}, - FrameData{(std::byte*)entry.payload, entry.length}); - return result.second; - }); - auto insertedFrames = mFrames.size() - nFrames; - if (insertedFrames > 0) { - mRowData.emplace_back(rowData); - } - return insertedFrames; - } - - /// clear the index, i.e. all internal lists - void clear() - { - mFrames.clear(); - mColumns.clear(); - mRowData.clear(); - } - - /// get number of columns in the created index - size_t getNColumns() const { return mColumns.size(); } - - /// get number of rows, i.e. number rows in the created index - size_t getNRows() const { return mRowData.size(); } - - /// get row data for a data set - const RowDescType& getRowData(size_t row) const - { - if (row < mRowData.size()) { - return mRowData[row]; - } - // TODO: better to throw exception? - static RowDescType dummy; - return dummy; - } - - // TODO: - // instead of a member with this pointer of parent class, the access - // function was supposed to be specified as a lambda. This definition - // was supposed to be the type of the function member. - // passing the access function to the iterator did not work because - // the typedef for the access function is without the capture, so there - // is no matching conversion. - // Solution would be to use std::function but that's probably slow and - // the function is called often. Can be checked later. - typedef FrameData (*AccessFct)(unsigned, unsigned); - - /// Iterator class for configurable direction, i.e. either row or column - class iterator - { // TODO: derive from forward_iterator - public: - struct value_type : public FrameData { - RowDescType desc; - }; - using self_type = iterator; - - enum IteratorDirections { - kAlongRow, - kAlongColumn - }; - - iterator() = delete; - ~iterator() = default; - iterator(IteratorDirections direction, TableView* parent, unsigned row = 0, unsigned column = 0) - : mDirection(direction), mRow(row), mColumn(column), mEnd(direction == kAlongRow ? parent->getNColumns() : parent->getNRows()), mParent(parent), mCache(), mIsCached(false) - { - while (!isValid() && !isEnd()) { - operator++(); - } - } - - self_type& operator++() - { - mIsCached = false; - if (mDirection == kAlongRow) { - if (mColumn < mEnd) { - mColumn++; - } - } else { - if (mRow < mEnd) { - mRow++; - } - } - while (!isEnd() && !isValid()) { - operator++(); - } - return *this; - } - - value_type operator*() const - { - if (!mIsCached) { - self_type* ncthis = const_cast(this); - mParent->get(mRow, mColumn, ncthis->mCache); - ncthis->mCache.desc = mParent->getRowData(mRow); - ncthis->mIsCached = true; - } - return mCache; - } - - bool operator==(const self_type& other) const - { - return mDirection == kAlongRow ? (mColumn == other.mColumn) : (mRow == other.mRow); - } - - bool operator!=(const self_type& other) const - { - return mDirection == kAlongRow ? (mColumn != other.mColumn) : (mRow != other.mRow); - } - - bool isEnd() const - { - return (mDirection == kAlongRow) ? (mColumn >= mEnd) : (mRow >= mEnd); - } - - bool isValid() const - { - if (!mIsCached) { - self_type* ncthis = const_cast(this); - ncthis->mIsCached = mParent->get(mRow, mColumn, ncthis->mCache); - ncthis->mCache.desc = mParent->getRowData(mRow); - } - return mIsCached; - } - - protected: - IteratorDirections mDirection; - unsigned mRow; - unsigned mColumn; - unsigned mEnd; - TableView* mParent; - value_type mCache; - bool mIsCached; - }; - - /// iterator for the outer access of the index, either row or column direction - template - class outerIterator : public iterator - { - public: - using base = iterator; - using value_type = typename base::value_type; - using self_type = outerIterator; - static const unsigned direction = Direction; - - outerIterator() = delete; - ~outerIterator() = default; - outerIterator(TableView* parent, unsigned index) - : iterator(typename iterator::IteratorDirections(direction), parent, direction == iterator::kAlongColumn ? index : 0, direction == iterator::kAlongRow ? index : 0) - { - } - - self_type& operator++() - { - if (base::mDirection == iterator::kAlongRow) { - if (base::mColumn < base::mEnd) { - base::mColumn++; - } - } else { - if (base::mRow < base::mEnd) { - base::mRow++; - } - } - return *this; - } - - /// begin the inner iteration - iterator begin() - { - return iterator((base::mDirection == iterator::kAlongColumn) ? iterator::kAlongRow : iterator::kAlongColumn, - base::mParent, - (base::mDirection == iterator::kAlongColumn) ? base::mRow : 0, - (base::mDirection == iterator::kAlongRow) ? base::mColumn : 0); - } - - /// end of the inner iteration - iterator end() - { - return iterator((base::mDirection == iterator::kAlongColumn) ? iterator::kAlongRow : iterator::kAlongColumn, - base::mParent, - (base::mDirection == iterator::kAlongRow) ? base::mParent->getNRows() : 0, - (base::mDirection == iterator::kAlongColumn) ? base::mParent->getNColumns() : 0); - } - }; - - /// definition of the outer iterator over column - using ColumnIterator = outerIterator; - /// definition of the outer iterator over row - using RowIterator = outerIterator; - - /// begin of the outer iteration - ColumnIterator begin() - { - return ColumnIterator(this, 0); - } - - /// end of outer iteration - ColumnIterator end() - { - return ColumnIterator(this, mColumns.size()); - } - - private: - /// private access function for the iterators - bool get(unsigned row, unsigned column, FrameData& data) - { - if (this->mColumns.size() == 0) { - return false; - } - auto element = this->mFrames.find(FrameIndex{this->mColumns[column], row}); - if (element != this->mFrames.end()) { - data = element->second; - return true; - } - return false; - } - - /// map of frame descriptors with key composed from header and row number - std::map mFrames; - /// list of indices in row direction - std::vector mColumns; - /// data descriptor of each row forming the columns - std::vector mRowData; -}; - -} // namespace algorithm - -} // namespace o2 - -#endif // ALGORITHM_TABLEVIEW_H diff --git a/Algorithm/test/StaticSequenceAllocator.h b/Algorithm/test/StaticSequenceAllocator.h deleted file mode 100644 index 8a684159afdb9..0000000000000 --- a/Algorithm/test/StaticSequenceAllocator.h +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file StaticSequenceAllocator.h -/// @author Matthias Richter, based on work by Mikolaj Krzewicki -/// @since 2017-09-21 -/// @brief An allocator for static sequences of object types - -namespace o2 -{ -namespace algorithm -{ - -/** - * Helper struct to define a composite element from a header, some payload - * and a trailer - */ -template -struct Composite { - using HeaderType = HeaderT; - using TrailerType = TrailerT; - size_t compositeLength = 0; - size_t trailerLength = 0; - size_t dataLength = 0; - - template - constexpr Composite(const HeaderType h, const char (&d)[N], - typename std::conditional::value, const TrailerType, int>::type t, - typename std::enable_if::value>::type* = nullptr) - : header(h), data(d), trailer(t) - { - dataLength = N; - trailerLength = sizeof(TrailerType); - compositeLength = sizeof(HeaderType) + dataLength + trailerLength; - } - - template - constexpr Composite(const HeaderType& h, const char (&d)[N], - typename std::enable_if::value>::type* = nullptr) - : header(h), data(d) - { - dataLength = N; - trailerLength = 0; - compositeLength = sizeof(HeaderType) + dataLength + trailerLength; - } - - constexpr size_t getLength() const noexcept - { - return compositeLength; - } - - constexpr size_t getDataLength() const noexcept - { - return dataLength; - } - - template - constexpr size_t insert(BufferT* buffer) const noexcept - { - static_assert(sizeof(BufferT) == 1, "buffer required to be of byte-type"); - size_t length = 0; - memcpy(buffer + length, &header, sizeof(HeaderType)); - length += sizeof(HeaderType); - memcpy(buffer + length, data, dataLength); - length += dataLength; - if (trailerLength > 0) { - memcpy(buffer + length, &trailer, trailerLength); - length += trailerLength; - } - return length; - } - - const HeaderType header; - const char* data = nullptr; - typename std::conditional::value, const TrailerType, int>::type trailer; -}; - -/// recursively calculate the length of the sequence -/// object types are fixed at compile time and so is the total length of the -/// sequence. The function is recursively invoked for all arguments of the -// variable list -template -constexpr size_t sequenceLength(const T& first, const TArgs... args) noexcept -{ - return sequenceLength(first) + sequenceLength(args...); -} - -/// template secialization of sequence length calculation for one argument, -/// this is also the terminating instance for the last argument of the recursive -/// invocation of the function template. -template -constexpr size_t sequenceLength(const T& first) noexcept -{ - return first.getLength(); -} - -/// recursive insert of variable number of objects -template -constexpr size_t sequenceInsert(BufferT* buffer, const T& first, const TArgs... args) noexcept -{ - static_assert(sizeof(BufferT) == 1, "buffer required to be of byte-type"); - auto length = sequenceInsert(buffer, first); - length += sequenceInsert(buffer + length, args...); - return length; -} - -/// terminating template specialization, i.e. for the last element -template -constexpr size_t sequenceInsert(BufferT* buffer, const T& element) noexcept -{ - // TODO: make a general algorithm, at the moment this serves the - // Composite class as a special case - return element.insert(buffer); -} - -/** - * Allocator for a buffer of a static sequence of multiple objects. - * - * The sequence of object types is fixed at compile time and given as - * a variable list of arguments to the constructor. The data of the objects - * is runtime dependent. - * - * TODO: probably the Composite struct needs to be reworked to allow this - * allocator to be more general - */ -struct StaticSequenceAllocator { - using value_type = unsigned char; - using BufferType = std::unique_ptr; - - BufferType buffer; - size_t bufferSize; - - size_t size() const { return bufferSize; } - - StaticSequenceAllocator() = delete; - - template - StaticSequenceAllocator(Targs... args) - { - bufferSize = sequenceLength(args...); - buffer = std::make_unique(bufferSize); - sequenceInsert(buffer.get(), args...); - } -}; - -} // namespace algorithm -} // namespace o2 diff --git a/Algorithm/test/o2formatparser.cxx b/Algorithm/test/o2formatparser.cxx deleted file mode 100644 index 2896631ebc5b8..0000000000000 --- a/Algorithm/test/o2formatparser.cxx +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file o2formatparser.cxx -/// @author Matthias Richter -/// @since 2017-10-18 -/// @brief Unit test for O2 format parser - -#define BOOST_TEST_MODULE Test Algorithm HeaderStack -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK -#include -#include -#include -#include // memcmp -#include "Headers/DataHeader.h" // hexdump, DataHeader -#include "../include/Algorithm/O2FormatParser.h" - -template -void hexDump(Targs... Fargs) -{ - // a simple redirect to enable/disable the hexdump printout - o2::header::hexDump(Fargs...); -} - -BOOST_AUTO_TEST_CASE(test_o2formatparser) -{ - std::vector thedata = { - "I'm raw data", - "reconstructed data"}; - unsigned dataidx = 0; - std::vector dataheaders; - dataheaders.emplace_back(o2::header::DataDescription("RAWDATA"), - o2::header::DataOrigin("DET"), - 0, - strlen(thedata[dataidx++])); - dataheaders.emplace_back(o2::header::DataDescription("RECODATA"), - o2::header::DataOrigin("DET"), - 0, - strlen(thedata[dataidx++])); - - std::vector> messages; - for (dataidx = 0; dataidx < thedata.size(); ++dataidx) { - messages.emplace_back(reinterpret_cast(&dataheaders[dataidx]), - sizeof(o2::header::DataHeader)); - messages.emplace_back(thedata[dataidx], - dataheaders[dataidx].payloadSize); - } - - // handler callback for parseO2Format method - auto insertFct = [&](const auto& dataheader, - auto ptr, - auto size) { - hexDump("header", &dataheader, sizeof(dataheader)); - hexDump("data", ptr, size); - BOOST_CHECK(dataheader == dataheaders[dataidx]); - BOOST_CHECK(strncmp(ptr, thedata[dataidx], size) == 0); - ++dataidx; - }; // end handler callback - - // handler callback to get the pointer for message - auto getPointerFct = [](auto arg) { return arg.first; }; - // handler callback to get the size for message - auto getSizeFct = [](auto arg) { return arg.second; }; - - dataidx = 0; - auto result = o2::algorithm::parseO2Format(messages, - getPointerFct, - getSizeFct, - insertFct); - - BOOST_REQUIRE(result >= 0); - BOOST_CHECK(result == 2); -} diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx deleted file mode 100644 index 7551c32d9d864..0000000000000 --- a/Algorithm/test/pageparser.cxx +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file parser.cxx -/// @author Matthias Richter -/// @since 2017-09-27 -/// @brief Unit test for parser of objects in memory pages - -#define BOOST_TEST_MODULE Test Algorithm Parser -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK -#include -#include -#include -#include -#include "Headers/DataHeader.h" // hexdump -#include "../include/Algorithm/PageParser.h" -#include "StaticSequenceAllocator.h" - -struct PageHeader { - uint32_t magic = 0x45474150; - uint32_t pageid; - - PageHeader(uint32_t id) : pageid(id) {} -}; - -struct ClusterData { - uint32_t magic = 0x54534c43; - uint32_t clusterid; - uint16_t x; - uint16_t y; - uint16_t z; - uint8_t e; - - ClusterData() - : clusterid(0), x(0), y(0), z(0), e(0) - { - } - - ClusterData(uint32_t _id, uint16_t _x, uint16_t _y, uint16_t _z, uint8_t _e) - : clusterid(_id), x(_x), y(_y), z(_z), e(_e) - { - } - - bool operator==(const ClusterData& rhs) const - { - return clusterid == rhs.clusterid && x == rhs.x && y == rhs.y && z == rhs.z && e == rhs.e; - } -}; - -template -std::pair, size_t> MakeBuffer(size_t pagesize, - PageHeaderT pageheader, - const ListT& dataset) -{ - static_assert(std::is_void::value || std::is_integral::value, - "Invalid group type"); - auto totalSize = dataset.size() * sizeof(typename ListT::value_type); - totalSize += o2::algorithm::pageparser::sizeofGroupHeader(); - auto maxElementsPerPage = pagesize - (sizeof(pageheader) + o2::algorithm::pageparser::sizeofGroupHeader()); - maxElementsPerPage /= sizeof(typename ListT::value_type); - - if (std::is_void::value || !GroupHeaderPerPage) { - unsigned nPages = 0; - do { - totalSize += sizeof(PageHeaderT); - ++nPages; - } while (nPages * pagesize < totalSize); - } else { - auto nRequiredPages = dataset.size() / maxElementsPerPage; - if (dataset.size() % maxElementsPerPage > 0) { - ++nRequiredPages; - } - totalSize = (nRequiredPages > 0 ? nRequiredPages : 1) * pagesize; - } - - auto buffer = std::make_unique(totalSize); - memset(buffer.get(), 0, totalSize); - - unsigned position = 0; - auto target = buffer.get(); - GroupT* groupHeader = nullptr; - size_t nElementsInCurrentGroup = 0; - for (auto element : dataset) { - if (GroupHeaderPerPage && nElementsInCurrentGroup == maxElementsPerPage) { - // write the number of elements in the group and forward to next - // page boundary - o2::algorithm::pageparser::set(groupHeader, nElementsInCurrentGroup); - nElementsInCurrentGroup = 0; - if (position % pagesize) { - target += pagesize - (position % pagesize); - position += pagesize - (position % pagesize); - } - } - auto source = reinterpret_cast(&element); - auto copySize = sizeof(typename ListT::value_type); - if ((position % pagesize) == 0) { - memcpy(target, &pageheader, sizeof(PageHeaderT)); - position += sizeof(PageHeaderT); - target += sizeof(PageHeaderT); - } - if (!std::is_void::value && - (position % pagesize) == sizeof(PageHeader) && - (GroupHeaderPerPage || position < pagesize)) { - // write one GroupHeader at the beginning of the data, currently - // GroupHeader must be of integral type - groupHeader = reinterpret_cast(target); - position += o2::algorithm::pageparser::sizeofGroupHeader(); - target += o2::algorithm::pageparser::sizeofGroupHeader(); - } - ++nElementsInCurrentGroup; - if ((position % pagesize) + copySize > pagesize) { - copySize -= ((position % pagesize) + copySize) - pagesize; - } - if (copySize > 0) { - memcpy(target, source, copySize); - position += copySize; - target += copySize; - source += copySize; - } - copySize = sizeof(typename ListT::value_type) - copySize; - if (copySize > 0) { - memcpy(target, &pageheader, sizeof(PageHeaderT)); - position += sizeof(PageHeaderT); - target += sizeof(PageHeaderT); - memcpy(target, source, copySize); - } - position += copySize; - target += copySize; - } - if (!std::is_void::value) { - o2::algorithm::pageparser::set(groupHeader, nElementsInCurrentGroup); - } - - std::pair, size_t> result; - result.first = std::move(buffer); - result.second = totalSize; - return result; -} - -template -void FillData(ListT& dataset, unsigned entries) -{ - for (unsigned i = 0; i < entries; i++) { - dataset.emplace_back(i, 0xaa, 0xbb, 0xcc, 0xd); - } -} - -template -void runParserTest(const DataSetT& dataset) -{ - std::cout << std::endl - << "Testing PageParser in grouped mode and " - << (GroupHeaderPerPage ? "multiple" : "single") - << " group header(s)" << std::endl - << " pagesize " << pagesize << std::endl; - auto buffer = MakeBuffer(pagesize, PageHeaderT(0), dataset); - o2::header::hexDump("pagebuffer", buffer.first.get(), buffer.second); - - using RawParser = o2::algorithm::PageParser; - const RawParser parser(buffer.first.get(), buffer.second); - - unsigned dataidx = 0; - for (auto i : parser) { - o2::header::hexDump("clusterdata", &i, sizeof(ClusterData)); - BOOST_REQUIRE(i == dataset[dataidx++]); - } -} - -BOOST_AUTO_TEST_CASE(test_pageparser) -{ - constexpr unsigned pagesize = 128; - std::vector dataset; - FillData(dataset, 20); - auto buffer = MakeBuffer(pagesize, PageHeader(0), dataset); - o2::header::hexDump("pagebuffer", buffer.first.get(), buffer.second); - - using RawParser = o2::algorithm::PageParser; - const RawParser parser(buffer.first.get(), buffer.second); - - unsigned dataidx = 0; - for (auto i : parser) { - o2::header::hexDump("clusterdata", &i, sizeof(ClusterData)); - BOOST_REQUIRE(i == dataset[dataidx++]); - } - - std::vector linearizedData; - linearizedData.insert(linearizedData.begin(), parser.begin(), parser.end()); - dataidx = 0; - for (auto i : linearizedData) { - BOOST_REQUIRE(i == dataset[dataidx++]); - } - - dataidx = 0; - RawParser writer(buffer.first.get(), buffer.second); - std::vector> xvalues; - for (auto& i : writer) { - i.x = (dataidx * 3) % 7; - xvalues.emplace_back(i.x, dataidx); - ++dataidx; - } - o2::header::hexDump("changed buffer", buffer.first.get(), buffer.second); - - dataidx = 0; - for (auto i : parser) { - o2::header::hexDump("clusterdata", &i, sizeof(ClusterData)); - BOOST_REQUIRE(i.x == xvalues[dataidx++].first); - } -} - -BOOST_AUTO_TEST_CASE(test_pageparser_group) -{ - using DataSetT = std::vector; - DataSetT dataset; - FillData(dataset, 20); - - runParserTest(dataset); - runParserTest(dataset); - runParserTest(dataset); -} - -BOOST_AUTO_TEST_CASE(test_pageparser_group_perpage) -{ - using DataSetT = std::vector; - DataSetT dataset; - FillData(dataset, 20); - - runParserTest(dataset); - runParserTest(dataset); - runParserTest(dataset); -} diff --git a/Algorithm/test/parser.cxx b/Algorithm/test/parser.cxx deleted file mode 100644 index 0f31df99ca825..0000000000000 --- a/Algorithm/test/parser.cxx +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file parser.cxx -/// @author Matthias Richter -/// @since 2017-09-20 -/// @brief Unit test for data parsing methods in Algorithm/Parser.h - -#define BOOST_TEST_MODULE Test Algorithm Parser -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK -#include -#include -#include -#include -#include "../include/Algorithm/Parser.h" -#include "StaticSequenceAllocator.h" - -// header test class -struct Header { - unsigned identifier = 0xdeadbeef; - size_t payloadSize = 0; - - Header(size_t ps) : payloadSize(ps) {} -}; - -// trailer test class -struct Trailer { - unsigned identifier = 0xaaffee00; - unsigned char flags = 0xaa; - - Trailer(unsigned char f) : flags(f) {} -}; - -// trailer test class including payload size -struct SizedTrailer { - unsigned identifier = 0xaaffee00; - unsigned char flags = 0xaa; - size_t payloadSize = 0; - - SizedTrailer(size_t s, unsigned char f) : flags(f), payloadSize(s) {} -}; - -BOOST_AUTO_TEST_CASE(test_forwardparser_header_and_trailer) -{ - using FrameT = o2::algorithm::Composite; - // note: the length of the data is set in the header word - using TestFrame = o2::algorithm::StaticSequenceAllocator; - TestFrame tf(FrameT(16, "lotsofsillydata", 0xaa), - FrameT(5, "test", 0xcc), - FrameT(10, "dummydata", 0x33)); - - using ParserT = o2::algorithm::ForwardParser; - - auto checkHeader = [](const typename FrameT::HeaderType& header) { - return header.identifier == 0xdeadbeef; - }; - auto checkTrailer = [](const typename FrameT::TrailerType& trailer) { - return trailer.identifier == 0xaaffee00; - }; - auto getFrameSize = [](const typename ParserT::HeaderType& header) { - // frame size includes total offset from header and trailer - return header.payloadSize + ParserT::totalOffset; - }; - - std::vector frames; - auto insert = [&frames](typename ParserT::FrameInfo& info) { - frames.emplace_back(info); - return true; - }; - - ParserT parser; - auto result = parser.parse(tf.buffer.get(), tf.size(), - checkHeader, - checkTrailer, - getFrameSize, - insert); - - BOOST_REQUIRE(result == 3); - BOOST_REQUIRE(frames.size() == 3); - - BOOST_CHECK(memcmp(frames[0].payload, "lotsofsillydata", frames[0].length) == 0); - BOOST_CHECK(memcmp(frames[1].payload, "test", frames[1].length) == 0); - BOOST_CHECK(memcmp(frames[2].payload, "dummydata", frames[2].length) == 0); -} - -BOOST_AUTO_TEST_CASE(test_forwardparser_header_and_void_trailer) -{ - using FrameT = o2::algorithm::Composite
; - // note: the length of the data is set in the header word - using TestFrame = o2::algorithm::StaticSequenceAllocator; - TestFrame tf(FrameT(16, "lotsofsillydata"), - FrameT(5, "test"), - FrameT(10, "dummydata")); - - using ParserT = o2::algorithm::ForwardParser; - - auto checkHeader = [](const typename FrameT::HeaderType& header) { - return header.identifier == 0xdeadbeef; - }; - - auto getFrameSize = [](const typename ParserT::HeaderType& header) { - // frame size includes total offset from header and trailer - return header.payloadSize + ParserT::totalOffset; - }; - - std::vector frames; - auto insert = [&frames](typename ParserT::FrameInfo& info) { - frames.emplace_back(info); - return true; - }; - - ParserT parser; - auto result = parser.parse(tf.buffer.get(), tf.size(), - checkHeader, - getFrameSize, - insert); - - BOOST_REQUIRE(result == 3); - BOOST_REQUIRE(frames.size() == 3); - - BOOST_CHECK(memcmp(frames[0].payload, "lotsofsillydata", frames[0].length) == 0); - BOOST_CHECK(memcmp(frames[1].payload, "test", frames[1].length) == 0); - BOOST_CHECK(memcmp(frames[2].payload, "dummydata", frames[2].length) == 0); -} - -BOOST_AUTO_TEST_CASE(test_forwardparser_no_frames) -{ - using FrameT = o2::algorithm::Composite
; - // note: the length of the data is set in the header word - using TestFrame = o2::algorithm::StaticSequenceAllocator; - TestFrame tf(FrameT(16, "lotsofsillydata"), - FrameT(5, "test"), - FrameT(10, "dummydata")); - - using ParserT = o2::algorithm::ForwardParser; - - auto checkHeader = [](const typename FrameT::HeaderType& header) { - // simply indicate invalid header to read no frames - return false; - }; - - auto getFrameSize = [](const typename ParserT::HeaderType& header) { - // frame size includes total offset from header and trailer - return header.payloadSize + ParserT::totalOffset; - }; - - std::vector frames; - auto insert = [&frames](typename ParserT::FrameInfo& info) { - frames.emplace_back(info); - return true; - }; - - ParserT parser; - auto result = parser.parse(tf.buffer.get(), tf.size(), - checkHeader, - getFrameSize, - insert); - - // check that there are really no frames found - BOOST_REQUIRE(result == 0); -} - -BOOST_AUTO_TEST_CASE(test_forwardparser_format_error) -{ - using FrameT = o2::algorithm::Composite
; - // note: the length of the data is set in the header word - using TestFrame = o2::algorithm::StaticSequenceAllocator; - TestFrame tf(FrameT(16, "lotsofsillydata"), - FrameT(4, "test"), // <- note wrong size - FrameT(10, "dummydata")); - - using ParserT = o2::algorithm::ForwardParser; - - auto checkHeader = [](const typename FrameT::HeaderType& header) { - return header.identifier == 0xdeadbeef; - }; - - auto getFrameSize = [](const typename ParserT::HeaderType& header) { - // frame size includes total offset from header and trailer - return header.payloadSize + ParserT::totalOffset; - }; - - std::vector frames; - auto insert = [&frames](typename ParserT::FrameInfo& info) { - frames.emplace_back(info); - return true; - }; - - ParserT parser; - auto result = parser.parse(tf.buffer.get(), tf.size(), - checkHeader, - getFrameSize, - insert); - - BOOST_REQUIRE(result == -1); -} - -BOOST_AUTO_TEST_CASE(test_reverseparser) -{ - using FrameT = o2::algorithm::Composite; - // note: the length of the data is set in the trailer word - using TestFrame = o2::algorithm::StaticSequenceAllocator; - TestFrame tf(FrameT(0, "lotsofsillydata", {16, 0xaa}), - FrameT(0, "test", {5, 0xcc}), - FrameT(0, "dummydata", {10, 0x33})); - - using ParserT = o2::algorithm::ReverseParser; - - auto checkHeader = [](const typename FrameT::HeaderType& header) { - return header.identifier == 0xdeadbeef; - }; - auto checkTrailer = [](const typename FrameT::TrailerType& trailer) { - return trailer.identifier == 0xaaffee00; - }; - auto getFrameSize = [](const typename ParserT::TrailerType& trailer) { - return trailer.payloadSize + ParserT::totalOffset; - }; - - std::vector frames; - auto insert = [&frames](const typename ParserT::FrameInfo& info) { - frames.emplace_back(info); - return true; - }; - - ParserT parser; - auto result = parser.parse(tf.buffer.get(), tf.size(), - checkHeader, - checkTrailer, - getFrameSize, - insert); - - BOOST_REQUIRE(result == 3); - BOOST_REQUIRE(frames.size() == 3); - - BOOST_CHECK(memcmp(frames[2].payload, "lotsofsillydata", frames[2].length) == 0); - BOOST_CHECK(memcmp(frames[1].payload, "test", frames[1].length) == 0); - BOOST_CHECK(memcmp(frames[0].payload, "dummydata", frames[0].length) == 0); -} diff --git a/Algorithm/test/tableview.cxx b/Algorithm/test/tableview.cxx deleted file mode 100644 index c303d531b541e..0000000000000 --- a/Algorithm/test/tableview.cxx +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file headerstack.cxx -/// @author Matthias Richter -/// @since 2017-09-21 -/// @brief Unit test for table view abstraction class - -#define BOOST_TEST_MODULE Test Algorithm TableView -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK -#include -#include -#include -#include // memcmp -#include "Headers/DataHeader.h" // hexdump, DataHeader -#include "Headers/HeartbeatFrame.h" // HeartbeatHeader, HeartbeatTrailer -#include "../include/Algorithm/TableView.h" -#include "../include/Algorithm/Parser.h" -#include "StaticSequenceAllocator.h" - -using DataHeader = o2::header::DataHeader; -using HeartbeatHeader = o2::header::HeartbeatHeader; -using HeartbeatTrailer = o2::header::HeartbeatTrailer; - -template -void hexDump(Targs... Fargs) -{ - // a simple redirect to enable/disable the hexdump printout - o2::header::hexDump(Fargs...); -} - -BOOST_AUTO_TEST_CASE(test_tableview_reverse) -{ - using FrameT = o2::algorithm::Composite; - using TestFrame = o2::algorithm::StaticSequenceAllocator; - // the length of the data is set in the trailer word - // the header is used as column description, using slightly different - // orbit numbers in the to data sets which will result in two complete - // columns at beginning and end, while the two in the middle only have - // one row entry - TestFrame tf1(FrameT({0x1100000000000000}, "heartbeatdata", {0x510000000000000e}), - FrameT({0x1100000000000001}, "test", {0x5100000000000005}), - FrameT({0x1100000000000003}, "dummydata", {0x510000000000000a})); - TestFrame tf2(FrameT({0x1100000000000000}, "frame2a", {0x5100000000000008}), - FrameT({0x1100000000000002}, "frame2b", {0x5100000000000008}), - FrameT({0x1100000000000003}, "frame2c", {0x5100000000000008})); - hexDump("Test frame 1", tf1.buffer.get(), tf1.size()); - hexDump("Test frame 2", tf2.buffer.get(), tf2.size()); - - // the payload length is set in the trailer, so we need a reverse parser - using ParserT = o2::algorithm::ReverseParser; - - // define the view type for DataHeader as row descriptor, - // HeartbeatHeader as column descriptor and the reverse parser - using ViewType = o2::algorithm::TableView; - ViewType heartbeatview; - - o2::header::DataHeader dh1; - dh1.dataDescription = o2::header::DataDescription("FIRSTROW"); - dh1.dataOrigin = o2::header::DataOrigin("TST"); - dh1.subSpecification = 0; - dh1.payloadSize = 0; - - o2::header::DataHeader dh2; - dh2.dataDescription = o2::header::DataDescription("SECONDROW"); - dh2.dataOrigin = o2::header::DataOrigin("TST"); - dh2.subSpecification = 0xdeadbeef; - dh2.payloadSize = 0; - - heartbeatview.addRow(dh1, (std::byte*)tf1.buffer.get(), tf1.size()); - heartbeatview.addRow(dh2, (std::byte*)tf2.buffer.get(), tf2.size()); - - std::cout << "slots: " << heartbeatview.getNRows() - << " columns: " << heartbeatview.getNColumns() - << std::endl; - - // definitions for the data check - const char* dataset1[] = { - "heartbeatdata", - "test", - "dummydata"}; - const char* dataset2[] = { - "frame2a", - "frame2b", - "frame2c"}; - - // four orbits are populated, 0 and 3 with 2 rows, 1 and 2 with one row - BOOST_REQUIRE(heartbeatview.getNColumns() == 4); - BOOST_REQUIRE(heartbeatview.getNRows() == 2); - unsigned requiredNofRowsInColumn[] = {2, 1, 1, 2}; - - unsigned colidx = 0; - unsigned dataset1idx = 0; - unsigned dataset2idx = 0; - for (auto columnIt = heartbeatview.begin(), end = heartbeatview.end(); - columnIt != end; ++columnIt, ++colidx) { - unsigned rowidx = 0; - std::cout << "---------------------------------------" << std::endl; - for (auto row : columnIt) { - auto dataset = (rowidx == 1 || colidx == 2) ? dataset2 : dataset1; - auto& datasetidx = (rowidx == 1 || colidx == 2) ? dataset2idx : dataset1idx; - hexDump("Entry", row.buffer, row.size); - BOOST_CHECK(memcmp(row.buffer, dataset[datasetidx++], row.size) == 0); - ++rowidx; - } - BOOST_CHECK(rowidx == requiredNofRowsInColumn[colidx]); - } -} - -BOOST_AUTO_TEST_CASE(test_tableview_formaterror) -{ - using FrameT = o2::algorithm::Composite; - using TestFrame = o2::algorithm::StaticSequenceAllocator; - // note: the length of the data is set in the trailer word - // specifying wrong length in the second entry, no frames should be added - TestFrame tf1(FrameT({0x1100000000000000}, "heartbeatdata", {0x510000000000000e}), - FrameT({0x1100000000000001}, "test", {0x5100000000000004}), - FrameT({0x1100000000000003}, "dummydata", {0x510000000000000a})); - - // the payload length is set in the trailer, so we need a reverse parser - using ParserT = o2::algorithm::ReverseParser; - - // define the view type for DataHeader as row descriptor, - // HeartbeatHeader as column descriptor and the reverse parser - using ViewType = o2::algorithm::TableView; - ViewType heartbeatview; - - o2::header::DataHeader dh; - dh.dataDescription = o2::header::DataDescription("FIRSTSLOT"); - dh.dataOrigin = o2::header::DataOrigin("TST"); - dh.subSpecification = 0; - dh.payloadSize = 0; - - heartbeatview.addRow(dh, (std::byte*)tf1.buffer.get(), tf1.size()); - - BOOST_CHECK(heartbeatview.getNRows() == 0); - BOOST_CHECK(heartbeatview.getNColumns() == 0); -} diff --git a/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx b/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx index a1268c02a2740..0e4432f970230 100644 --- a/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx +++ b/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx @@ -15,7 +15,6 @@ /// @author Matthias Richter #include "DataFormatsTPC/ClusterNativeHelper.h" -#include "Algorithm/Parser.h" #include #include #include diff --git a/Detectors/TPC/workflow/src/ZSSpec.cxx b/Detectors/TPC/workflow/src/ZSSpec.cxx index c24647f6ae240..eaedb537faba8 100644 --- a/Detectors/TPC/workflow/src/ZSSpec.cxx +++ b/Detectors/TPC/workflow/src/ZSSpec.cxx @@ -26,7 +26,6 @@ #include "GPUHostDataTypes.h" #include "GPUO2InterfaceConfiguration.h" #include "TPCBase/Sector.h" -#include "Algorithm/Parser.h" #include #include // for make_shared #include diff --git a/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h b/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h index b8fc08831cd09..7b6bffcac9e1e 100644 --- a/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h +++ b/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h @@ -23,7 +23,6 @@ #include "Framework/InitContext.h" #include "Framework/CompletionPolicy.h" #include "GPUCommonAlignedAlloc.h" -#include "Algorithm/Parser.h" #include #include #include diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx index 18409ac68e29f..e54d1dadb6ce2 100644 --- a/GPU/Workflow/src/GPUWorkflowSpec.cxx +++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx @@ -65,7 +65,6 @@ #include "TPCBaseRecSim/DeadChannelMapCreator.h" #include "SimulationDataFormat/ConstMCTruthContainer.h" #include "SimulationDataFormat/MCCompLabel.h" -#include "Algorithm/Parser.h" #include "DataFormatsGlobalTracking/RecoContainer.h" #include "DataFormatsTRD/RecoInputContainer.h" #include "TRDBase/Geometry.h" diff --git a/GPU/Workflow/src/GPUWorkflowTPC.cxx b/GPU/Workflow/src/GPUWorkflowTPC.cxx index e9b379168b118..e4d4cbd0a0417 100644 --- a/GPU/Workflow/src/GPUWorkflowTPC.cxx +++ b/GPU/Workflow/src/GPUWorkflowTPC.cxx @@ -61,7 +61,6 @@ #include "TPCBaseRecSim/DeadChannelMapCreator.h" #include "SimulationDataFormat/ConstMCTruthContainer.h" #include "SimulationDataFormat/MCCompLabel.h" -#include "Algorithm/Parser.h" #include "DataFormatsGlobalTracking/RecoContainer.h" #include "DataFormatsTPC/AltroSyncSignal.h" #include "CommonUtils/VerbosityConfig.h"