NexusCPP  3.5.0
nxwriter.h
Go to the documentation of this file.
1 //*****************************************************************************
17 //*****************************************************************************
18 
19 #ifndef __NX_WRITER_H__
20 #define __NX_WRITER_H__
21 
22 // ============================================================================
41 // ============================================================================
42 
43 // standard library objets
44 #include <iostream>
45 #include <vector>
46 #include <set>
47 
48 // yat library
49 #include <yat/CommonHeader.h>
50 #include <yat/memory/SharedPtr.h>
51 #include <yat/memory/MemBuf.h>
52 #include <yat/utils/String.h>
53 #include <yat/utils/Logging.h>
54 #include <yat/time/Timer.h>
55 #include <yat/threading/Mutex.h>
56 
57 #include <nexuscpp/nxfile.h>
58 
59 namespace nxcpp
60 {
61 
62 // Forward declaration
63 class NexusFileWriterImpl;
64 
65 typedef yat::SharedPtr<NexusDataSet, yat::Mutex> NexusDataSetPtr;
66 typedef yat::SharedPtr<NexusAttr, yat::Mutex> NexusAttrPtr;
67 
68 // internal free functions
69 void message_to_console(yat::ELogLevel lvl, const yat::String& msg);
70 void exception_to_console(const yat::Exception& e);
71 
72 //=============================================================================
74 //=============================================================================
75 class MMetadata
76 {
77 public:
78  enum Type
79  {
80  NONE = 0,
82  INT,
85  };
86 
87 private:
88  std::map<std::string, std::string> m_mapString; // "std::string" type metadata
89  std::map<std::string, int> m_mapInteger; // "Integer" type metadata
90  std::map<std::string, long> m_mapLong; // "Long Integer" type metadata
91  std::map<std::string, double> m_mapDouble; // "Float" type metadata
92 
93 public:
98  void AddMetadata(const std::string &strKey, const std::string &strValue);
99 
104  void AddMetadata(const std::string &strKey, const char *pszValue);
105 
110  void AddMetadata(const std::string &strKey, int iValue);
111 
116  void AddMetadata(const std::string &strKey, long lValue);
117 
122  void AddMetadata(const std::string &strKey, double dValue);
123 
127  bool HasMetadata(const std::string &strKey) const;
128 
136  bool GetMetadata(const std::string &strKey, std::string *pstrValue, bool bThrow=true) const;
137 
144  bool GetStringMetadata(const std::string &strKey, std::string *pstrValue, bool bThrow=true) const;
145 
151  std::string StringMetadata(const std::string &strKey) const;
152 
159  bool GetIntegerMetadata(const std::string &strKey, int *piValue, bool bThrow=true) const;
160 
166  int IntegerMetadata(const std::string &strKey) const;
167 
174  bool GetLongIntegerMetadata(const std::string &strKey, long *plValue, bool bThrow=true) const;
175 
181  long LongIntegerMetadata(const std::string &strKey) const;
182 
189  bool GetDoubleMetadata(const std::string &strKey, double *pdValue, bool bThrow=true) const;
190 
196  double DoubleMetadata(const std::string &strKey) const;
197 
203  Type GetMetadataType(const std::string &strKey, bool bThrow=true) const;
204 
206  std::list<std::string> MetadataKeys() const;
207 
209  void ClearMetadata();
210 };
211 
212 //=============================================================================
214 //=============================================================================
216 {
217 public:
219  virtual void OnNexusException(const NexusException &e)=0;
220 };
221 
222 //=============================================================================
224 //=============================================================================
226 {
227 public:
232  virtual void OnNexusMessage(yat::ELogLevel lvl, const yat::String& msg)=0;
233 };
234 
235 //=============================================================================
240 //=============================================================================
242 {
243 public:
245  {
246  SYNCHRONOUS = 0,
248 
251  DELAYED
253  };
254 
256  typedef struct Statistics
257  {
258  Statistics();
259  yat::uint64 ui64WrittenBytes;
260  yat::uint64 ui64TotalBytes;
264  } Statistics;
265 
266  //=============================================================================
268  //=============================================================================
269  class INotify
270  {
271  public:
278  virtual void OnWriteSubSet(NexusFileWriter* source_p, const std::string& dataset_path, int start_pos[MAX_RANK], int dim[MAX_RANK])=0;
279 
284  virtual void OnWrite(NexusFileWriter* source_p, const std::string& dataset_path)=0;
285 
290  virtual void OnCloseFile(NexusFileWriter* source_p, const std::string& nexus_file_path)=0;
291  };
292 
293 private:
294  NexusFileWriterImpl *m_pImpl; // Implementation
295 
296  static std::size_t s_attempt_max_; // max attempts for every nexus action
297  static std::size_t s_attempt_delay_ms_;
298 
299 public:
305  NexusFileWriter(const std::string &strFilePath, WriteMode eMode=ASYNCHRONOUS, unsigned int uiWritePeriod=2);
306 
308  ~NexusFileWriter();
309 
311  void SetExceptionHandler(IExceptionHandler *pHandler);
312 
314  void SetMessageHandler(IMessageHandler *pHandler);
315 
317  void SetNotificationHandler(INotify *pHandler);
318 
320  void AddNotificationHandler(INotify* pHandler);
321 
323  void SetCacheSize(yat::uint16 usCacheSize);
324 
334  void CreateDataSet(const std::string &strPath, NexusDataType eDataType, int iRank,
335  int *piDim, CompressionFilter filter=CompressNone,
336  FilterConfig filter_config=FilterConfig(), int *piChunk=0);
337 
342  void PushDataSet(const std::string &strPath, NexusDataSetPtr ptrDataSet);
343 
349  void PushDatasetAttr(const std::string &strPath, NexusAttrPtr ptrAttr);
350 
353  void Abort();
354 
356  bool IsError();
357 
359  bool IsEmpty();
360 
362  bool Done();
363 
365  void Synchronize(bool bSendSyncMsg=true);
366 
367  // Hold the data, force the NeXuisFileWriter to cache data and not writting it into the filename
368  // Until either Synchronize() or Hold(false) is calles
369  // Available only in asynchronous mode
370  void Hold(bool b=true);
371 
372  // Return true if it hold the data
373  // Make sense only in asynchronous mode
374  bool IsHold() const;
375 
377  const std::string &File() const;
378 
380  void SetFileAutoClose(bool b);
381 
383  void CloseFile();
384 
386  void SetUseLock();
387 
389 
391  static std::size_t AttemptMax() { return s_attempt_max_; }
392 
394  static std::size_t AttemptDelay() { return s_attempt_delay_ms_; }
395 
397  static void SetAttemptMax(std::size_t n);
398 
400  static void SetAttemptDelay(std::size_t ms);
401 
403 
405 
407  void ResetStatistics();
408 
410  Statistics GetStatistics() const;
411 
413 };
414 
415 typedef yat::MemBuf DatasetBuf;
416 
418 typedef yat::SharedPtr<NexusFileWriter, yat::Mutex> NexusFileWriterPtr;
419 
421 typedef std::vector<std::size_t> DataShape;
422 
423 // Empty shape
424 extern const DataShape g_empty_shape;
425 
426 //==============================================================================
430 //==============================================================================
432 {
433 public:
435  {
436  public:
437  virtual void OnFlushData(DatasetWriter* pWriter) = 0;
438  };
439 
441  typedef struct Config
442  {
444  DataShape shape_data_item;
445 
447  DataShape shape_matrix;
448 
450  yat::uint16 cache_mb;
451 
453  yat::uint16 write_tmout;
454 
457 
460 
463 
466  {
467  compress_filter = CompressNone;
468  cache_mb = 64;
469  write_tmout = 0;
470  min_bytes_for_compression = 1024;
471  }
472  } Config;
473 
474 private:
475  Config m_cfg;
476  std::string m_strPath; // NeXus group path
477  std::string m_strDataset; // Dataset name
478  DatasetBuf m_mbData;
479  DataShape m_shapeMatrix; // shape of the dataitem matrix
480  yat::uint16 m_usMaxMB; // Max cached mega-bytes before flush
481  NexusFileWriterPtr m_ptrNeXusFileWriter;
482  NexusDataType m_NexusDataType;
483  std::size_t m_nDataItemRank;
484  int m_aiDataItemDim[MAX_RANK];
485  yat::uint32 m_ulDataItemSize;
486  std::size_t m_nTotalRank;
487  int m_aiTotalDim[MAX_RANK];
488  yat::uint64 m_ui64TotalSize;
489  bool m_bDatasetCreated;
490  std::size_t m_nDataItemCount;
491  IFlushNotification* m_pFlushListener;
492  bool m_bUnlimited; //#### true: Nexus dataset size is unlimited
493  std::size_t m_nWrittenCount; // Count of written data items
494  yat::Timeout m_tmWriteTimeout; // if expired, we should write the bufferized data
495 
496 private:
497  void CreateDataset();
498  void Flush(bool call_on_flush);
499  bool DirectFlush(const void *pData);
500  void CheckWriter();
501  void PreAllocBuffer();
502  template <class TYPE> void AdjustSizes();
503  void init(yat::uint16 usMaxMB, yat::uint16 m_usWriteTimeout);
504 
505 public:
507  template <class TYPE>
508  void SetDataType();
509 
511  void Reset();
512 
514  void SetNeXusDataType(NexusDataType eDataType);
515 
520  void SetShapes(const DataShape &shapeDataItem, const DataShape &shapeMatrix);
521 
523  void set_compression(CompressionFilter filter=CompressZLIB, FilterConfig filter_config=FilterConfig());
524 
531  DatasetWriter(const DataShape &shapeDataItem, const DataShape &shapeMatrix=g_empty_shape, yat::uint16 usMaxMB=64, yat::uint16 m_usWriteTimeout=0);
532 
539  DatasetWriter(const DataShape &shapeDataItem, std::size_t one_dim_size, yat::uint16 usMaxMB=64, yat::uint16 m_usWriteTimeout=0);
540 
545  DatasetWriter(yat::uint16 usMaxMB=64, yat::uint16 m_usWriteTimeout=0);
546  DatasetWriter(const Config& cfg);
547 
549  DatasetWriter(const std::vector<int>&, const std::vector<int>&, yat::uint16 usMaxMB=64);
550  DatasetWriter(const std::vector<int>&, std::size_t, yat::uint16 usMaxMB=64);
552 
554  virtual ~DatasetWriter();
555 
560  void SetNexusFileWriter(NexusFileWriterPtr ptrWriter);
561 
566  void SetFlushListener(IFlushNotification* pListener) { m_pFlushListener = pListener; }
567 
572  void SetMatrix(const DataShape &shapeMatrix=g_empty_shape);
573 
575  void SetPath(const std::string &strPath, const std::string &strDataset);
576 
578  void SetFullPath(const std::string &strFullPath);
579 
581  void SetDatasetName(const std::string &strDatasetName);
582 
584  void SetCacheSize(yat::uint16 usMaxMB) { m_usMaxMB = usMaxMB; }
585 
587  void AddAttribute(const NexusAttrPtr &ptrAttr);
588 
590  void AddFloatAttribute(const std::string &strName, double dValue);
591 
593  void AddIntegerAttribute(const std::string &strName, long lValue);
594 
596  void AddStringAttribute(const std::string &strName, const std::string &strValue);
597 
599  void PushData(const void *pData, std::size_t nDataCount=1, bool bNoCopy=false);
600 
602  void Stop();
603 
605  void Abort();
606 
608  const DataShape &MatrixShape() const { return m_shapeMatrix; }
609  DataShape DataItemShape() const;
610  NexusDataType DataType() const { return m_NexusDataType; }
611  std::string FullPath() const;
612  const std::string &DatasetName() const { return m_strDataset; }
613  std::size_t TotalRank() const { return m_nTotalRank; }
614  std::size_t DataItemCount() const { return m_nDataItemCount; }
615  yat::uint64 TotalSize() const { return m_ui64TotalSize; }
616  yat::uint32 DataItemSize() const { return m_ulDataItemSize; }
617  yat::uint32 MaxDataItemsCount() const { return (unsigned long)(m_ui64TotalSize / m_ulDataItemSize); }
619 };
620 
622 typedef yat::SharedPtr<DatasetWriter, yat::Mutex> DatasetWriterPtr;
623 
624 //==============================================================================
628 //==============================================================================
629 template <class TYPE>
631 {
632 public:
637  GenericDatasetWriter(const DataShape &shapeDataItem, const DataShape &shapeMatrix=g_empty_shape, unsigned short usMaxMB=100);
638 
641 };
642 
643 //==============================================================================
647 //==============================================================================
648 template <class TYPE>
650 {
651 public:
656  AxisDatasetWriter(int iDim, int iSize, int iOrder=1);
657 
660 
661  void PushPosition(TYPE TValue);
662 };
663 
664 //==============================================================================
669 //==============================================================================
670 template <class TYPE>
672 {
673 public:
678  SignalDatasetWriter(const DataShape &shapeData, const DataShape &shapeMatrix=g_empty_shape, int iSignal=1);
679 
682 
686  void PushSignal(TYPE *pValue);
687 };
688 
690 
691 } // namespace nxcpp
692 
693 
694 #endif
Definition: nxfile.h:104
yat::uint64 ui64TotalBytes
Definition: nxwriter.h:260
Definition: nxwriter.h:241
DataShape shape_matrix
Dimensions array of acquisition space.
Definition: nxwriter.h:447
bool GetStringMetadata(const std::string &strKey, std::string *pstrValue, bool bThrow=true) const
yat::SharedPtr< NexusFileWriter, yat::Mutex > NexusFileWriterPtr
Referenced pointer definition.
Definition: nxwriter.h:418
Definition: nxwriter.h:80
Config()
default c-tor
Definition: nxwriter.h:465
#define NEXUSCPP_DECL
Definition: nxfile.h:67
~GenericDatasetWriter()
d-tor
Definition: nxwriter.h:640
FilterConfig filter_config
Compress filter parameters.
Definition: nxwriter.h:459
float fAverageMbPerSec
Instant rate in MBytes/s.
Definition: nxwriter.h:262
yat::uint16 write_tmout
useless parameter (to be deleted)
Definition: nxwriter.h:453
Definition: nxwriter.h:84
const std::string & DatasetName() const
Definition: nxwriter.h:612
Exception handling interface.
Definition: nxwriter.h:215
static std::size_t AttemptMax()
Max attempts for write actions.
Definition: nxwriter.h:391
~AxisDatasetWriter()
d-tor
Definition: nxwriter.h:659
Definition: nxwriter.h:81
float fInstantMbPerSec
Definition: nxwriter.h:261
std::size_t DataItemCount() const
Definition: nxwriter.h:614
std::size_t min_bytes_for_compression
minimal dataset size for compression, smaller datasets are not compressed
Definition: nxwriter.h:462
Mix-in that add metadata capabilitie to objects.
Definition: nxwriter.h:75
Asynchronous mode.
Definition: nxwriter.h:250
static std::size_t AttemptDelay()
delay (in ms) between to attempts
Definition: nxwriter.h:394
Type
Definition: nxwriter.h:78
NexusDataType
Definition: nxfile.h:137
yat::MemBuf DatasetBuf
Definition: nxwriter.h:415
yat::SharedPtr< NexusDataSet, yat::Mutex > NexusDataSetPtr
Definition: nxwriter.h:63
Definition: nxwriter.h:82
DatasetWriter configuration.
Definition: nxwriter.h:441
const int MAX_RANK
Max datasets rank.
Definition: nxfile.h:175
Definition: nxwriter.h:671
void ClearMetadata()
Clear all metadata.
double dTotalWriteTime
Average rate in MBytes/s.
Definition: nxwriter.h:263
void exception_to_console(const yat::Exception &e)
Definition: nxdebug.h:31
Message handling interface.
Definition: nxwriter.h:225
Definition: nxfile.h:105
NexusDataType DataType() const
Definition: nxwriter.h:610
long LongIntegerMetadata(const std::string &strKey) const
yat::SharedPtr< DatasetWriter, yat::Mutex > DatasetWriterPtr
Shared pointer definition.
Definition: nxwriter.h:622
DataShape shape_data_item
Dimensions array of canonical data.
Definition: nxwriter.h:444
std::list< std::string > MetadataKeys() const
Returns the metadata keys std::list.
bool HasMetadata(const std::string &strKey) const
const DataShape g_empty_shape
int IntegerMetadata(const std::string &strKey) const
yat::uint64 TotalSize() const
Definition: nxwriter.h:615
bool GetMetadata(const std::string &strKey, std::string *pstrValue, bool bThrow=true) const
yat::uint64 ui64WrittenBytes
Definition: nxwriter.h:259
yat::SharedPtr< NexusAttr, yat::Mutex > NexusAttrPtr
Definition: nxwriter.h:66
yat::uint16 cache_mb
Write cache size.
Definition: nxwriter.h:450
std::map< FilterOption, int > FilterConfig
Filter options to be passed with CreateCompressedDataSet method.
Definition: nxfile.h:93
std::string StringMetadata(const std::string &strKey) const
Definition: nxwriter.h:83
void SetFlushListener(IFlushNotification *pListener)
Definition: nxwriter.h:566
yat::uint32 MaxDataItemsCount() const
Definition: nxwriter.h:617
Definition: nxfile.h:182
Definition: nxwriter.h:630
Definition: nxwriter.h:431
CompressionFilter
List of known compression filters.
Definition: nxfile.h:102
CompressionFilter compress_filter
Compress filter.
Definition: nxwriter.h:456
std::vector< std::size_t > DataShape
Data shape type.
Definition: nxwriter.h:421
~SignalDatasetWriter()
d-tor
Definition: nxwriter.h:681
Writing speed statistics.
Definition: nxwriter.h:256
double DoubleMetadata(const std::string &strKey) const
WriteMode
Definition: nxwriter.h:244
Definition: nxwriter.h:649
const DataShape & MatrixShape() const
Definition: nxwriter.h:608
Synchronous mode.
Definition: nxwriter.h:247
bool GetIntegerMetadata(const std::string &strKey, int *piValue, bool bThrow=true) const
std::size_t TotalRank() const
Definition: nxwriter.h:613
yat::uint32 DataItemSize() const
Definition: nxwriter.h:616
void AddMetadata(const std::string &strKey, const std::string &strValue)
bool GetLongIntegerMetadata(const std::string &strKey, long *plValue, bool bThrow=true) const
Type GetMetadataType(const std::string &strKey, bool bThrow=true) const
bool GetDoubleMetadata(const std::string &strKey, double *pdValue, bool bThrow=true) const
Write notification interface.
Definition: nxwriter.h:269
void SetCacheSize(yat::uint16 usMaxMB)
Sets the buffer size in Mega bytes.
Definition: nxwriter.h:584
void message_to_console(yat::ELogLevel lvl, const yat::String &msg)