19 #ifndef __NX_WRITER_H__ 20 #define __NX_WRITER_H__ 49 #include <yat/CommonHeader.h> 50 #include <yat/memory/SharedPtr.h> 51 #include <yat/memory/MemBuf.h> 52 #include <yat/utils/String.h> 53 #include <yat/utils/Logging.h> 54 #include <yat/time/Timer.h> 55 #include <yat/threading/Mutex.h> 63 class NexusFileWriterImpl;
88 std::map<std::string, std::string> m_mapString;
89 std::map<std::string, int> m_mapInteger;
90 std::map<std::string, long> m_mapLong;
91 std::map<std::string, double> m_mapDouble;
98 void AddMetadata(
const std::string &strKey,
const std::string &strValue);
104 void AddMetadata(
const std::string &strKey,
const char *pszValue);
110 void AddMetadata(
const std::string &strKey,
int iValue);
116 void AddMetadata(
const std::string &strKey,
long lValue);
122 void AddMetadata(
const std::string &strKey,
double dValue);
136 bool GetMetadata(
const std::string &strKey, std::string *pstrValue,
bool bThrow=
true)
const;
144 bool GetStringMetadata(
const std::string &strKey, std::string *pstrValue,
bool bThrow=
true)
const;
159 bool GetIntegerMetadata(
const std::string &strKey,
int *piValue,
bool bThrow=
true)
const;
189 bool GetDoubleMetadata(
const std::string &strKey,
double *pdValue,
bool bThrow=
true)
const;
232 virtual void OnNexusMessage(yat::ELogLevel lvl,
const yat::String& msg)=0;
278 virtual void OnWriteSubSet(
NexusFileWriter* source_p,
const std::string& dataset_path,
int start_pos[
MAX_RANK],
int dim[MAX_RANK])=0;
284 virtual void OnWrite(
NexusFileWriter* source_p,
const std::string& dataset_path)=0;
290 virtual void OnCloseFile(
NexusFileWriter* source_p,
const std::string& nexus_file_path)=0;
294 NexusFileWriterImpl *m_pImpl;
296 static std::size_t s_attempt_max_;
297 static std::size_t s_attempt_delay_ms_;
317 void SetNotificationHandler(
INotify *pHandler);
320 void AddNotificationHandler(
INotify* pHandler);
323 void SetCacheSize(yat::uint16 usCacheSize);
334 void CreateDataSet(
const std::string &strPath,
NexusDataType eDataType,
int iRank,
342 void PushDataSet(
const std::string &strPath, NexusDataSetPtr ptrDataSet);
349 void PushDatasetAttr(
const std::string &strPath, NexusAttrPtr ptrAttr);
365 void Synchronize(
bool bSendSyncMsg=
true);
370 void Hold(
bool b=
true);
377 const std::string &File()
const;
380 void SetFileAutoClose(
bool b);
397 static void SetAttemptMax(std::size_t n);
400 static void SetAttemptDelay(std::size_t ms);
407 void ResetStatistics();
470 min_bytes_for_compression = 1024;
476 std::string m_strPath;
477 std::string m_strDataset;
479 DataShape m_shapeMatrix;
480 yat::uint16 m_usMaxMB;
481 NexusFileWriterPtr m_ptrNeXusFileWriter;
483 std::size_t m_nDataItemRank;
485 yat::uint32 m_ulDataItemSize;
486 std::size_t m_nTotalRank;
488 yat::uint64 m_ui64TotalSize;
489 bool m_bDatasetCreated;
490 std::size_t m_nDataItemCount;
493 std::size_t m_nWrittenCount;
494 yat::Timeout m_tmWriteTimeout;
497 void CreateDataset();
498 void Flush(
bool call_on_flush);
499 bool DirectFlush(
const void *pData);
501 void PreAllocBuffer();
502 template <
class TYPE>
void AdjustSizes();
503 void init(yat::uint16 usMaxMB, yat::uint16 m_usWriteTimeout);
507 template <
class TYPE>
520 void SetShapes(
const DataShape &shapeDataItem,
const DataShape &shapeMatrix);
531 DatasetWriter(
const DataShape &shapeDataItem,
const DataShape &shapeMatrix=g_empty_shape, yat::uint16 usMaxMB=64, yat::uint16 m_usWriteTimeout=0);
539 DatasetWriter(
const DataShape &shapeDataItem, std::size_t one_dim_size, yat::uint16 usMaxMB=64, yat::uint16 m_usWriteTimeout=0);
545 DatasetWriter(yat::uint16 usMaxMB=64, yat::uint16 m_usWriteTimeout=0);
549 DatasetWriter(
const std::vector<int>&,
const std::vector<int>&, yat::uint16 usMaxMB=64);
550 DatasetWriter(
const std::vector<int>&, std::size_t, yat::uint16 usMaxMB=64);
560 void SetNexusFileWriter(NexusFileWriterPtr ptrWriter);
572 void SetMatrix(
const DataShape &shapeMatrix=g_empty_shape);
575 void SetPath(
const std::string &strPath,
const std::string &strDataset);
578 void SetFullPath(
const std::string &strFullPath);
581 void SetDatasetName(
const std::string &strDatasetName);
587 void AddAttribute(
const NexusAttrPtr &ptrAttr);
590 void AddFloatAttribute(
const std::string &strName,
double dValue);
593 void AddIntegerAttribute(
const std::string &strName,
long lValue);
596 void AddStringAttribute(
const std::string &strName,
const std::string &strValue);
599 void PushData(
const void *pData, std::size_t nDataCount=1,
bool bNoCopy=
false);
609 DataShape DataItemShape()
const;
611 std::string FullPath()
const;
615 yat::uint64
TotalSize()
const {
return m_ui64TotalSize; }
617 yat::uint32
MaxDataItemsCount()
const {
return (
unsigned long)(m_ui64TotalSize / m_ulDataItemSize); }
629 template <
class TYPE>
637 GenericDatasetWriter(
const DataShape &shapeDataItem,
const DataShape &shapeMatrix=g_empty_shape,
unsigned short usMaxMB=100);
648 template <
class TYPE>
661 void PushPosition(TYPE TValue);
670 template <
class TYPE>
678 SignalDatasetWriter(
const DataShape &shapeData,
const DataShape &shapeMatrix=g_empty_shape,
int iSignal=1);
686 void PushSignal(TYPE *pValue);
yat::uint64 ui64TotalBytes
Definition: nxwriter.h:260
Definition: nxwriter.h:241
DataShape shape_matrix
Dimensions array of acquisition space.
Definition: nxwriter.h:447
yat::SharedPtr< NexusFileWriter, yat::Mutex > NexusFileWriterPtr
Referenced pointer definition.
Definition: nxwriter.h:418
Config()
default c-tor
Definition: nxwriter.h:465
#define NEXUSCPP_DECL
Definition: nxfile.h:67
~GenericDatasetWriter()
d-tor
Definition: nxwriter.h:640
FilterConfig filter_config
Compress filter parameters.
Definition: nxwriter.h:459
float fAverageMbPerSec
Instant rate in MBytes/s.
Definition: nxwriter.h:262
yat::uint16 write_tmout
useless parameter (to be deleted)
Definition: nxwriter.h:453
const std::string & DatasetName() const
Definition: nxwriter.h:612
Exception handling interface.
Definition: nxwriter.h:215
static std::size_t AttemptMax()
Max attempts for write actions.
Definition: nxwriter.h:391
~AxisDatasetWriter()
d-tor
Definition: nxwriter.h:659
float fInstantMbPerSec
Definition: nxwriter.h:261
std::size_t DataItemCount() const
Definition: nxwriter.h:614
std::size_t min_bytes_for_compression
minimal dataset size for compression, smaller datasets are not compressed
Definition: nxwriter.h:462
Asynchronous mode.
Definition: nxwriter.h:250
static std::size_t AttemptDelay()
delay (in ms) between to attempts
Definition: nxwriter.h:394
NexusDataType
Definition: nxfile.h:137
yat::MemBuf DatasetBuf
Definition: nxwriter.h:415
yat::SharedPtr< NexusDataSet, yat::Mutex > NexusDataSetPtr
Definition: nxwriter.h:63
DatasetWriter configuration.
Definition: nxwriter.h:441
const int MAX_RANK
Max datasets rank.
Definition: nxfile.h:175
Definition: nxwriter.h:671
double dTotalWriteTime
Average rate in MBytes/s.
Definition: nxwriter.h:263
void exception_to_console(const yat::Exception &e)
Message handling interface.
Definition: nxwriter.h:225
NexusDataType DataType() const
Definition: nxwriter.h:610
yat::SharedPtr< DatasetWriter, yat::Mutex > DatasetWriterPtr
Shared pointer definition.
Definition: nxwriter.h:622
DataShape shape_data_item
Dimensions array of canonical data.
Definition: nxwriter.h:444
const DataShape g_empty_shape
yat::uint64 TotalSize() const
Definition: nxwriter.h:615
yat::uint64 ui64WrittenBytes
Definition: nxwriter.h:259
yat::SharedPtr< NexusAttr, yat::Mutex > NexusAttrPtr
Definition: nxwriter.h:66
yat::uint16 cache_mb
Write cache size.
Definition: nxwriter.h:450
std::map< FilterOption, int > FilterConfig
Filter options to be passed with CreateCompressedDataSet method.
Definition: nxfile.h:93
void SetFlushListener(IFlushNotification *pListener)
Definition: nxwriter.h:566
yat::uint32 MaxDataItemsCount() const
Definition: nxwriter.h:617
Definition: nxwriter.h:630
Definition: nxwriter.h:431
Definition: nxwriter.h:434
CompressionFilter
List of known compression filters.
Definition: nxfile.h:102
CompressionFilter compress_filter
Compress filter.
Definition: nxwriter.h:456
std::vector< std::size_t > DataShape
Data shape type.
Definition: nxwriter.h:421
~SignalDatasetWriter()
d-tor
Definition: nxwriter.h:681
Writing speed statistics.
Definition: nxwriter.h:256
WriteMode
Definition: nxwriter.h:244
Definition: nxwriter.h:649
const DataShape & MatrixShape() const
Definition: nxwriter.h:608
Synchronous mode.
Definition: nxwriter.h:247
std::size_t TotalRank() const
Definition: nxwriter.h:613
yat::uint32 DataItemSize() const
Definition: nxwriter.h:616
Write notification interface.
Definition: nxwriter.h:269
void SetCacheSize(yat::uint16 usMaxMB)
Sets the buffer size in Mega bytes.
Definition: nxwriter.h:584
void message_to_console(yat::ELogLevel lvl, const yat::String &msg)