libvisiontransfer  8.1.0
datablockprotocol.h
1 /*******************************************************************************
2  * Copyright (c) 2020 Nerian Vision GmbH
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to deal
6  * in the Software without restriction, including without limitation the rights
7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8  * copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *******************************************************************************/
14 
15 #ifndef VISIONTRANSFER_DATABLOCKPROTOCOL_H
16 #define VISIONTRANSFER_DATABLOCKPROTOCOL_H
17 
18 #include <map>
19 #include <vector>
20 #include <memory>
21 #include <chrono>
22 #include <deque>
23 
24 #include "visiontransfer/alignedallocator.h"
25 #include "visiontransfer/exceptions.h"
26 
27 namespace visiontransfer {
28 namespace internal {
29 
47 public:
48  enum ProtocolType {
49  PROTOCOL_TCP,
50  PROTOCOL_UDP
51  };
52 
53  //
54  static const int MAX_DATA_BLOCKS = 8;
55 
56  // Constants that are also used in other places.
57  static const int MAX_TCP_BYTES_TRANSFER = 0xFFFF; //64K - 1
58  static const int MAX_UDP_RECEPTION = 0x4000; //16K
59  static const int MAX_OUTSTANDING_BYTES = 2*MAX_TCP_BYTES_TRANSFER;
60 
61 #pragma pack(push,1)
62  // Extends previous one-channel 6-byte raw header buffer
63  // Legacy transfers can be detected via non-zero netTransferSizeDummy
64  struct HeaderPreamble {
65  uint16_t netHeaderSize;
66  int32_t netTransferSizeDummy; // layout compatibility, legacy detection
67  uint32_t netTransferSizes[MAX_DATA_BLOCKS]; // per-block total size
68  };
70  uint32_t segmentOffset;
71  };
73  uint32_t fragmentSize;
74  uint32_t segmentOffset;
75  };
76 #pragma pack(pop)
77 
86  DataBlockProtocol(bool server, ProtocolType protType, int maxUdpPacketSize);
87 
92  int getProtocolOverhead() const {
93  return protType == PROTOCOL_UDP ? sizeof(int) : 0;
94  }
95 
99  int getMaxReceptionSize() const;
100 
104  void resetTransfer();
105 
120  void setTransferHeader(unsigned char* data, int headerSize, int blocks);
121 
130  void setTransferBytes(int block, long bytes);
131 
146  void setTransferData(int block, unsigned char* data, int validBytes = 0x7FFFFFFF);
147 
157  void setTransferValidBytes(int block, int validBytes);
158 
168  const unsigned char* getTransferMessage(int& length);
169 
173  bool transferComplete();
174 
183  unsigned char* getNextReceiveBuffer(int maxLength);
184 
191  void resetReception(bool dropped);
192 
202  void processReceivedMessage(int length, bool& transferComplete);
203 
213  unsigned char* getReceivedData(int& length);
214 
226  unsigned char* getReceivedHeader(int& length);
227 
232  int getDroppedReceptions() const {
233  return droppedReceptions;
234  }
235 
243  bool newClientConnected();
244 
251  bool isConnected() const;
252 
263  const unsigned char* getNextControlMessage(int& length);
264 
265  unsigned char* getBlockReceiveBuffer(int block) {
266  if (block >= numReceptionBlocks) {
267  throw ProtocolException("Tried to get receive buffer beyond initialized block range");
268  }
269  return &blockReceiveBuffers[block][0];
270  }
271  int getBlockValidSize(int block) {
272  if (block >= numReceptionBlocks) {
273  throw ProtocolException("Tried to get valid buffer index beyond initialized block range");
274  }
275  return blockValidSize[block];
276  }
277  bool isBlockDone(int block) {
278  if (block >= numReceptionBlocks) {
279  throw ProtocolException("Tried to get completion status of uninitialized block");
280  }
281  return blockValidSize[block] >= blockReceiveSize[block];
282  }
283  bool allBlocksDone() {
284  for (int i=0; i<numReceptionBlocks; ++i) {
285  if (!isBlockDone(i)) return false;
286  }
287  return true;
288  }
289  bool anyPayloadReceived() {
290  for (int i=0; i<numReceptionBlocks; ++i) {
291  if (blockReceiveOffsets[i] > 0) return true;
292  }
293  return false;
294  }
295 
296  std::string statusReport();
297 
298 private:
299  // The pimpl idiom is not necessary here, as this class is usually not
300  // used directly
301 
302  struct MissingReceiveSegment {
303  int offset;
304  int length;
305  bool isEof;
306  unsigned char subsequentData[4];
307  };
308 
309  static constexpr int HEARTBEAT_INTERVAL_MS = 1000;
310  static constexpr int RECONNECT_TIMEOUT_MS = 1000;
311 
312  static constexpr unsigned char CONNECTION_MESSAGE = 0x01;
313  static constexpr unsigned char CONFIRM_MESSAGE = 0x02;
314  static constexpr unsigned char HEADER_MESSAGE = 0x03;
315  static constexpr unsigned char RESEND_MESSAGE = 0x04;
316  static constexpr unsigned char EOF_MESSAGE = 0x05;
317  static constexpr unsigned char HEARTBEAT_MESSAGE = 0x06;
318 
319  bool isServer;
320  ProtocolType protType;
321  int maxPayloadSize;
322  int minPayloadSize;
323 
324  // Transfer related variables
325  bool transferDone;
326  unsigned char* rawDataArr[MAX_DATA_BLOCKS];
327  int rawDataArrStrideHackOrig[MAX_DATA_BLOCKS];
328  int rawDataArrStrideHackRepl[MAX_DATA_BLOCKS];
329  int rawValidBytes[MAX_DATA_BLOCKS];
330  int transferOffset[MAX_DATA_BLOCKS];
331  int transferSize[MAX_DATA_BLOCKS];
332  char overwrittenTransferData[sizeof(SegmentHeaderTCP)];
333  int overwrittenTransferIndex;
334  int overwrittenTransferBlock;
335  unsigned char* transferHeaderData;
336  int transferHeaderSize;
337  int totalBytesCompleted;
338  int totalTransferSize;
339  int numTransferBlocks;
340  int lastTransmittedBlock;
341 
342  // Reliability related variables
343  std::deque<MissingReceiveSegment> missingReceiveSegments;
344  std::deque<std::pair<int, int> > missingTransferSegments;
345  bool waitingForMissingSegments;
346  int totalReceiveSize;
347 
348  unsigned char controlMessageBuffer[1024];
349 
350  // Connection related variables
351  bool connectionConfirmed;
352  bool confirmationMessagePending;
353  bool eofMessagePending;
354  bool clientConnectionPending;
355  bool resendMessagePending;
356  std::chrono::steady_clock::time_point lastRemoteHostActivity;
357  std::chrono::steady_clock::time_point lastSentHeartbeat;
358  std::chrono::steady_clock::time_point lastReceivedHeartbeat;
359 
360  // Reception related variables
361  std::vector<unsigned char, AlignedAllocator<unsigned char> > receiveBuffer;
362  std::vector<unsigned char, AlignedAllocator<unsigned char> > blockReceiveBuffers[MAX_DATA_BLOCKS];
363  int blockReceiveOffsets[MAX_DATA_BLOCKS];
364  int blockReceiveSize[MAX_DATA_BLOCKS];
365  int blockValidSize[MAX_DATA_BLOCKS];
366  std::vector<unsigned char> receivedHeader;
367  bool finishedReception;
368  int droppedReceptions;
369  int completedReceptions;
370  double lostSegmentRate;
371  int lostSegmentBytes;
372  unsigned char unprocessedMsgPart[MAX_OUTSTANDING_BYTES];
373  int unprocessedMsgLength;
374  bool headerReceived;
375  bool legacyTransfer;
376  int numReceptionBlocks;
377  int receiveOffset;
378 
379  const unsigned char* extractPayload(const unsigned char* data, int& length, bool& error);
380  bool processControlMessage(int length);
381  void restoreTransferBuffer();
382  bool generateResendRequest(int& length);
383  void getNextTransferSegment(int& block, int& offset, int& length);
384  void parseResendMessage(int length);
385  void parseEofMessage(int length);
386  void integrateMissingUdpSegments(int block, int lastSegmentOffset, int lastSegmentSize);
387  void processReceivedUdpMessage(int length, bool& transferComplete);
388  void processReceivedTcpMessage(int length, bool& transferComplete);
389  void resizeReceiveBuffer();
390  int parseReceivedHeader(int length, int offset);
391  void zeroStructures();
392  void splitRawOffset(int rawSegmentOffset, int& dataBlockID, int& segmentOffset);
393  int mergeRawOffset(int dataBlockID, int segmentOffset, int reserved=0);
394 
395 };
396 
397 }} // namespace
398 
399 #endif
unsigned char * getNextReceiveBuffer(int maxLength)
Gets a buffer for receiving the next network message.
const unsigned char * getTransferMessage(int &length)
Gets the next network message for the current transfer.
const unsigned char * getNextControlMessage(int &length)
If a control message is pending to be transmitted, then the message data will be returned by this met...
void setTransferData(int block, unsigned char *data, int validBytes=0x7FFFFFFF)
Sets the payload data for the next transfer.
bool isConnected() const
Returns true if a remote connection is established.
int getDroppedReceptions() const
Returns the internal counter of dropped transfers during reception.
void setTransferBytes(int block, long bytes)
Sets the per-block transfer size.
bool newClientConnected()
Returns true if the last network message has established a new connection from a client.
int getMaxReceptionSize() const
Returns the maximum payload size that can be received.
bool transferComplete()
Returns true if the current transfer has been completed.
void resetTransfer()
Resets all transfer related internal variables.
unsigned char * getReceivedData(int &length)
Returns the data that has been received for the current transfer.
void setTransferValidBytes(int block, int validBytes)
Updates the number of valid bytes in a partial transfer.
int getProtocolOverhead() const
Returns the size of the overhead data that is required for transferring a single network message...
void setTransferHeader(unsigned char *data, int headerSize, int blocks)
Sets a user-defined header that shall be transmitted with the next transfer.
A protocol for transmitting large blocks of data over a network.
unsigned char * getReceivedHeader(int &length)
Returns the header data that has been received for the current transfer.
void processReceivedMessage(int length, bool &transferComplete)
Handles a received network message.
Exception class that is used for all protocol exceptions.
Definition: exceptions.h:25
DataBlockProtocol(bool server, ProtocolType protType, int maxUdpPacketSize)
Creates a new instance.
void resetReception(bool dropped)
Resets the message reception.
Nerian Vision Technologies