CMakeLists.txt

@@ -521,10 +521,19 @@ endif()
 add_executable(test_cpp_rfc EXCLUDE_FROM_ALL test/test_cpp_rfc.cpp ${HEADERS_ONLY} ${HEADERS})
 target_compile_options(
     test_cpp_rfc PRIVATE
-    ${CXX_COMPILER_FLAGS}
+    ${CXX_COMPILER_FLAGS} "-DTEST_HDR_ONLY"
 )
 target_link_libraries(test_cpp_rfc ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
 
+# CPP interface - RFC interface (linked version)
+add_executable(test_cpp_rfc_linked EXCLUDE_FROM_ALL test/test_cpp_rfc.cpp ${HEADERS_ONLY} ${HEADERS})
+target_compile_options(
+    test_cpp_rfc_linked PRIVATE
+    ${CXX_COMPILER_FLAGS}
+)
+add_dependencies(test_cpp_rfc_linked RaptorQ)
+target_link_libraries(test_cpp_rfc_linked RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
+
 # CPP interface - RAW interface (header only)
 add_executable(test_cpp_raw EXCLUDE_FROM_ALL test/test_cpp_raw.cpp ${HEADERS_ONLY} ${HEADERS})
 target_compile_options(
@@ -569,7 +578,7 @@ target_compile_options(
 )
 target_link_libraries(example_cpp_raw ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
 
-add_custom_target(examples DEPENDS test_c test_cpp_rfc test_cpp_raw libRaptorQ-test example_cpp_raw)
+add_custom_target(examples DEPENDS test_c test_cpp_rfc test_cpp_rfc_linked test_cpp_raw test_cpp_raw_linked libRaptorQ-test example_cpp_raw)
 
 
 

src/RaptorQ/RFC6330_v1.hpp

 /*
- * Copyright (c) 2015-2016, Luca Fulchir<luca@fulchir.it>, All rights reserved.
+ * Copyright (c) 2015-2018, Luca Fulchir<luca@fulchir.it>, All rights reserved.
  *
  * This file is part of "libRaptorQ".
  *
@@ -21,7 +21,7 @@
 #pragma once
 
 #include "RaptorQ/v1/common.hpp"
+#include "RaptorQ/v1/caches.hpp"
 #include "RaptorQ/v1/wrapper/CPP_RFC_API.hpp"
 #include "RaptorQ/v1/wrapper/CPP_RFC_API_void.hpp"
 #include "RaptorQ/v1/Thread_Pool.hpp"
-

src/RaptorQ/v1/De_Interleaver.hpp

@@ -62,6 +62,8 @@ size_t De_Interleaver<Fwd_It>::operator() (Fwd_It &start, const Fwd_It end,
                                                     const uint8_t skip,
                                                     const uint16_t from_esi)
 {
+    if (start == end)
+        return 0;
     // return number of BYTES written
     size_t written = 0;
     int32_t byte = 0;
@@ -78,21 +80,21 @@ size_t De_Interleaver<Fwd_It>::operator() (Fwd_It &start, const Fwd_It end,
     using T = typename std::iterator_traits<Fwd_It>::value_type;
     T element = static_cast<T> (0);
     if (skip != 0) {
+        assert (skip < sizeof(T) && "De_Interleaver: skip too big");
         uint8_t *p = reinterpret_cast<uint8_t *> (&*start);
         for (size_t keep = 0; keep < skip; ++keep) {
             element += static_cast<T> (*(p++)) << keep * 8;
         }
     }
-    while (start != end && sub_blk < (_sub_blocks.num(0) + _sub_blocks.num(1))){
-        if ((written + offset_al) >= max_bytes)
-            break;
+    while ((start != end && sub_blk < (_sub_blocks.num(0) + _sub_blocks.num(1)))
+                            && (written + offset_al) < (max_bytes + skip)) {
         element += static_cast<T> (static_cast<uint8_t>((*_symbols)(esi, byte)))
                                                             << offset_al * 8;
         ++offset_al;
         if (offset_al == sizeof(T)) {
             *start = element;
             ++start;
-            written += offset_al;
+            written += sizeof(T);
             element = static_cast<T> (0);
             offset_al = 0;
         }
@@ -115,6 +117,7 @@ size_t De_Interleaver<Fwd_It>::operator() (Fwd_It &start, const Fwd_It end,
             }
         }
     }
+    assert(!(start == end && offset_al != 0) && "De_Interleaver: can't write");
     if (start != end && offset_al != 0) {
         // we have more stuff in "element", but not enough to fill
         // the iterator. Do not overwrite additional data of the iterator.
@@ -125,8 +128,8 @@ size_t De_Interleaver<Fwd_It>::operator() (Fwd_It &start, const Fwd_It end,
         ++start;
         written += offset_al;
     }
-    if (written > 0)
-        written -= skip;
+    written -= skip;
+    assert (written <= max_bytes && "De_Interleaver: too much writing");
     return written;
 }
 

src/RaptorQ/v1/RFC.hpp

@@ -141,8 +141,8 @@ public:
     size_t encode (Fwd_It &output, const Fwd_It end, const uint32_t esi,
                                                             const uint8_t sbn);
     // id: 8-bit sbn + 24 bit esi
-    size_t encode (Fwd_It &output, const Fwd_It end, const uint32_t &id);
-    size_t encode_packet (Fwd_It &output, const Fwd_It end, const uint32_t &id);
+    size_t encode (Fwd_It &output, const Fwd_It end, const uint32_t id);
+    size_t encode_packet (Fwd_It &output, const Fwd_It end, const uint32_t id);
 
     void free (const uint8_t sbn);
     uint8_t blocks() const;
@@ -685,7 +685,7 @@ std::pair<Error, uint8_t> Encoder<Rnd_It, Fwd_It>::get_report (
 
 template <typename Rnd_It, typename Fwd_It>
 size_t Encoder<Rnd_It, Fwd_It>::encode (Fwd_It &output, const Fwd_It end,
-                                                            const uint32_t &id)
+                                                            const uint32_t id)
 {
     const uint32_t host_id = RaptorQ__v1::Impl::Endian::b_to_h<uint32_t> (id);
     constexpr uint32_t mask = ~(static_cast<uint32_t>(0xFF) << 24);
@@ -740,7 +740,7 @@ size_t Encoder<Rnd_It, Fwd_It>::encode (Fwd_It &output, const Fwd_It end,
 
 template <typename Rnd_It, typename Fwd_It>
 size_t Encoder<Rnd_It, Fwd_It>::encode_packet (Fwd_It &output, const Fwd_It end,
-                                                            const uint32_t &id)
+                                                            const uint32_t id)
 {
     // RFC packet, section 4.4.2 page 11
 
@@ -1548,8 +1548,6 @@ uint64_t Decoder<In_It, Fwd_It>::decode_bytes (Fwd_It &start, const Fwd_It end,
                     sizeof(typename std::iterator_traits<Fwd_It>::value_type);
         if (bytes_written == 0)
             return written;
-        //new_skip = block_size (sbn) %
-        //          sizeof(typename std::iterator_traits<Fwd_It>::value_type);
         // if we ended decoding in the middle of a Fwd_It, do not advance
         // start too much, or we will end up having additional zeros.
         if (new_skip == 0) {
@@ -1557,14 +1555,8 @@ uint64_t Decoder<In_It, Fwd_It>::decode_bytes (Fwd_It &start, const Fwd_It end,
         } else {
             uint64_t it_written = bytes_and_skip /
                     sizeof(typename std::iterator_traits<Fwd_It>::value_type);
-            // RaptorQ handles at most 881GB per rfc, so
-            // casting uint64 to int64 is safe
             // we can not do "--start" since it's a forward iterator
-            #pragma clang diagnostic push
-            #pragma clang diagnostic ignored "-Wshorten-64-to-32"
-            #pragma clang diagnostic ignored "-Wsign-conversion"
-            start += std::max (static_cast<uint64_t>(0), it_written - 1);
-            #pragma clang diagnostic pop
+            start += it_written;
         }
     }
     return written;
@@ -1697,21 +1689,25 @@ uint32_t Decoder<In_It, Fwd_It>::block_size (const uint8_t sbn) const
     if (!operator bool())
         return 0;
 
-    size_t ret = 0;
+    uint32_t ret = 0;
     if (sbn < part.num (0)) {
         ret = part.size (0) * _symbol_size;
-    } else if (sbn - part.num (0) < part.num (1)) {
+    } else if ((sbn - part.num (0)) < part.num (1)) {
         ret = part.size (1) * _symbol_size;
     }
-    if (ret != 0 && sbn == (part.num (0) + part.num (1) - 1)) {
+    if (ret != 0 && (sbn + 1) == (part.num (0) + part.num (1))) {
         // the size of the data (_size) is different from the sum of the size of
         // all blocks. Get the real size, so we do not write more.
         // we obviously need to consider this only for the last block.
-        size_t left = ret;
-        left -= part.num (0) * part.size (0) * _symbol_size;
-        if (part.num (1) > 1)
-            left -= (part.num (1) - 1) * part.size (1) * _symbol_size;
-        ret = static_cast<uint32_t> (left);
+        size_t size_without_last;
+        if (sbn < part.num (0)) {
+            size_without_last = (part.num (0) - 1) * part.size (0) *
+                                                                _symbol_size;
+        } else {
+            size_without_last = part.num (0) * part.size (0) * _symbol_size +
+                            (part.num (1) - 1) * part.size (1) * _symbol_size;
+        }
+        ret = static_cast<uint32_t> (_size - size_without_last);
     }
     return ret;
 }

src/RaptorQ/v1/RFC_Iterators.hpp

@@ -73,8 +73,18 @@ public:
     {
         if (_enc == nullptr)
             return 0;
+        #ifdef RQ_HEADER_ONLY
         return _enc->encode (start, end,
                             RaptorQ__v1::Impl::Endian::h_to_b<uint32_t> (_id));
+        #else
+        void **_from = reinterpret_cast<void**> (&start);
+        void *_to = reinterpret_cast<void*> (end);
+        uint64_t res = _enc->encode (_from, _to,
+                            RaptorQ__v1::Impl::Endian::h_to_b<uint32_t> (_id));
+        Fwd_It *tmp = reinterpret_cast<Fwd_It*> (_from);
+        start = *tmp;
+        return res;
+        #endif
     }
     uint32_t id() const
         { return RaptorQ__v1::Impl::Endian::h_to_b<uint32_t> (_id); }
@@ -264,10 +274,18 @@ public:
 
     uint64_t operator() (In_It &start, const In_It end)
     {
-        if (_dec == nullptr || esi (_id) > _dec->symbols (block (_id)))
+        if (_dec == nullptr || esi() >= _dec->symbols (block()))
             return 0;
-        return _dec->decode_symbol (start, end,
-                                static_cast<uint16_t> (esi (_id)), block (_id));
+        #ifdef RQ_HEADER_ONLY
+        return _dec->decode_symbol (start, end, esi(), block());
+        #else
+        void **_from = reinterpret_cast<void**> (&start);
+        void *_to = reinterpret_cast<void*> (end);
+        uint64_t res = _dec->decode_symbol (_from, _to, esi(), block());
+        Fwd_It *tmp = reinterpret_cast<Fwd_It*> (_from);
+        start = *tmp;
+        return res;
+        #endif
     }
     uint32_t id() const
         { return RaptorQ__v1::Impl::Endian::h_to_b<uint32_t> (_id); }

src/RaptorQ/v1/RaptorQ_Iterators.hpp

@@ -83,8 +83,10 @@ public:
         #else
         void **_from = reinterpret_cast<void**> (&start);
         void *_to = reinterpret_cast<void*> (end);
-        return _enc->encode (_from, _to, _esi);
-        start = *reinterpret_cast<Fwd_It*> (_from);
+        uint64_t res = _enc->encode (_from, _to, _esi);
+        Fwd_It *tmp = reinterpret_cast<Fwd_It*> (_from);
+        start = *tmp;
+        return res;
         #endif
     }
     uint32_t id() const
@@ -158,8 +160,10 @@ public:
         #else
         void **_from = reinterpret_cast<void**> (&start);
         void *_to = reinterpret_cast<void*> (end);
-        return _dec->decode_symbol (_from, _to, _esi);
-        start = *reinterpret_cast<Fwd_It*> (_from);
+        Error err = _dec->decode_symbol (_from, _to, _esi);
+        Fwd_It *tmp = reinterpret_cast<Fwd_It*> (_from);
+        start = *tmp;
+        return err;
         #endif
     }
     uint16_t id() const

src/RaptorQ/v1/wrapper/CPP_RAW_API.hpp

@@ -172,7 +172,7 @@ Encoder<Rnd_It, Fwd_It>::~Encoder()
 
 template <typename Rnd_It, typename Fwd_It>
 Encoder<Rnd_It, Fwd_It>::operator bool() const
-    { return static_cast<bool> (*_encoder); }
+    { return static_cast<bool> (_encoder); }
 
 template <typename Rnd_It, typename Fwd_It>
 uint16_t Encoder<Rnd_It, Fwd_It>::symbols() const
@@ -267,7 +267,8 @@ size_t Encoder<Rnd_It, Fwd_It>::encode (Fwd_It &output, const Fwd_It end,
     void **_from = reinterpret_cast<void**> (&output);
     void *_to = reinterpret_cast<void*> (end);
     auto ret = _encoder.encode (_from, _to, id);
-    output = *reinterpret_cast<Fwd_It*> (_from);
+    Fwd_It *tmp = reinterpret_cast<Fwd_It*> (_from);
+    output = *tmp;
     return ret;
 }
 
@@ -311,7 +312,7 @@ Decoder<In_It, Fwd_It>::~Decoder()
 
 template <typename In_It, typename Fwd_It>
 Decoder<In_It, Fwd_It>::operator bool() const
-    { return static_cast<bool> (*_decoder); }
+    { return static_cast<bool> (_decoder); }
 
 template <typename In_It, typename Fwd_It>
 uint16_t Decoder<In_It, Fwd_It>::symbols() const
@@ -348,7 +349,8 @@ Error Decoder<In_It, Fwd_It>::add_symbol (In_It &from, const In_It to,
     void **_from = reinterpret_cast<void**> (&from);
     void *_to = reinterpret_cast<void*> (to);
     auto ret = _decoder.add_symbol (_from, _to, esi);
-    from = *reinterpret_cast<In_It*> (_from);
+    In_It *tmp = reinterpret_cast<In_It*> (_from);
+    from = *tmp;
     return ret;
 }
 
@@ -402,7 +404,8 @@ Error Decoder<In_It, Fwd_It>::decode_symbol (Fwd_It &start, const Fwd_It end,
     void **_from = reinterpret_cast<void**> (&start);
     void *_to = reinterpret_cast<void*> (end);
     auto ret = _decoder.decode_symbol (_from, _to, esi);
-    start = *reinterpret_cast<Fwd_It*> (_from);
+    Fwd_It *tmp = reinterpret_cast<Fwd_It*> (_from);
+    start = *tmp;
     return ret;
 }
 
@@ -415,7 +418,8 @@ Decoder_written Decoder<In_It, Fwd_It>::decode_bytes (Fwd_It &start,
     void **_from = reinterpret_cast<void**> (&start);
     void *_to = reinterpret_cast<void*> (end);
     auto ret = _decoder.decode_bytes (_from, _to, from_byte, skip);
-    start = *reinterpret_cast<Fwd_It*> (_from);
+    Fwd_It *tmp = reinterpret_cast<Fwd_It*> (_from);
+    start = *tmp;
     return ret;
 }
 

src/RaptorQ/v1/wrapper/CPP_RFC_API_void.cpp

@@ -354,7 +354,7 @@ size_t Encoder_void::encode (void** output, const void* end, const uint32_t esi,
     return ret;
 }
 
-size_t Encoder_void::encode (void** output, const void* end, const uint32_t &id)
+size_t Encoder_void::encode (void** output, const void* end, const uint32_t id)
 {
     const cast_enc _enc (_encoder);
     uint8_t *p_8;

src/RaptorQ/v1/wrapper/CPP_RFC_API_void.hpp

@@ -79,7 +79,7 @@ public:
     size_t precompute_max_memory ();
     size_t encode (void** output, const void* end, const uint32_t esi,
                                                             const uint8_t sbn);
-    size_t encode (void** output, const void* end, const uint32_t &id);
+    size_t encode (void** output, const void* end, const uint32_t id);
     void free (const uint8_t sbn);
     uint8_t blocks() const;
     uint32_t block_size (const uint8_t sbn) const;

test/test_cpp_rfc.cpp

@@ -18,7 +18,11 @@
  * along with libRaptorQ.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "../src/RaptorQ/RFC6330_v1_hdr.hpp"
+#if defined (TEST_HDR_ONLY)
+    #include "../src/RaptorQ/RFC6330_v1_hdr.hpp"
+#else
+    #include "../src/RaptorQ/RFC6330_v1.hpp"
+#endif
 #include <fstream>
 #include <iostream>
 #include <random>
@@ -67,11 +71,13 @@ bool decode (const uint32_t mysize, std::mt19937_64 &rnd, float drop_prob,
                 std::ceil(static_cast<float> (mysize) / sizeof(in_enc_align))));
     in_enc_align tmp = 0;
     uint8_t shift = 0;
-    uint8_t insert = 0;
+    uint8_t data_counter;
+    data_counter = 0;
     for (uint32_t i = 0; i < mysize; ++i) {
-        tmp += static_cast<in_enc_align> (insert) << shift * 8;
-        ++insert;
-        //tmp += static_cast<in_enc_align> (i) << shift * 8;
+        // insert predicatable data: for debugging
+        //tmp += static_cast<in_enc_align> (insert++) << shift * 8;
+        // insert random data
+        tmp += static_cast<in_enc_align> (distr (rnd)) << shift * 8;
         ++shift;
         if (shift >= sizeof(in_enc_align)) {
             myvec.push_back (tmp);
@@ -99,12 +105,11 @@ bool decode (const uint32_t mysize, std::mt19937_64 &rnd, float drop_prob,
     std::cout << "Subsymbol: " << subsymbol << " Symbol: " << symbol_size<<"\n";
     size_t aligned_symbol_size = static_cast<size_t> (
         std::ceil(static_cast<float> (symbol_size) / sizeof(out_enc_align)));
-    auto enc_it = myvec.begin();
+    auto enc_it = myvec.begin().base();
 
-    std::uniform_int_distribution<uint32_t> mem_distr (100, 200000);
-    RFC6330::Encoder<typename std::vector<in_enc_align>::iterator,
-                            typename std::vector<out_enc_align>::iterator> enc (
-                enc_it, myvec.end(), subsymbol, symbol_size, mem_distr(rnd));
+    std::uniform_int_distribution<uint32_t> mem_distr (10, 20000);
+    RFC6330::Encoder<in_enc_align*, out_enc_align*> enc (
+            enc_it, myvec.end().base(), subsymbol, symbol_size, mem_distr(rnd));
     std::cout << "Size: " << mysize << " Blocks: " <<
                                     static_cast<int32_t>(enc.blocks()) << "\n";
     if (!enc) {
@@ -143,9 +148,9 @@ bool decode (const uint32_t mysize, std::mt19937_64 &rnd, float drop_prob,
             std::vector<out_enc_align> source_sym;
             source_sym.reserve (aligned_symbol_size);
             source_sym.insert (source_sym.begin(), aligned_symbol_size, 0);
-            auto it = source_sym.begin();
+            auto it = source_sym.begin().base();
             // save the symbol
-            auto written = (*sym_it) (it, source_sym.end());
+            auto written = (*sym_it) (it, source_sym.end().base());
             if (written != aligned_symbol_size) {
                 std::cout << written << "-vs-" << aligned_symbol_size <<
                                     " Could not get the whole source symbol!\n";
@@ -169,9 +174,9 @@ bool decode (const uint32_t mysize, std::mt19937_64 &rnd, float drop_prob,
             std::vector<out_enc_align> repair_sym;
             repair_sym.reserve (aligned_symbol_size);
             repair_sym.insert (repair_sym.begin(), aligned_symbol_size, 0);
-            auto it = repair_sym.begin();
+            auto it = repair_sym.begin().base();
             // save the repair symbol
-            auto written = (*sym_it) (it, repair_sym.end());
+            auto written = (*sym_it) (it, repair_sym.end().base());
             if (written != aligned_symbol_size) {
                 std::cout << written << "-vs-" << aligned_symbol_size <<
                                     " Could not get the whole repair symbol!\n";
@@ -193,8 +198,7 @@ bool decode (const uint32_t mysize, std::mt19937_64 &rnd, float drop_prob,
     // encoding done. now "encoded" is the vector with the trnasmitted data.
     // let's decode it
 
-    RFC6330::Decoder<typename std::vector<in_dec_align>::iterator,
-                            typename std::vector<out_dec_align>::iterator>
+    RFC6330::Decoder<in_dec_align*, out_dec_align*>
                                                 dec (oti_common, oti_scheme);
     if (!dec) {
         std::cout << "Could not initialize decoder\n";
@@ -216,8 +220,8 @@ bool decode (const uint32_t mysize, std::mt19937_64 &rnd, float drop_prob,
     received.resize (out_size, 0);
 
     for (size_t i = 0; i < encoded.size(); ++i) {
-        auto it = encoded[i].second.begin();
-        auto err = dec.add_symbol (it, encoded[i].second.end(),
+        auto it = encoded[i].second.begin().base();
+        auto err = dec.add_symbol (it, encoded[i].second.end().base(),
                                                             encoded[i].first);
         if (err != RFC6330::Error::NONE && err != RFC6330::Error::NOT_NEEDED) {
             std::cout << "error adding?\n";
@@ -229,12 +233,12 @@ bool decode (const uint32_t mysize, std::mt19937_64 &rnd, float drop_prob,
 
     async_dec.wait();
 
-    auto re_it = received.begin();
+    auto re_it = received.begin().base();
     // decode all blocks
     // you can actually call ".decode(...)" as many times
     // as you want. It will only start decoding once
     // it has enough data.
-    auto decoded = dec.decode_bytes (re_it, received.end(), 0);
+    auto decoded = dec.decode_bytes (re_it, received.end().base(), 0);
 
     assert (mysize == dec.bytes());
     // "decoded" can be more than mysize if the alignments are different
@@ -299,6 +303,7 @@ int main (void)
                                 rnd_size (rnd, sizeof(uint8_t)), rnd, 20.0, 4);
         if (!ret)
             return -1;
+#if defined (TEST_HDR_ONLY)
         std::cout << "08-08-16\n";
         ret = decode<uint8_t, uint8_t, uint16_t> (
                                 rnd_size (rnd, sizeof(uint8_t)), rnd, 20.0, 4);
@@ -359,11 +364,13 @@ int main (void)
                                 rnd_size (rnd, sizeof(uint16_t)), rnd, 20.0, 4);
         if (!ret)
             return -1;
+#endif
         std::cout << "16-16-16\n";
         ret = decode<uint16_t, uint16_t, uint16_t> (
                                 rnd_size (rnd, sizeof(uint16_t)), rnd, 20.0, 4);
         if (!ret)
             return -1;
+#if defined (TEST_HDR_ONLY)
         std::cout << "16-16-32\n";
         ret = decode<uint16_t, uint16_t, uint32_t> (
                                 rnd_size (rnd, sizeof(uint16_t)), rnd, 20.0, 4);
@@ -424,6 +431,7 @@ int main (void)
                                 rnd_size (rnd, sizeof(uint32_t)), rnd, 20.0, 4);
         if (!ret)
             return -1;
+#endif
         std::cout << "32-32-32\n";
         ret = decode<uint32_t, uint32_t, uint32_t> (
                                 rnd_size (rnd, sizeof(uint32_t)), rnd, 20.0, 4);