From 9706325110b8cda429797bad1ce06e069ffa2c95 Mon Sep 17 00:00:00 2001
From: Anders Martinsson <anders.martinsson@intinor.se>
Date: Thu, 1 Mar 2018 10:20:01 +0100
Subject: [PATCH] Fails to often

Example program that show that decoding fails to often for K'+1 and K'+2.
The fail rate for K'-symbols is within the specification (5.8. in RFC 6330).

Signed-off-by: Anders Martinsson <anders.martinsson@intinor.se>
---
 issues/Makefile                      |  34 ++++++++
 issues/fails_to_often.cpp            | 147 +++++++++++++++++++++++++++++++++++
 src/RaptorQ/v1/wrapper/C_RAW_API.cpp |  58 ++++++++++++--
 src/RaptorQ/v1/wrapper/C_RAW_API.h   |   1 +
 4 files changed, 235 insertions(+), 5 deletions(-)
 create mode 100644 issues/Makefile
 create mode 100644 issues/fails_to_often.cpp

diff --git a/issues/Makefile b/issues/Makefile
new file mode 100644
index 0000000..980468b
--- /dev/null
+++ b/issues/Makefile
@@ -0,0 +1,34 @@
+CXX = g++
+CXXFLAGS += -O2 -g -std=c++11
+
+MACHINE := $(shell uname -m)
+ifeq ($(MACHINE),x86_64)
+	RAPTORQ__ARCH=x86_64
+	OBJDIR=bin64
+else
+	RAPTORQ__ARCH=i686
+	OBJDIR=bin32
+endif
+
+RAPTORQ_SDK_PATH = ../src
+RAPTORQ_LIB      = ../build/lib
+RAPTORQ_SDK_INC  = -I$(RAPTORQ_SDK_PATH) -I../external/eigen3
+RAPTORQ_SDK_LDFLAGS = -L$(RAPTORQ_LIB) -lRaptorQ -lpthread
+
+PROGS = fails_to_often
+PROGS := $(patsubst %,$(OBJDIR)/%,$(PROGS))
+all: $(PROGS) | objdirs
+
+objdirs:
+	@mkdir -p $(OBJDIR)
+
+
+# fails_to_often
+$(OBJDIR)/fails_to_often.o: fails_to_often.cpp Makefile | objdirs
+	$(CXX) ${CXXFLAGS} $(RAPTORQ_SDK_INC) -c -o $@ $<
+
+$(OBJDIR)/fails_to_often: $(OBJDIR)/fails_to_often.o | objdirs
+	$(CXX) -o $@ $(OBJDIR)/fails_to_often.o $(RAPTORQ_SDK_INC) $(RAPTORQ_SDK_LDFLAGS)
+
+clean:
+	rm -f $(OBJDIR)/*.o $(PROGS)
diff --git a/issues/fails_to_often.cpp b/issues/fails_to_often.cpp
new file mode 100644
index 0000000..c118381
--- /dev/null
+++ b/issues/fails_to_often.cpp
@@ -0,0 +1,147 @@
+#include <RaptorQ/v1/wrapper/C_RAW_API.h>
+#include <cassert>
+#include <cstring>
+#include <iostream>
+
+struct RaptorQ_v1 *raptorq_api;
+
+
+
+static bool run (const int64_t symbol_size,
+                                        const uint32_t symbols,
+                                        const uint32_t repair)
+{
+    char * encode_buffer = new char[(symbols +repair) * symbol_size];
+    char * swap_buffer = new char[(symbols +repair) * symbol_size];
+    char * decode_buffer = new char[symbols * symbol_size];
+    int * swap_order = new int[symbols + repair];
+
+    int buffer_offset = 0;
+
+    for (int i = 0; i < symbols; i++) {
+        for (int j = 0; j < symbol_size; j++) {
+            encode_buffer[buffer_offset + j] = rand() % 256;
+        }
+        buffer_offset += symbol_size;
+    }
+
+    struct RaptorQ_ptr *encoder;
+    struct RaptorQ_ptr *decoder;
+    struct RaptorQ_future_enc *future; // Async encoder init future
+
+    encoder = raptorq_api->Encoder(RQ_ENC_8, (RaptorQ_Block_Size)symbols, symbol_size);
+    assert (encoder != NULL);
+
+    int64_t res = raptorq_api->set_data(encoder, (void**)&encode_buffer, symbol_size * symbols);
+    assert(res == symbols * symbol_size);
+    future = raptorq_api->compute(encoder);
+
+    if (future) {
+        raptorq_api->future_wait((struct RaptorQ_future *)future);
+        RaptorQ_Error result = raptorq_api->enc_future_get(future);
+        assert(result == 0);
+        raptorq_api->future_free((struct RaptorQ_future **)&future);
+    }
+
+    raptorq_api->compute_sync(encoder);
+
+    for (uint32_t sym_id = 0; sym_id < symbols + repair; ++sym_id) {
+        swap_order[sym_id] = sym_id;
+    }
+    for (uint32_t rep_id = symbols; rep_id < symbols + repair; ++rep_id) {
+        void *temp = encode_buffer + buffer_offset;
+        uint64_t size = raptorq_api->encode(encoder, &temp, symbol_size, rep_id);
+
+        buffer_offset += symbol_size;
+        assert(size == symbol_size);
+    }
+
+    // Scramble
+    memcpy(swap_buffer, encode_buffer, symbol_size * (symbols + repair));
+    int swap_num_temp;
+    char swap_temp[symbol_size];
+    for (int i = 0; i < symbols + repair;i++) {
+        int j = rand() % (i + 1);
+        if (j != i) {
+            memcpy(swap_temp, swap_buffer + symbol_size * j, symbol_size);
+            memcpy(swap_buffer + symbol_size * j, swap_buffer + symbol_size * i, symbol_size);
+            memcpy(swap_buffer + symbol_size * i, swap_temp, symbol_size);
+
+            swap_num_temp = swap_order[j];
+            swap_order[j] = swap_order[i];
+            swap_order[i] = swap_num_temp;
+        }
+    }
+
+    // Decode
+    decoder = raptorq_api->Decoder(RQ_DEC_8, (RaptorQ_Block_Size)symbols, symbol_size, RQ_COMPLETE);
+    for (uint32_t sym_id = 0; sym_id < symbols + repair; sym_id++) {
+        void *temp = swap_buffer + symbol_size * sym_id;
+        raptorq_api->end_of_input(decoder);
+        raptorq_api->wait_sync(decoder);
+
+        RaptorQ_Error error = raptorq_api->add_symbol(decoder, &temp, symbol_size, swap_order[sym_id]);
+        if (raptorq_api->needed_symbols(decoder) <= 0) {
+            RaptorQ_Decoder_Result decoder_result = raptorq_api->decode_once(decoder);
+            if (decoder_result == RQ_DEC_DECODED) {
+                break;
+            }
+            std::cout << "Failed to decode: " << sym_id << std::endl;
+        }
+    }
+    for (uint32_t sym_id = 0; sym_id < symbols; sym_id++) {
+        void *temp = decode_buffer + symbol_size * sym_id;
+        RaptorQ_Error error = raptorq_api->decode_symbol(decoder, &temp, symbol_size, sym_id);
+    }
+
+    if (memcmp(encode_buffer, decode_buffer, symbol_size * symbols) != 0) {
+        std::cout << "Differ on: " << memcmp(encode_buffer, decode_buffer, symbol_size*symbols) << std::endl;
+        std::cout << "Size     : " << symbol_size*symbols << std::endl;
+    }
+    assert(memcmp(encode_buffer, decode_buffer, symbol_size * symbols) == 0);
+
+    delete decode_buffer;
+    delete encode_buffer;
+    delete swap_buffer;
+    delete swap_order;
+    raptorq_api->free(&decoder);
+    raptorq_api->free(&encoder);
+    return true;
+}
+
+int main (int argc, char **argv)
+{
+    raptorq_api = (struct RaptorQ_v1 *)RaptorQ_api(1);
+    assert (raptorq_api != NULL);
+
+    // max 100MB of cache
+    // up to block size 1012 (more or less)
+    raptorq_api->set_compression (RQ_COMPRESS_NONE);
+    raptorq_api->local_cache_size (100*1024*1024);
+
+    int block_size = 101;
+    int repair = 101 / 5;
+    int symbol_size = 1316;
+
+    int seed = 9873;
+    std::cout << "Random seed: " << seed << std::endl;
+    srand( seed );
+    run(symbol_size, block_size, repair);
+    seed = 14557;
+    std::cout << "Random seed: " << seed << std::endl;
+    srand( seed );
+    run(symbol_size, block_size, repair);
+    seed = 25007;
+    std::cout << "Random seed: " << seed << std::endl;
+    srand( seed );
+    run(symbol_size, block_size, repair);
+
+    for (int i = 0; i < 100000;i++) {
+        std::cout << "Random seed: " << i << std::endl;
+        srand( i );
+        run(symbol_size, block_size, repair);
+    }
+    RaptorQ_free_api((struct RaptorQ_base_api **)&raptorq_api);
+    return 0;
+}
+
diff --git a/src/RaptorQ/v1/wrapper/C_RAW_API.cpp b/src/RaptorQ/v1/wrapper/C_RAW_API.cpp
index d6ee89a..9ff8673 100644
--- a/src/RaptorQ/v1/wrapper/C_RAW_API.cpp
+++ b/src/RaptorQ/v1/wrapper/C_RAW_API.cpp
@@ -122,6 +122,7 @@ static RaptorQ_Error v1_add_symbol (const RaptorQ_ptr *dec, void **from,
                                                             const uint32_t esi);
 static bool v1_can_decode (const RaptorQ_ptr *dec);
 static uint16_t v1_needed_symbols (const RaptorQ_ptr *dec);
+static RaptorQ_Decoder_Result v1_decode_once (const RaptorQ_ptr *dec);
 static RaptorQ_Dec_Result v1_poll (const RaptorQ_ptr *dec);
 static RaptorQ_Dec_Result v1_wait_sync (const RaptorQ_ptr *dec);
 static RaptorQ_future_dec* v1_wait (const RaptorQ_ptr *dec);
@@ -196,6 +197,7 @@ RaptorQ_v1::RaptorQ_v1()
     add_symbol (&v1_add_symbol),
     can_decode (&v1_can_decode),
     needed_symbols (&v1_needed_symbols),
+    decode_once (&v1_decode_once),
     poll (&v1_poll),
     wait_sync (&v1_wait_sync),
     wait (&v1_wait),
@@ -650,34 +652,42 @@ static void v1_free (struct RaptorQ_ptr **ptr)
     case RaptorQ_type::RQ_ENC_8:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Encoder<uint8_t*, uint8_t*>*> ((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_ENC_16:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Encoder<uint16_t*, uint16_t*>*>((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_ENC_32:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Encoder<uint32_t*, uint32_t*>*>((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_ENC_64:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Encoder<uint64_t*, uint64_t*>*>((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_DEC_8:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Decoder<uint8_t*, uint8_t*>*> ((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_DEC_16:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Decoder<uint16_t*, uint16_t*>*>((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_DEC_32:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Decoder<uint32_t*, uint32_t*>*>((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_DEC_64:
         delete reinterpret_cast<
                 RaptorQ__v1::Impl::Decoder<uint64_t*, uint64_t*>*>((*ptr)->ptr);
+        delete *ptr;
         break;
     case RaptorQ_type::RQ_NONE:
         break;
@@ -1093,7 +1103,8 @@ static RaptorQ_Error v1_add_symbol (const RaptorQ_ptr *dec, void **from,
                                                     const size_t size,
                                                     const uint32_t esi)
 {
-    if (dec == nullptr || dec->ptr == nullptr)
+    if (dec == nullptr || dec->ptr == nullptr ||
+                                        from == nullptr || *from == nullptr)
         return RaptorQ_Error::RQ_ERR_WRONG_INPUT;
     uint8_t *f_8;
     uint16_t *f_16;
@@ -1102,7 +1113,7 @@ static RaptorQ_Error v1_add_symbol (const RaptorQ_ptr *dec, void **from,
     RaptorQ_Error err = RaptorQ_Error::RQ_ERR_NONE;
     switch (dec->type) {
     case RaptorQ_type::RQ_DEC_8:
-        f_8 = reinterpret_cast<uint8_t*> (from);
+        f_8 = reinterpret_cast<uint8_t*> (*from);
         err = static_cast<RaptorQ_Error> (reinterpret_cast<
                             RaptorQ__v1::Impl::Decoder<uint8_t*, uint8_t*>*> (
                                             dec->ptr)->add_symbol (f_8,
@@ -1110,7 +1121,7 @@ static RaptorQ_Error v1_add_symbol (const RaptorQ_ptr *dec, void **from,
         *from = f_8;
         return err;
     case RaptorQ_type::RQ_DEC_16:
-        f_16 = reinterpret_cast<uint16_t*> (from);
+        f_16 = reinterpret_cast<uint16_t*> (*from);
         err = static_cast<RaptorQ_Error> (reinterpret_cast<
                             RaptorQ__v1::Impl::Decoder<uint16_t*, uint16_t*>*> (
                                             dec->ptr)->add_symbol (f_16,
@@ -1118,7 +1129,7 @@ static RaptorQ_Error v1_add_symbol (const RaptorQ_ptr *dec, void **from,
         *from = f_16;
         return err;
     case RaptorQ_type::RQ_DEC_32:
-        f_32 = reinterpret_cast<uint32_t*> (from);
+        f_32 = reinterpret_cast<uint32_t*> (*from);
         err = static_cast<RaptorQ_Error> (reinterpret_cast<
                             RaptorQ__v1::Impl::Decoder<uint32_t*, uint32_t*>*> (
                                             dec->ptr)->add_symbol (f_32,
@@ -1126,7 +1137,7 @@ static RaptorQ_Error v1_add_symbol (const RaptorQ_ptr *dec, void **from,
         *from = f_32;
         return err;
     case RaptorQ_type::RQ_DEC_64:
-        f_64 = reinterpret_cast<uint64_t*> (from);
+        f_64 = reinterpret_cast<uint64_t*> (*from);
         err = static_cast<RaptorQ_Error> (reinterpret_cast<
                             RaptorQ__v1::Impl::Decoder<uint64_t*, uint64_t*>*> (
                                             dec->ptr)->add_symbol (f_64,
@@ -1205,6 +1216,43 @@ static uint16_t v1_needed_symbols (const RaptorQ_ptr *dec)
     return 0;
 }
 
+static RaptorQ_Decoder_Result v1_decode_once (const RaptorQ_ptr *dec)
+{
+    if (dec == nullptr || dec->ptr == nullptr)
+        return RQ_DEC_NEED_DATA;
+
+    RaptorQ_Decoder_Result cpp_res = RQ_DEC_STOPPED;
+    switch (dec->type) {
+    case RaptorQ_type::RQ_DEC_8:
+        cpp_res = static_cast<RaptorQ_Decoder_Result>(reinterpret_cast<
+                            RaptorQ__v1::Impl::Decoder<uint8_t*, uint8_t*>*> (
+                                                            dec->ptr)->decode_once());
+        break;
+    case RaptorQ_type::RQ_DEC_16:
+        cpp_res = static_cast<RaptorQ_Decoder_Result>(reinterpret_cast<
+                            RaptorQ__v1::Impl::Decoder<uint16_t*, uint16_t*>*> (
+                                                            dec->ptr)->decode_once());
+        break;
+    case RaptorQ_type::RQ_DEC_32:
+        cpp_res = static_cast<RaptorQ_Decoder_Result>(reinterpret_cast<
+                            RaptorQ__v1::Impl::Decoder<uint32_t*, uint32_t*>*> (
+                                                            dec->ptr)->decode_once());
+        break;
+    case RaptorQ_type::RQ_DEC_64:
+        cpp_res = static_cast<RaptorQ_Decoder_Result>(reinterpret_cast<
+                            RaptorQ__v1::Impl::Decoder<uint64_t*, uint64_t*>*> (
+                                                            dec->ptr)->decode_once());
+        break;
+    case RaptorQ_type::RQ_ENC_8:
+    case RaptorQ_type::RQ_ENC_16:
+    case RaptorQ_type::RQ_ENC_32:
+    case RaptorQ_type::RQ_ENC_64:
+    case RaptorQ_type::RQ_NONE:
+        break;
+    }
+    return cpp_res;
+}
+
 static RaptorQ_Dec_Result v1_poll (const RaptorQ_ptr *dec)
 {
 
diff --git a/src/RaptorQ/v1/wrapper/C_RAW_API.h b/src/RaptorQ/v1/wrapper/C_RAW_API.h
index 30f08c8..6398cd2 100644
--- a/src/RaptorQ/v1/wrapper/C_RAW_API.h
+++ b/src/RaptorQ/v1/wrapper/C_RAW_API.h
@@ -129,6 +129,7 @@ extern "C"
         bool (*const can_decode) (const struct RaptorQ_ptr *dec);
         uint16_t (*const needed_symbols) (const struct RaptorQ_ptr *dec);
 
+        RaptorQ_Decoder_Result (*const decode_once) (const struct RaptorQ_ptr *dec);
         struct RaptorQ_Dec_Result (*const poll) (const struct RaptorQ_ptr *dec);
         struct RaptorQ_Dec_Result (*const wait_sync) (
                                                 const struct RaptorQ_ptr *dec);
-- 
2.7.4

