Skip to content
Commits on Source (204)
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ a394b22a
CMakeFiles/
CMakeCache.txt
CMakeLists.txt.user
cmake_install.cmake
Makefile
/doc/*.aux
/doc/*.bbl
/doc/*.blg
/doc/*.idx
/doc/*.ilg
/doc/*.ind
/doc/*.log
/doc/*.out
/doc/*.pdf
/doc/*.synctex.gz
/doc/*.toc
/CMakeFiles/
/CMakeCache.txt
/CMakeLists.txt.user
/cmake_install.cmake
/Makefile
*~
build/
/build*
.gitlab-ci.yml
View file @ a394b22a
stages:
- makedoc
- deploy
- deploydoc
- build
job2:
job1:
stage: makedoc
script: mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=Release ../; make everything; cp doc/libRaptorQ.pdf ~/
script: mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=Release ../; make docs; cp doc/libRaptorQ-RFC6330.pdf ~/; cp doc/libRaptorQ-RaptorQ.pdf ~/
job2:
stage: deploydoc
script: git diff-tree --no-commit-id --name-only -r HEAD | grep ^doc/ || exit 0; cd /var/lib/gitlab-runner/libRaptorQ.wiki; git pull; cp ~/libRaptorQ-RFC6330.pdf .; cp ~/libRaptorQ-RaptorQ.pdf .; git add libRaptorQ-RFC6330.pdf; git add libRaptorQ-RaptorQ.pdf; git commit -m '(gitlab-runner) updated pdf'; git push;
job3:
stage: deploy
script: git diff-tree --no-commit-id --name-only -r HEAD | grep ^doc/ || exit 0; cd /var/lib/gitlab-runner/libRaptorQ.wiki; git pull; cp ~/libRaptorQ.pdf .; git add libRaptorQ.pdf; git commit -m '(gitlab-runner) updated pdf'; git push;
stage: build
script: mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=Release ../; make everything;
.gitmodules 0 → 100644
View file @ a394b22a
[submodule "external/lz4"]
path = external/lz4
url = https://github.com/Cyan4973/lz4.git
CHANGELOG
View file @ a394b22a
v1.0.0 (beta):
* C++98 compatibility (untested)
* we are now a header-only library that can be compiled and linked to C/C++98, with RFC and RAW API
* RAW API block size is now typesafe.
* RAW API does not use the interleaving code.
* API reworked to easy manage old/new API in the future
* new, simpler RAW API (single-block)
* computations can be stopped
* precomputation caching
* cached result can be compressed (raw/LZ4)
* RFC API: global multithreading instead of per-class
* RFC compliance
* CLI tool for raw API
* OSX/BSD support
v0.1.10 (unreleased):
* backported MAC/FreeBSD support (and incomplete windows)
* backported RFC compliance
v0.1.9:
* remember to bump up version in the cmake -.-'
* squashed warnings for gcc & clang in 32 bit mode.
v0.1.8:
* someone forgot to update the changelog :p
* was bugfixing only, no new feature
* fixed initializing encoder/decoder with more than 256 blocks
* fixed untriggerable bug on writing decoding data.
v0.1.4:
* finally reached a usable state
CMakeLists.txt
View file @ a394b22a
#
# Copyright (c) 2015, Luca Fulchir<luca@fulchir.it>, All rights reserved.
# Copyright (c) 2015-2016, Luca Fulchir<luca@fulchir.it>, All rights reserved.
#
# This file is part of "libRaptorQ".
#
......@@ -17,347 +17,584 @@
# and a copy of the GNU Lesser General Public License
# along with libRaptorQ. If not, see <http://www.gnu.org/licenses/>.
PROJECT(libRaptorQ)
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
AUX_SOURCE_DIRECTORY(. SRC_LIST)
SET(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build mode") # default build type
SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS Debug Release RelWithDebInfo MinSizeRel)
MESSAGE(STATUS "Build selected: ${CMAKE_BUILD_TYPE}")
# easy compiler choice...
SET(CMAKE_CXX_COMPILER "g++" CACHE STRING "C++ Compiler") # default compiler
SET(CMAKE_C_COMPILER "gcc" CACHE STRING "C Compiler") # default compiler
SET(STDLIB "libstdc++" CACHE STRING "Standard library to use")
SET_PROPERTY(CACHE CMAKE_CXX_COMPILER PROPERTY STRINGS clang++ g++ c++)
SET_PROPERTY(CACHE CMAKE_C_COMPILER PROPERTY STRINGS clang gcc cc)
SET_PROPERTY(CACHE STDLIB PROPERTY STRINGS libc++ libstdc++)
MARK_AS_ADVANCED(CLEAR
CMAKE_CXX_COMPILER
CMAKE_C_COMPILER
cmake_minimum_required (VERSION 2.8.12)
project (libRaptorQ)
enable_language(CXX)
enable_language(C)
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake; ${CMAKE_MODULE_PATH})
set(RQ_VERSION 1.0.0-rc2)
set(RQ_ABI 1)
message(STATUS "libRaptorQ version ${RQ_VERSION}")
add_definitions(-DRQ_VERSION="${RQ_VERSION}")
add_definitions(-DEIGEN_MPL2_ONLY)
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release")
endif()
include(GNUInstallDirs)
option(DYNAMIC_LIB "Build dynamic library" ON)
option(STATIC_LIB "Build static library" ON)
option(CLANG_STDLIB "Use clang's libc++" OFF)
option(CLI "BUild CLI tools" ON)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build Type")
set(USE_LZ4 "ON" CACHE STRING "Use LZ4 compression for result caching")
set(RQ_LINKER CACHE STRING "linker to use (auto/gold/ld/bsd)")
set(RQ_ENDIANNESS CACHE STRING "endianness of your system")
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS Debug Release RelWithDebInfo MinSizeRel)
set_property(CACHE USE_LZ4 PROPERTY STRINGS ON BUNDLED OFF)
set_property(CACHE RQ_LINKER PROPERTY STRINGS GOLD LD BSD)
set_property(CACHE RQ_ENDIANNESS PROPERTY STRINGS Auto BigEndian LittleEndian)
mark_as_advanced(FORCE RQ_LINKER RQ_ENDIANNESS RQ_GOLD RQ_LZ4_LIB)
if (USE_LZ4 MATCHES "BUNDLED")
set(USE_LZ4 "ON")
set(RQ_LZ4_USE_OWN TRUE)
endif()
if (NOT RQ_ENDIANNESS)
set(RQ_ENDIANNESS Auto)
endif()
if (RQ_ENDIANNESS MATCHES "Auto")
include(TestBigEndian)
TEST_BIG_ENDIAN(IS_BIG_ENDIAN)
if (IS_BIG_ENDIAN)
add_definitions(-DRQ_BIG_ENDIAN)
message (STATUS "Detected big endian machine")
else()
add_definitions(-DRQ_LITTLE_ENDIAN)
message (STATUS "Detected little endian machine")
endif()
elseif (RQ_ENDIANNESS MATCHES "BigEndian")
add_definitions(-DRQ_BIG_ENDIAN)
message (STATUS "Forced to big endian")
elseif (RQ_ENDIANNESS MATCHES "LittleEndian")
add_definitions(-DRQ_LITTLE_ENDIAN)
message (STATUS "Forced to little endian")
endif()
# supported linkers: gold, linux standard, bsd
if(NOT RQ_LINKER)
if(CMAKE_SYSTEM_NAME MATCHES "Darwin" OR CMAKE_SYSTEM_NAME MATCHES "FreeBSD" OR CMAKE_SYSTEM_NAME MATCHES "Windows")
set(RQ_LINKER BSD)
else()
find_program(RQ_GOLD ld.gold)
if (NOT RQ_GOLD MATCHES "RQ_GOLD-NOTFOUND")
set(RQ_LINKER GOLD)
else()
set(RQ_LINKER LD)
endif()
endif()
endif()
if(RQ_LINKER MATCHES GOLD)
message(STATUS "Using the GOLD linker")
elseif(RQ_LINKER MATCHES LD)
message(STATUS "Using the LD linker")
elseif(RQ_LINKER MATCHES BSD)
message(STATUS "Using the BSD linker")
else()
message(WARNING "Linker not found? Default no option.")
set(RQ_LINKER BSD)
endif()
# defaults: only enable LTO/PROFILING with known compilers
# exception: osx/BSD. llvm linker/profiling not default?
if ((CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") AND
(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") AND (NOT ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD"))
option(LTO "Link Time Optimization" ON)
option(PROFILING "Profiling: speedup library" ON)
else()
option(LTO "Link Time Optimization" OFF)
option(PROFILING "Profiling: speedup library" OFF)
endif()
message(STATUS "Build selected: ${CMAKE_BUILD_TYPE}")
if(NOT (STATIC_LIB MATCHES "ON"))
if (NOT (DYNAMIC_LIB MATCHES "ON"))
message(FATAL_ERROR "Do you want to actually build the library?")
endif()
endif()
# with LTO each compiler needs its own AR/NM/RANLIB
if(LTO MATCHES "ON")
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
find_program(RQ_AR "llvm-ar")
find_program(RQ_NM "llvm-nm")
find_program(RQ_RANLIB "llvm-ranlib")
find_program(RQ_LLVM_PROF llvm-profdata)
mark_as_advanced(FORCE RQ_LLVM_PROF)
if (RQ_LLVM_PROF MATCHES "RQ_LLVM_PROF-NOTFOUND")
message(FATAL_ERROR "Profiling support with clang requires \"llvm-profdata\". Disable profiling with cmake option \"-DPROFILING=OFF\"")
endif()
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
find_program(RQ_AR "gcc-ar")
find_program(RQ_NM "gcc-nm")
find_program(RQ_RANLIB "gcc-ranlib")
endif()
# check if we found the linker
if (RQ_AR STREQUAL "RQ_AR-NOTFOUND" OR RQ_NM STREQUAL "RQ_NM-NOTFOUND" OR RQ_RANLIB STREQUAL "RQ_RANLIB-NOTFOUND")
message(FATAL_ERROR "Can't find the linker programs. Please disable LTO with cmake option \"-DLTO=OFF\"")
endif()
set(CMAKE_AR ${RQ_AR})
set(CMAKE_NM ${RQ_NM})
set(CMAKE_RANLIB ${RQ_RANLIB})
mark_as_advanced(FORCE RQ_AR RQ_NM RQ_RANLIB)
message(STATUS "Link Time Optimization activated")
else()
message(FATAL_ERROR "Sorry, don't know how to do LTO with your compiler")
endif()
else()
message(STATUS "Link Time Optimization deactivated")
endif()
# Profiling is pretty compiler-specific....
if(PROFILING MATCHES "ON")
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")
message(STATUS "Profiling activated")
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
message(WARNING "Clang's profile support breaks deterministic builds!")
endif()
else()
message(FATAL_ERROR "Sorry, don't know how to profile with your compiler")
endif()
else()
message(STATUS "Profiling deactivated")
endif()
if(CLANG_STDLIB MATCHES "ON")
if(NOT (CMAKE_CXX_COMPILER_ID MATCHES "Clang"))
message(FATAL_ERROR "libc++ is only supported by clang")
endif()
endif()
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
set(STDLIB stdc++)
elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
if(CLANG_STDLIB MATCHES "ON")
set(RQ_STDLIB_FLAG -stdlib=libc++)
set(STDLIB c++)
else()
set(STDLIB stdc++)
endif()
endif()
find_package(Threads REQUIRED)
find_package(git)
find_package(eigen REQUIRED)
include_directories(SYSTEM PUBLIC ${EIGEN3_INCLUDE_DIR})
include_directories(SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/external/optionparser-1.4")
if (USE_LZ4 MATCHES "ON")
find_package(RQ_LZ4 REQUIRED)
include_directories(SYSTEM PUBLIC ${RQ_LZ4_INCLUDE_DIR})
add_definitions(-DRQ_USE_LZ4)
endif()
#lz4 build if necessary
include(${CMAKE_CURRENT_SOURCE_DIR}/external/build_deps.cmake)
if (CLI MATCHES "ON")
message(STATUS "Building CLI tools")
else()
message(STATUS "NOT Building CLI tools")
endif()
# our include tree
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
# LATEX documentation
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/doc)
if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.git/ AND GIT_FOUND)
execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --sq HEAD OUTPUT_VARIABLE RQ_SEED)
else()
set(RQ_SEED RaptorQ_nongit_v${RQ_VERSION})
message(WARNING "Not inside a git repository. Compiler seed is now constant.")
endif()
set(DETERMINISTIC -frandom-seed=${RQ_SEED})
if(CMAKE_SYSTEM_NAME MATCHES "Windows")
add_definitions(-DRQ_WINDOWS)
message(WARNING, "RaptorQ builds are not deterministic in Windows yet")
else()
add_definitions(-DRQ_UNIX)
endif()
SET(SOURCES
src/RaptorQ/v1/wrapper/C_RAW_API.cpp
src/RaptorQ/v1/wrapper/C_RFC_API.cpp
src/RaptorQ/v1/wrapper/CPP_RAW_API_void.cpp
src/RaptorQ/v1/wrapper/CPP_RFC_API_void.cpp
src/RaptorQ/v1/wrapper/CPP_caches.cpp
)
SET(HEADERS
src/RaptorQ/v1/block_sizes.hpp
src/RaptorQ/v1/caches.hpp
src/RaptorQ/v1/caches.ipp
src/RaptorQ/v1/common.hpp
src/RaptorQ/v1/De_Interleaver.hpp
src/RaptorQ/v1/Decoder.hpp
src/RaptorQ/v1/degree.hpp
src/RaptorQ/v1/Encoder.hpp
src/RaptorQ/v1/Interleaver.hpp
src/RaptorQ/v1/multiplication.hpp
src/RaptorQ/v1/Octet.hpp
src/RaptorQ/v1/Operation.hpp
src/RaptorQ/v1/Parameters.hpp
src/RaptorQ/v1/Precode_Matrix.hpp
src/RaptorQ/v1/Precode_Matrix_Init.hpp
src/RaptorQ/v1/Precode_Matrix_Solver.hpp
src/RaptorQ/v1/Rand.hpp
src/RaptorQ/v1/RaptorQ.hpp
src/RaptorQ/v1/RaptorQ_Iterators.hpp
src/RaptorQ/v1/RFC.hpp
src/RaptorQ/v1/RFC_Iterators.hpp
src/RaptorQ/v1/Shared_Computation/Decaying_LF.hpp
src/RaptorQ/v1/table2.hpp
src/RaptorQ/v1/Thread_Pool.hpp
src/RaptorQ/v1/util/Bitmask.hpp
src/RaptorQ/v1/util/div.hpp
src/RaptorQ/v1/util/endianess.hpp
src/RaptorQ/v1/util/Graph.hpp
)
SET(HEADERS_LINKED
src/RaptorQ/RaptorQ.h
src/RaptorQ/RaptorQ_v1.hpp
src/RaptorQ/RFC6330.h
src/RaptorQ/RFC6330_v1.hpp
src/RaptorQ/v1/wrapper/C_common.h
src/RaptorQ/v1/wrapper/C_RAW_API.h
src/RaptorQ/v1/wrapper/C_RFC_API.h
src/RaptorQ/v1/wrapper/CPP_RFC_API.hpp
src/RaptorQ/v1/wrapper/CPP_RFC_API_void.hpp
src/RaptorQ/v1/wrapper/CPP_RAW_API.hpp
src/RaptorQ/v1/wrapper/CPP_RAW_API_void.hpp
)
SET(HEADERS_ONLY
src/RaptorQ/RaptorQ_v1_hdr.hpp
src/RaptorQ/RFC6330_v1_hdr.hpp
)
if(CMAKE_SYSTEM_NAME MATCHES "Windows")
# windows-only placeholder
else()
# linux-only placeholder
endif()
if (USE_LZ4 MATCHES "ON")
SET (HEADERS ${HEADERS}
src/RaptorQ/v1/Shared_Computation/LZ4_Wrapper.hpp)
endif()
include(cmake/compiler_flags.cmake)
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
add_definitions(-DUSING_CLANG)
if (LTO MATCHES "ON")
if (RQ_LINKER MATCHES GOLD)
set(LD_OPT "-flto -fuse-ld=gold" )
else()
set(LD_OPT "-flto" )
endif()
endif()
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
set(LD_OPT "")
endif()
########################################
## TODO: if windows, add notelemetry.obj
########################################
if(PROFILING MATCHES "ON")
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set(PROFILE_GET -fprofile-instr-generate)
set(PROFILE_GET_FILE RaptorQ.profraw)
set(PROFILE_SET_FILE RaptorQ.profdata)
set(PROFILE_SET "-fprofile-instr-use=${PROFILE_SET_FILE}")
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
set(PROFILE_GET -fprofile-generate)
# gcc creates a couple more files than clang for prifiling. Track them.
set(PROFILE_GET_FILE
CMakeFiles/test_c_profiled.dir/test/test_c.c.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Shared_Computation/Decaying_LF.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Graph.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Rand.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Operation.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/RFC.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Precode_Matrix_Instantiation.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Parameters.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Bitmask.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/Thread_Pool.cpp.gcda
CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/API_Wrapper_C.cpp.gcda)
set(PROFILE_SET_FILE ${PROFILE_GET_FILE})
set(PROFILE_SET -fprofile-use)
endif()
# PRE-run: build a library, generate the profile.
add_library(RaptorQ_Static_Profiling STATIC ${SOURCES} ${HEADERS} ${HEADERS_LINKED})
target_link_libraries(RaptorQ_Static_Profiling ${STDLIB} ${RQ_LZ4_DEP} ${RQ_UBSAN})
add_dependencies(RaptorQ_Static_Profiling LZ4)
target_compile_options(
RaptorQ_Static_Profiling PRIVATE
${CXX_COMPILER_FLAGS}
${PROFILE_GET}
)
set_target_properties(RaptorQ_Static_Profiling PROPERTIES COMPILER_FLAGS ${PROFILE_GET})
# build the C example test for profiling
add_executable(test_c_profiled test/test_c.c)
add_dependencies(test_c_profiled RaptorQ_Static_Profiling)
target_compile_options(
test_c_profiled PRIVATE
${C_COMPILER_FLAGS}
${PROFILE_GET}
)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(test_c_profiled RaptorQ_Static_Profiling ${STDLIB} m ${CMAKE_THREAD_LIBS_INIT} ${RQ_UBSAN})
else()
target_link_libraries(test_c_profiled RaptorQ_Static_Profiling ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_UBSAN})
endif()
set_target_properties(test_c_profiled PROPERTIES LINK_FLAGS ${PROFILE_GET})
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
add_custom_command(
OUTPUT ${PROFILE_GET_FILE}
COMMAND LLVM_PROFILE_FILE=${PROFILE_GET_FILE} ./test_c_profiled
DEPENDS test_c_profiled
COMMENT "Running profiling test..."
VERBATIM
)
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
add_custom_command(
OUTPUT ${PROFILE_GET_FILE}
COMMAND ./test_c_profiled
DEPENDS test_c_profiled
COMMENT "Running profiling test..."
VERBATIM
)
endif()
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
# the profile must be translated into readable form
add_custom_command(
OUTPUT ${PROFILE_SET_FILE}
COMMAND llvm-profdata merge -output=${PROFILE_SET_FILE} ${PROFILE_GET_FILE}
DEPENDS ${PROFILE_GET_FILE}
COMMENT "Creating profile data..."
VERBATIM
)
endif()
add_custom_target(
profile ALL
DEPENDS ${PROFILE_SET_FILE}
)
endif()
# build the static library
if(STATIC_LIB MATCHES "ON")
add_library(RaptorQ_Static STATIC ${SOURCES} ${HEADERS} ${HEADERS_LINKED})
add_dependencies(RaptorQ_Static LZ4)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
set_target_properties(RaptorQ_Static PROPERTIES OUTPUT_NAME RaptorQ.${RQ_ABI})
endif()
target_compile_options(
RaptorQ_Static PRIVATE
${CXX_COMPILER_FLAGS}
)
if(LTO MATCHES "ON")
target_compile_options(RaptorQ_Static PRIVATE -flto)
endif()
if(PROFILING MATCHES "ON")
add_dependencies(RaptorQ_Static profile)
target_compile_options(RaptorQ_Static PRIVATE ${PROFILE_SET})
endif()
if (NOT RQ_LINKER MATCHES BSD)
set_target_properties (
RaptorQ_Static
PROPERTIES
LINK_FLAGS "-Wl,-z,now,-z,relro${RQ_LZ4_EXCLUDE_SYM} -pie ${LD_OPT}"
)
endif()
set_property(TARGET RaptorQ_Static PROPERTY ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib")
if(CMAKE_SYSTEM_NAME MATCHES "Windows")
# not really deterministic?
# don't know how to do that in Windows.
add_custom_target(
make_static_deterministic ALL
DEPENDS RaptorQ_Static
)
else()
# make the library deterministic (only on *nix)
add_executable(make_deterministic src/deterministic.cpp)
target_compile_options(
make_deterministic PRIVATE
${CXX_COMPILER_FLAGS}
)
target_link_libraries(make_deterministic ${STDLIB} ${RQ_UBSAN})
add_custom_command(
OUTPUT deterministic.run
COMMAND make_deterministic ${CMAKE_CURRENT_BINARY_DIR}/lib/libRaptorQ.${RQ_ABI}.a
DEPENDS RaptorQ_Static
COMMENT "Removing creation date from library..."
VERBATIM
)
add_custom_target(
make_static_deterministic ALL
DEPENDS deterministic.run
)
endif()
endif()
#build dynamic library
if(DYNAMIC_LIB MATCHES "ON")
add_definitions(-DRAPTORQ_DLL)
add_definitions(-DRAPTORQ_DLL_EXPORTS)
add_library(RaptorQ SHARED ${SOURCES} ${HEADERS} ${HEADERS_LINKED})
add_dependencies(RaptorQ LZ4)
target_link_libraries(RaptorQ ${RQ_LZ4_DEP} ${RQ_UBSAN})
target_compile_options(
RaptorQ PRIVATE
${CXX_COMPILER_FLAGS}
)
if(LTO MATCHES "ON")
target_compile_options(RaptorQ PRIVATE -flto)
endif()
if(PROFILING MATCHES "ON")
add_dependencies(RaptorQ profile)
target_compile_options(RaptorQ PRIVATE ${PROFILE_SET})
endif()
if (RQ_LINKER MATCHES BSD)
# keep the last sace at the end here, otherwise cmake gets called
# with the wrong number of parameters in the next set_target_properties
set(RQ_LINK_LIB "${LD_OPT} ")
else()
set(RQ_LINK_LIB "-Wl,-z,now,-z,relro${RQ_LZ4_EXCLUDE_SYM} ${LD_OPT}")
endif()
set_target_properties (
RaptorQ
PROPERTIES
LINK_FLAGS ${RQ_LINK_LIB}
SOVERSION ${RQ_ABI}
VERSION ${RQ_VERSION}
)
set_property(TARGET RaptorQ PROPERTY LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib")
endif()
# benchamrks (header only)
add_executable(libRaptorQ-test EXCLUDE_FROM_ALL test/rfc_test.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options(
libRaptorQ-test PRIVATE
${CXX_COMPILER_FLAGS}
)
target_link_libraries(libRaptorQ-test ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# Link time optimization: smaller, better optimized libraries
SET(LTO "yes" CACHE STRING "Link Time Optimization")
SET_PROPERTY(CACHE LTO PROPERTY STRINGS yes no)
# profiling support: optimize the library
SET(PROFILING "yes" CACHE STRING "Optimize library assembly")
SET_PROPERTY(CACHE PROFILING PROPERTY STRINGS yes no)
SET(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake; ${CMAKE_MODULE_PATH})
FIND_PACKAGE(Threads REQUIRED)
FIND_PACKAGE(git)
FIND_PACKAGE(eigen REQUIRED)
INCLUDE_DIRECTORIES(SYSTEM ${EIGEN3_INCLUDE_DIR})
#This only works if *FIRST* you checkout a commit and *THEN* you run cmake
IF(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.git/)
MESSAGE(STATUS "directory found")
IF(GIT_FOUND)
EXECUTE_PROCESS(COMMAND ${GIT_EXECUTABLE} rev-parse --sq HEAD OUTPUT_VARIABLE SEED)
SET(DETERMINISTIC "-frandom-seed=${SEED}")
ENDIF()
ELSE()
SET(DETERMINISTIC "-frandom-seed=RaptorQ_nongit")
MESSAGE(WARNING "Not inside a git repository. compiler seed is now constant.")
ENDIF()
IF("${LTO}" STREQUAL "yes")
MESSAGE(STATUS "Link Time Optimization activated")
ELSE()
MESSAGE(STATUS "Link Time Optimization deactivated")
ENDIF()
#activate profiling if requested.
# profiling needs to build a test, which needs threading.
IF (PROFILING STREQUAL "yes")
MESSAGE(STATUS "Profiling activated")
ELSE()
MESSAGE(STATUS "Profiling deactivated")
SET (PROFILING "no")
ENDIF()
# setup the compiler and the standard library
IF(STDLIB)
IF(${STDLIB} STREQUAL "libc++" AND NOT (${CMAKE_CXX_COMPILER} MATCHES "^.*c[+][+]$" OR ${CMAKE_CXX_COMPILER} MATCHES "^(.*/)g[+][+]$"))
# gcc is particoularly unhappy with libc++ headers...
INCLUDE_DIRECTORIES(SYSTEM ${CLANG_C_INCLUDE_DIR})
FIND_PACKAGE(std_clang REQUIRED)
INCLUDE_DIRECTORIES(SYSTEM ${CLANG_STD_INCLUDE_DIR})
ENDIF()
IF(${STDLIB} STREQUAL "libstdc++" OR (${CMAKE_CXX_COMPILER} MATCHES "^.*c[+][+]$" OR ${CMAKE_CXX_COMPILER} MATCHES "^(.*/)g[+][+]$"))
INCLUDE_DIRECTORIES(SYSTEM ${GCC_C_INCLUDE_DIR})
FIND_PACKAGE(std_gcc REQUIRED)
INCLUDE_DIRECTORIES(SYSTEM ${GCC_STD_INCLUDE_DIR})
ENDIF()
ENDIF(STDLIB)
# generic standard library support, put it as compiler option:
IF(${STDLIB} MATCHES "^.+$")
SET(STDLIB_FLAG "-stdlib=${STDLIB}")
ENDIF()
# build examples
# C interface (linked)
add_executable(test_c EXCLUDE_FROM_ALL test/test_c.c)
target_compile_options(
test_c PRIVATE
${C_COMPILER_FLAGS}
)
#
# C++ options
#
IF(${CMAKE_CXX_COMPILER} MATCHES ".*clang.*")
IF (PROFILING STREQUAL "yes")
MESSAGE(WARNING "Profiling and clang breaks deterministic builds")
SET (PROFILE_GET "-fprofile-instr-generate -g")
SET (PROFILE_GET_FILE "RaptorQ.profraw")
SET (PROFILE_SET_FILE "RaptorQ.profdata")
SET (PROFILE_SET "-fprofile-instr-use=${PROFILE_SET_FILE}")
ENDIF()
IF("${LTO}" STREQUAL "yes")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -fuse-ld=gold")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flto -fuse-ld=gold")
FIND_PROGRAM(LLVM_AR "llvm-ar")
FIND_PROGRAM(LLVM_NM "llvm-nm")
FIND_PROGRAM(LLVM_RANLIB "llvm-ranlib")
SET(CMAKE_AR ${LLVM_AR})
SET(LINK_FLAGS "-rng-seed=${SEED} --build-id=none")
SET(CMAKE_NM ${LLVM_NM})
SET(CMAKE_RANLIB ${LLVM_RANLIB})
ENDIF()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STDLIB_FLAG} ${DETERMINISTIC} -std=c++11 -fno-rtti -fno-exceptions -Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat -Wno-padded -fstack-protector-all -fstrict-aliasing ")
SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} ${STDLIB_FLAG} ${DETERMINISTIC} -std=c++11 -O0 -g -fno-rtti -fno-exceptions -Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat -Wno-padded -fstack-protector-all -fstrict-aliasing ")
SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS} ${STDLIB_FLAG} ${DETERMINISTIC} -std=c++11 -Os -DNDEBUG -fno-rtti -fno-exceptions -Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat -Wno-padded -fstack-protector-all -fstrict-aliasing ")
SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} ${STDLIB_FLAG} ${DETERMINISTIC} -std=c++11 -Ofast -fvisibility=hidden -fvisibility-inlines-hidden -DNDEBUG -fno-rtti -fno-exceptions -Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -Wformat -Wformat-security -fPIE -fPIC ")
SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS} ${STDLIB_FLAG} ${DETERMINISTIC} -std=c++11 -Ofast -g -fvisibility=hidden -fvisibility-inlines-hidden -DNDEBUG -fno-rtti -fno-exceptions -Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -Wformat -Wformat-security -fPIE -fPIC ")
ADD_DEFINITIONS(-DUSING_CLANG)
SET(USE_CLANG 1)
ELSE()
# warn: clang+ and g++ get the same *g++ regexp, keep the "else" above or match the "/" in the regexp
IF(${CMAKE_CXX_COMPILER} MATCHES "^.*c[+][+]$" OR ${CMAKE_CXX_COMPILER} MATCHES "^(.*/)g[+][+]$")
IF (PROFILING STREQUAL "yes")
SET (PROFILE_GET "-fprofile-generate")
# gcc creates a couple more files than clang for prifiling. Track them.
SET (PROFILE_GET_FILE
CMakeFiles/RaptorQ_static_profiling.dir/src/Graph.cpp.gcda
CMakeFiles/RaptorQ_static_profiling.dir/src/Parameters.cpp.gcda
CMakeFiles/RaptorQ_static_profiling.dir/src/Rand.cpp.gcda
CMakeFiles/RaptorQ_static_profiling.dir/src/Precode_Matrix.cpp.gcda
CMakeFiles/RaptorQ_static_profiling.dir/src/Precode_Matrix_solver.cpp.gcda
CMakeFiles/RaptorQ_static_profiling.dir/src/cRaptorQ.cpp.gcda
CMakeFiles/test_c_profiled.dir/test/test_c.c.gcda)
SET (PROFILE_SET_FILE ${PROFILE_GET_FILE})
SET (PROFILE_SET "-fprofile-use")
ENDIF()
IF("${LTO}" STREQUAL "yes")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -fuse-ld=gold")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flto -fuse-ld=gold")
FIND_PROGRAM(GCC_AR "gcc-ar")
FIND_PROGRAM(GCC_NM "gcc-nm")
FIND_PROGRAM(GCC_RANLIB "gcc-ranlib")
SET(CMAKE_AR ${GCC_AR})
SET(LINK_FLAGS "-D")
SET(CMAKE_NM ${GCC_NM})
SET(CMAKE_RANLIB ${GCC_RANLIB})
ENDIF()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 ${DETERMINISTIC} -fno-rtti -fno-exceptions -Wno-unknown-pragmas -Wall -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv")
# FIXME: some of these are gcc4.9+ only
SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -std=c++11 ${DETERMINISTIC} -fno-rtti -fno-exceptions -Wno-unknown-pragmas -O0 -g -Wall -pedantic -Wno-padded -fstack-protector-all -Wno-aggressive-loop-optimizations -fstrict-aliasing -ftrapv -fwrapv -fsanitize=undefined -fsanitize=shift -fsanitize=integer-divide-by-zero -fsanitize=vla-bound -fsanitize=null -fsanitize=return -fsanitize=signed-integer-overflow -fisolate-erroneous-paths-dereference -fisolate-erroneous-paths-attribute ")
SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS} -std=c++11 ${DETERMINISTIC} -fno-rtti -fno-exceptions -Wno-unknown-pragmas -Os -DNDEBUG -Wall -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing ")
SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -std=c++11 ${DETERMINISTIC} -fno-rtti -fno-exceptions -Wno-unknown-pragmas -Ofast -fvisibility=hidden -fvisibility-inlines-hidden -O4 -DNDEBUG -Wall -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -Wformat -Wformat-security -fPIE -fPIC ")
SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS} -std=c++11 ${DETERMINISTIC} -fno-rtti -fno-exceptions -Wno-unknown-pragmas -Ofast -g -fvisibility=hidden -fvisibility-inlines-hidden -O4 -DNDEBUG -Wall -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -Wformat -Wformat-security -fPIE -fPIC ")
ENDIF()
# else I do not know your compiler options :)
ENDIF()
#
# We hae a C interface example to build.
# set the proper options for that, too
#
IF(${CMAKE_C_COMPILER} MATCHES ".*clang.*")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${DETERMINISTIC} -std=c11 -Wall -pedantic -Weverything -Wno-padded -fstack-protector-all -fstrict-aliasing -Wno-disabled-macro-expansion ")
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS} ${DETERMINISTIC} -std=c11 -O0 -g -Wall -pedantic -Weverything -Wno-padded -fstack-protector-all -fstrict-aliasing -Wno-disabled-macro-expansion ")
SET(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS} ${DETERMINISTIC} -std=c11 -Os -fvisibility=hidden -fvisibility-inlines-hidden -DNDEBUG -Wall -pedantic -Weverything -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -Wformat -Wformat-security -fPIE -fPIC -Wno-disabled-macro-expansion ")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} ${DETERMINISTIC} -std=c11 -Ofast -fvisibility=hidden -fvisibility-inlines-hidden -DNDEBUG -Wall -pedantic -Weverything -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -Wformat -Wformat-security -fPIE -fPIC -Wno-disabled-macro-expansion ")
ADD_DEFINITIONS(-DUSING_CLANG)
ELSE()
# warn: clang+ and g++ get the same *g++ regexp, keep the "else" above or match the "/" in the regexp
IF(${CMAKE_C_COMPILER} MATCHES "^.*cc$" OR ${CMAKE_C_COMPILER} MATCHES "^(.*/)gcc$")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 ${DETERMINISTIC} -Wno-unknown-pragmas -Wall -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv")
# FIXME: some of these are gcc4.9+ only
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS} -std=c11 ${DETERMINISTIC} -Wno-unknown-pragmas -O0 -g -Wall -pedantic -Wno-padded -fstack-protector-all -Wno-aggressive-loop-optimizations -fstrict-aliasing -ftrapv -fwrapv -fsanitize=undefined -fsanitize=shift -fsanitize=integer-divide-by-zero -fsanitize=vla-bound -fsanitize=null -fsanitize=return -fsanitize=signed-integer-overflow -fisolate-erroneous-paths-dereference -fisolate-erroneous-paths-attribute ")
SET(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS} -std=c11 ${DETERMINISTIC} -Wno-unknown-pragmas -Os -DNDEBUG -Wall -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing ")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} -std=c11 ${DETERMINISTIC} -Wno-unknown-pragmas -Ofast -fvisibility=hidden -DNDEBUG -Wall -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -Wformat -Wformat-security -fPIE -fPIC ")
ENDIF()
# else I do not know your compiler options :)
ENDIF()
IF(APPLE)
MESSAGE(STATUS "MacOSX system found. WARN: not tested!")
ADD_DEFINITIONS(-DUNIX)
ENDIF()
IF(UNIX)
MESSAGE(STATUS "UNIX system found")
ADD_DEFINITIONS(-DUNIX)
ELSEIF(WIN32)
MESSAGE(STATUS "Wndows system found. WARN: not tested!")
ADD_DEFINITION(-DWIN32)
ENDIF()
SET(SOURCES src/Precode_Matrix_solver.cpp
src/Precode_Matrix.cpp
src/Rand.cpp
src/Parameters.cpp
src/Graph.cpp
src/cRaptorQ.cpp)
SET(HEADERS src/Interleaver.hpp
src/multiplication.hpp
src/table2.hpp
src/degree.hpp
src/common.hpp
src/Encoder.hpp
src/Decoder.hpp
src/Rand.hpp
src/Precode_Matrix.hpp
src/Parameters.hpp
src/Graph.hpp
src/De_Interleaver.hpp
src/RaptorQ.hpp
src/cRaptorQ.h)
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
add_dependencies(test_c RaptorQ)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
# Linux, *BSD
target_link_libraries(test_c RaptorQ m ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
else()
# windows: no "m" library
target_link_libraries(test_c RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
endif()
# CPP interface - RFC interface (header only)
add_executable(test_cpp_rfc EXCLUDE_FROM_ALL test/test_cpp_rfc.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options(
test_cpp_rfc PRIVATE
${CXX_COMPILER_FLAGS} "-DTEST_HDR_ONLY"
)
target_link_libraries(test_cpp_rfc ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# LATEX documentation
add_subdirectory(${CMAKE_SOURCE_DIR}/doc)
IF(PROFILING STREQUAL "yes")
# we need to build a profiling version of the static library.
# then we will run the profiling, and then finally rebuild
# everything with the profiling optimizations
# build the static library for profiling
ADD_LIBRARY(RaptorQ_static_profiling STATIC ${SOURCES} ${HEADERS})
# ar: add deterministic build
SET_TARGET_PROPERTIES (
RaptorQ_static_profiling
PROPERTIES
LINK_FLAGS "-Wl,-z,now,-z,relro -pie"
)
TARGET_LINK_LIBRARIES(RaptorQ_static_profiling stdc++)
SET_TARGET_PROPERTIES(RaptorQ_static_profiling PROPERTIES COMPILE_FLAGS ${PROFILE_GET})
# build the C example test for profiling
ADD_EXECUTABLE(test_c_profiled test/test_c.c)
ADD_DEPENDENCIES(test_c_profiled RaptorQ_static_profiling)
SET_TARGET_PROPERTIES(test_c_profiled PROPERTIES COMPILE_FLAGS ${PROFILE_GET})
TARGET_LINK_LIBRARIES(test_c_profiled ${LIBRARY_OUTPUT_PATH}/libRaptorQ_static_profiling.a)
STRING(SUBSTRING ${STDLIB} 3 -1 STDLIB_NAME)
TARGET_LINK_LIBRARIES(test_c_profiled ${STDLIB_NAME})
TARGET_LINK_LIBRARIES(test_c_profiled m) # math library
TARGET_LINK_LIBRARIES(test_c_profiled ${CMAKE_THREAD_LIBS_INIT})
SET_TARGET_PROPERTIES(test_c_profiled PROPERTIES LINK_FLAGS "${PROFILE_GET}")
# run the profiling and create the profile
IF(USE_CLANG)
add_custom_command(
OUTPUT ${PROFILE_GET_FILE}
COMMAND LLVM_PROFILE_FILE=${PROFILE_GET_FILE} ./test_c_profiled
DEPENDS test_c_profiled
COMMENT "Running profiling test..."
VERBATIM
)
ELSE()
add_custom_command(
OUTPUT ${PROFILE_GET_FILE}
COMMAND ./test_c_profiled
DEPENDS test_c_profiled
COMMENT "Running profiling test..."
VERBATIM
)
ENDIF()
IF(USE_CLANG)
# the profile must be translated into readable form
add_custom_command(
OUTPUT ${PROFILE_SET_FILE}
COMMAND llvm-profdata merge -output=${PROFILE_SET_FILE} ${PROFILE_GET_FILE}
DEPENDS ${PROFILE_GET_FILE}
COMMENT "Creating profile data..."
VERBATIM
)
ENDIF()
add_custom_target(
profile ALL
DEPENDS ${PROFILE_SET_FILE}
)
# now we can rebuild the test, the static library with the profiling optimizations.
ENDIF()
ADD_LIBRARY(RaptorQ_static STATIC ${SOURCES} ${HEADERS})
SET_TARGET_PROPERTIES (
RaptorQ_static
PROPERTIES
LINK_FLAGS "-Wl,-z,now,-z,relro -pie"
# CPP interface - RFC interface (linked version)
add_executable(test_cpp_rfc_linked EXCLUDE_FROM_ALL test/test_cpp_rfc.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options(
test_cpp_rfc_linked PRIVATE
${CXX_COMPILER_FLAGS}
)
add_dependencies(test_cpp_rfc_linked RaptorQ)
target_link_libraries(test_cpp_rfc_linked RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# CPP interface - RAW interface (header only)
add_executable(test_cpp_raw EXCLUDE_FROM_ALL test/test_cpp_raw.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options(
test_cpp_raw PRIVATE
${CXX_COMPILER_FLAGS} "-DTEST_HDR_ONLY"
)
IF(PROFILING STREQUAL "yes")
ADD_DEPENDENCIES(RaptorQ_static profile)
SET_TARGET_PROPERTIES(RaptorQ_static PROPERTIES COMPILE_FLAGS ${PROFILE_SET})
ENDIF()
ADD_EXECUTABLE(make_deterministic src/deterministic.cpp)
add_custom_command(
OUTPUT deterministic.run
COMMAND make_deterministic ${LIBRARY_OUTPUT_PATH}/libRaptorQ_static.a
DEPENDS RaptorQ_static
COMMENT "Removing creation date from library..."
VERBATIM
target_link_libraries(test_cpp_raw RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# CPP interface - RAW interface (linked version)
add_executable(test_cpp_raw_linked EXCLUDE_FROM_ALL test/test_cpp_raw.cpp)
target_compile_options(
test_cpp_raw_linked PRIVATE
${CXX_COMPILER_FLAGS}
)
add_custom_target(
Make_static_deterministic ALL
DEPENDS deterministic.run
add_dependencies(test_cpp_raw_linked RaptorQ)
target_link_libraries(test_cpp_raw_linked RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# CLI tool - RAW API interface (header only)
set(CLI_raw_sources src/cli/RaptorQ.cpp external/optionparser-1.4/optionparser.h ${HEADERS} ${HEADERS_ONLY})
if(CLI MATCHES "ON")
add_executable(CLI_raw ${CLI_raw_sources})
add_custom_target(CLI_tools DEPENDS CLI_raw)
else()
add_executable(CLI_raw EXCLUDE_FROM_ALL ${CLI_raw_sources})
add_custom_target(CLI_tools "")
endif()
target_compile_options(
CLI_raw PRIVATE
${CXX_COMPILER_FLAGS}
)
set_target_properties(CLI_raw PROPERTIES OUTPUT_NAME RaptorQ
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
target_link_libraries(CLI_raw ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
#### EXAMPLES
ADD_DEFINITIONS(-DRAPTORQ_DLL)
ADD_DEFINITIONS(-DRAPTORQ_DLL_EXPORTS)
ADD_LIBRARY(RaptorQ SHARED ${SOURCES} ${HEADERS})
# add profiling dependency if requested
IF(PROFILING STREQUAL "yes")
ADD_DEPENDENCIES(RaptorQ profile)
SET_TARGET_PROPERTIES(RaptorQ PROPERTIES COMPILE_FLAGS ${PROFILE_SET})
ENDIF()
SET_TARGET_PROPERTIES (
RaptorQ
PROPERTIES
LINK_FLAGS "-Wl,-z,now,-z,relro"
# CPP interface - RAW interface (header only)
add_executable(example_cpp_raw EXCLUDE_FROM_ALL examples/example_cpp_raw.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options(
example_cpp_raw PRIVATE
${CXX_COMPILER_FLAGS}
)
target_link_libraries(example_cpp_raw ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
add_custom_target(examples DEPENDS test_c test_cpp_rfc test_cpp_rfc_linked test_cpp_raw test_cpp_raw_linked libRaptorQ-test example_cpp_raw)
# add main tests
ADD_EXECUTABLE(libRaptorQ-test EXCLUDE_FROM_ALL test/rfc_test.cpp)
ADD_DEPENDENCIES(libRaptorQ-test RaptorQ)
TARGET_LINK_LIBRARIES(libRaptorQ-test ${CMAKE_THREAD_LIBS_INIT})
# link the library we just built, too
TARGET_LINK_LIBRARIES(libRaptorQ-test ${LIBRARY_OUTPUT_PATH}/libRaptorQ.so)
INSTALL(TARGETS libRaptorQ-test RUNTIME DESTINATION bin/ OPTIONAL)
ADD_CUSTOM_TARGET(tests DEPENDS libRaptorQ-test)
# build examples
# C interface
ADD_EXECUTABLE(test_c EXCLUDE_FROM_ALL test/test_c.c)
ADD_DEPENDENCIES(test_c RaptorQ)
TARGET_LINK_LIBRARIES(test_c ${CMAKE_THREAD_LIBS_INIT})
# link the library we just built, too
TARGET_LINK_LIBRARIES(test_c ${LIBRARY_OUTPUT_PATH}/libRaptorQ.so)
# CPP interface
ADD_EXECUTABLE(test_cpp EXCLUDE_FROM_ALL test/test_cpp.cpp)
ADD_DEPENDENCIES(test_cpp RaptorQ)
TARGET_LINK_LIBRARIES(test_cpp ${CMAKE_THREAD_LIBS_INIT})
# link the library we just built, too
TARGET_LINK_LIBRARIES(test_cpp ${LIBRARY_OUTPUT_PATH}/libRaptorQ.so)
ADD_CUSTOM_TARGET(examples DEPENDS test_c test_cpp)
ADD_CUSTOM_TARGET(everything DEPENDS Make_static_deterministic tests examples docs)
INSTALL(FILES ${HEADERS} DESTINATION include/RaptorQ/)
INSTALL(TARGETS RaptorQ RaptorQ_static
LIBRARY DESTINATION lib COMPONENT libraries
ARCHIVE DESTINATION lib COMPONENT libraries)
add_custom_target(everything DEPENDS make_static_deterministic examples docs CLI_tools)
foreach (hdr ${HEADERS} ${HEADERS_ONLY} ${HEADERS_LINKED})
get_filename_component (full_dir_of_file ${hdr} PATH)
# remove the initial "src/":
string (SUBSTRING ${full_dir_of_file} 4 -1 dir_of_file)
install (FILES ${hdr} DESTINATION "include/${dir_of_file}")
endforeach()
install(TARGETS RaptorQ RaptorQ_Static
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib COMPONENT libraries)
install(TARGETS CLI_raw
RUNTIME DESTINATION bin COMPONENT runtime)
CONTRIBUTING.md
View file @ a394b22a
......@@ -4,6 +4,9 @@
Third-party patches are always welcome, for any purpose.
Although the repository is available from GitHub, please
use the main site for bug reporting/merges/discussions.
Please be aware that libRaptorQ is a dual-licensed software,
and thus it's required that all contributors sign a CLA agreement.
The agreement lets us relicense the code and protect us from patents,
......@@ -63,7 +66,7 @@ developer accepted the CLA and is the author of the commit.
* Always sign your commits with the "-s" option, to add the "Signed-off-by" line in the commit message
* Only the main developers are required to GPG-sign the commits.
* Run the tests to make sure you have not broken something by mistake
* Please keep commits short and to the point. Nobody wants to review 2000+ lines of code that touch everything.
## Documentation
......
README deleted 100644 → 0
View file @ e5537ce0
RaptorQ is a Forward Error Correction algorithm designed to deliver your data
efficiently and without retransmissions for lost packets.
After sending K packets of your data as-is, RaptorQ generates as many repair
symbols as you need. Once the receiver has at least K symbols, be it the source
symbols, repair symbols or any combination of the two, it can reconstruct the
whole input it was meant to receive.
This is called a Fountain code, and RaptorQ is the latest and most efficient
code in this category.
libRaptorQ implements RFC6330, which specifies the RaptorQ algorithm.
NOTE: heavy cluster testing is needed to claim compliance with the
RFC. As we do not have the resources, we do not make such a claim.
libRaptorQ is entirely written in C++11, and uses eigen to handle matrix
manipulation.
Currently it's only been tested under Linux, but should work well under
*BSD and MacOSX, too.
Windows *should* work too, provided you use a complier that understands C++11
(hint: you might have some problems with VisualC++, but that's not a C++
compiler anyway)
##############
#
# Developers
#
##############
See the "CONTRIBUTING.md" file.
##############
#
# The Source Code
#
##############
Although things seems to work, no stable release has been released yet.
This means you can only check this out with git.
to check out the repository:
$ git clone https://github.com/LucaFulchir/libRaptorQ.git
you can also get it from our main server:
$ git clone https://www.fenrirproject.org/Luker/libRaptorQ.git
-- GPG source check --
once you have cloned it, it's always a good thing to check the repository gpg
signatures, so you can import my key with:
$ gpg --keyserver pgp.mit.edu --recv-key D42DDF0A
please check the full fingerprint, it should be like this:
$ gpg2 --fingerprint D42DDF0A
pub rsa3072/D42DDF0A 2015-01-01 [expires: 2016-01-01]
Key fingerprint = AB35 E45F 5CA5 E35B 8B55 818F 0157 D133 D42D DF0A
uid [ unknown] Luca Fulchir (2015 key) <luker@fenrirproject.org>
Now you have the source, and the key, it's enough to check the signature of the
last commit:
$ git log -n 1 --show-signature
The important part is that you get something like this:
gpg: Signature made Fri 27 Mar 2015 20:59:59 CET using RSA key ID D42DDF0A
gpg: Good signature from "Luca Fulchir (2015 key) <luker@fenrirproject.org>"
[unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: AB35 E45F 5CA5 E35B 8B55 818F 0157 D133 D42D DF0A
Author: Luca Fulchir <luker@fenrirproject.org>
And as long as you got the right key, and you find the "gpg: Goog signature",
you can be sure you have the right code.
TDB: The repository is using the maintainer private key for now,
it will start using a dedicated key in the (near) future.
##############
#
# Install
#
##############
Before building libRaptorQ, make sure your system has the eigen3 library
headers, or it won't compile.
Eigen3 is the only dependency of libRaptorQ
The build system uses CMake. So enter the source directory and we'll create a
directory and build everything there:
$ mkdir build
$ cd build
$ cmake -DCMAKE_BUILD_TYPE=Release ../
$ make -j 4
Optional targets are available:
$ make tests examples docs
where:
tests: benchmarks, rfc tests.
examples: C/C++ examples
docs: LATEX documentation.
or simply:
$ make everything
finally, install everything
$ make install
libRaptorQ uses deterministic (reproducible) builds,
so if you compile it twice, or on two different computers
(but with the same compiler), the hash of the resulting
libraries will be the same.
The only configuration that breaks deterministic builds
currently is the clang + profiling one. It probably can not
be fixed.
You can customize the CMake build with tthe following variables:
PROFILING yes/no: Default:yes. Activate or deactivate profiling.
Profiling compiles everything, then runs a test to see
which code paths are more used. Then it recompiles everything
but optimizing for for those code paths.
Only for gcc/clang.
LTO yes/no: Default:yes. Activate or deactivate Link time Optimization
Makes the library smaller and better optimized.
Only for gcc/clang.
STDLIB libc++,libstdc++. Choose the standard library to use.
Note: gcc can only use its own libstdc++
CMAKE_C_COMPILER gcc, clang...
CMAKE_CXX_COMPILER choose between g++ or clang++.
CMAKE_BUILD_TYPE Debug,MinSizeRel,Release,RelWithDebInfo
CMAKE_INSTALL_PREFIX Default: /usr/local
##############
#
# Using libRaptorQ
#
##############
Compile the documentation:
$ make docs
or simply visit:
https://www.fenrirproject.org/projects/libraptorq/documents
for the full up-to-date documentation.
includes the information in this readme, plus a quick description
of the RaptorQ algorithm, and the C and C++ interfaces.
README.md 0 → 100644
View file @ a394b22a
# RaptorQ #
[![build status](https://www.fenrirproject.org/Luker/libRaptorQ/badges/master/build.svg)](https://www.fenrirproject.org/Luker/libRaptorQ/commits/master)
[![Donate](https://img.shields.io/badge/Donate-PayPal-blue.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=79L8P7TXWFQK4)
[![Donate](https://img.shields.io/badge/Donate-Patreon-orange.svg)](https://patreon.com/Luker)
[Homepage](https://www.fenrirproject.org/Luker/libRaptorQ/wikis/home)
old release: **v0.1.10**
> Note: v0.1.10 is not compatible with prefivous versions, which were not RFC compliant!
current release: **v1.0.0-rc2** (use this in new projects)
RaptorQ is a **Forward Error Correction** algorithm designed to deliver your data
efficiently and without retransmissions for lost packets.
After sending K packets of your data as-is, RaptorQ generates as many repair
symbols as you need. Once the receiver has at least K symbols, be it the source
symbols, repair symbols or any combination of the two, it can reconstruct the
whole input it was meant to receive.
This is called a Fountain code, and RaptorQ is the latest and most efficient
code in this category.
**libRaptorQ** implements **RFC6330**, which specifies the RaptorQ algorithm.
Since the RFC is really complex, there is also a simpler (and maybe slightly faster)
RAW API that you can use. The RFC API is not recommended due to the RFC complexity.
libRaptorQ is a header-only library written in *C++11*, and uses eigen to handle matrix
manipulation.
Although header-only, the library can be compiled to create shared and static libraries
with both C and C++98 compatibility.
Currently it's only been tested under Linux, but should work well under
*BSD and MacOSX, too.
Unfortunately Windows is not supported. support for <future> is broken,
so you can not compile it with Visual Studio 2015.
## Features
- RFC6330 API (complex, not recommended)
- RAW API (simpler, recommended)
- multi language:
- header only C++11
- compiled, C
- compiled, C++98
- cached precomputations (configurable)
## Developers ##
See the [CONTRIBUTING](CONTRIBUTING.md) file.
## The Source Code ##
Although things seems to work, no stable release has been released yet.
This means you can only check this out with git.
to check out the repository:
``$ git clone https://github.com/LucaFulchir/libRaptorQ.git``
you can also get it from our main server:
``$ git clone https://www.fenrirproject.org/Luker/libRaptorQ.git``
### GPG source check ###
Once you have cloned it, it's always a good thing to check the repository gpg
signatures, so you can import my key with:
new long term key: **7393 DAD2 55BE B575 1DBD A04A B11C D823 BA27 8C85**
``$ gpg --keyserver pgp.mit.edu --recv-key 7393DAD255BEB5751DBDA04AB11CD823BA278C85``
2016 key:
``$ gpg --keyserver pgp.mit.edu --recv-key F61F6137``
2015 key:
``$ gpg --keyserver pgp.mit.edu --recv-key D42DDF0A``
please check the full fingerprint.
Now you have the source, and the key, it's enough to check the signature of the
last commit:
``$ git log -n 1 --show-signature``
As long as you got the right key, and you find the "gpg: Goog signature",
you can be sure you have the right code.
## Install ##
#### Dependencies ####
libRaptorQ depends from **Eigen3** and **LZ4**
In case your system does not have Eigen3, Eigenv3.2.8 is included
in the source files, so you do not need it.
LZ4 is included as a git submodule, so if you do not have it, run:
```git submodule init```
```git submodule update```
To get the library, which will be statically linked and **NOT INSTALLED**
on your system.
#### Building ####
The build system uses CMake. So enter the source directory and we'll create a
directory and build everything there:
```bash
$ mkdir build
$ cd build
$ cmake -DCMAKE_BUILD_TYPE=Release ../
$ make -j 4
```
Optional targets are available:
``$ make tests examples docs``
where:
* tests: benchmarks, rfc tests.
* examples: C/C++ examples
* docs: LATEX documentation.
or simply:
``$ make -j 4 everything``
finally, install everything
``$ make install``
libRaptorQ uses **deterministic (reproducible) builds**,
so if you compile it twice, or on two different computers
(but with the same compiler), the hash of the resulting
libraries will be the same.
_There are combinations of compiler and LTO/Profiling that
break deterministic builds, so check the cmake warnings._
You can customize the CMake build with the following variables:
```
PROFILING ON/OFF: Default:ON. Activate or deactivate profiling.
Profiling compiles everything, then runs a test to see
which code paths are more used. Then it recompiles everything
but optimizing for for those code paths.
Only for gcc/clang.
LTO ON/OFF: Default:ON. Activate or deactivate Link time Optimization
Makes the library smaller and better optimized.
Only for gcc/clang.
CLANG_STDLIB ON/OFF: Default:OFF. use clang's libc++
Note: only clang can use its standard library
USE_LZ4 ON/OFF: use the lz4 compression for caching precomputations.
Default: ON
CLI ON/OFF Build Command Line Interface tools.
CMAKE_C_COMPILER gcc, clang...
CMAKE_CXX_COMPILER choose between g++ or clang++.
RQ_LINKER gold/ld/bsd Choose your linker. Default:autodetect.
CMAKE_BUILD_TYPE Debug,MinSizeRel,Release,RelWithDebInfo
CMAKE_INSTALL_PREFIX Default: /usr/local
```
## Using libRaptorQ ##
For the C++11, header-only version, you can include:
- "RaptorQ/RaptorQ_v1_hdr.hpp"
- "RaptorQ/RFC6330_v1_hdr.hpp"
For the linked, C+98/C++11 API:
- "RaptorQ/RaptorQ_v1.hpp"
- "RaptorQ/RFC6330_v1.hpp"
For the linked, C API:
- "RaptorQ/RaptorQ.h"
- "RaptorQ/RFC6330.h"
The C++ api is completely in sync between the linked and header-only version,
so you can switch between the two just by changing the included header file.
You can compile a PDF of the documentation by doing:
``$ make docs``
Or you can simply visit the [wiki](https://www.fenrirproject.org/Luker/libRaptorQ/wikis/libRaptorQ.pdf)
for the full up-to-date documentation.
cmake/Findstd_clang.cmake → cmake/FindRQ_LZ4.cmake
View file @ a394b22a
......@@ -18,33 +18,55 @@
# along with libRaptorQ. If not, see <http://www.gnu.org/licenses/>.
#
FIND_PATH(CLANG_STD_INCLUDE_DIR
NAMES unordered_map
PATH_SUFFIXES include/c++ include/c++/v1
PATHS "${PROJECT_SOURCE_DIR}/../libc++/"
${CLANG_STD_ROOT}
$ENV{CLANG_STD_ROOT}
/usr/local/
/usr/
FIND_PATH(RQ_LZ4_INCLUDE_DIR
NAMES lz4.h
PATH_SUFFIXES include/ include/lz4/ lz4/
PATHS
${LZ4_ROOT}
$ENV{LZ4_ROOT}
/usr/
/usr/local/
${CMAKE_CURRENT_SOURCE_DIR}/external/lz4/lib
)
message("Found clang stdandard library at: ${CLANG_STD_INCLUDE_DIR}.")
FIND_LIBRARY(RQ_LZ4_LIB
NAMES lz4
PATH_SUFFIXES lib/
PATHS
${LZ4_ROOT}
$ENV{LZ4_ROOT}
/usr/
/usr/local/
)
IF(CLANG_STD_INCLUDE_DIR)
SET(CLANG_STD_FOUND TRUE)
ELSE(CLANG_STD_INCLUDE_DIR)
SET(CLANG_STD_FOUND FALSE)
ENDIF(CLANG_STD_INCLUDE_DIR)
IF(RQ_LZ4_INCLUDE_DIR)
SET(RQ_LZ4_FOUND TRUE)
ELSE(RQ_LZ4_INCLUDE_DIR)
SET(RQ_LZ4_FOUND FALSE)
ENDIF(RQ_LZ4_INCLUDE_DIR)
IF(CLANG_STD_FOUND)
MESSAGE(STATUS "Found clang standard library in ${CLANG_STD_INCLUDE_DIR}")
ELSE(CLANG_STD_FOUND)
IF(CLANG_STD_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find the clang standard library")
ENDIF(CLANG_STD_FIND_REQUIRED)
ENDIF(CLANG_STD_FOUND)
IF(RQ_LZ4_FOUND)
IF(RQ_LZ4_USE_OWN)
# force our own lz4 libary
SET(RQ_LZ4_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/lz4/lib)
ENDIF()
MESSAGE(STATUS "Found lz4 in ${RQ_LZ4_INCLUDE_DIR}")
# we need both headers and library, or we use our own
IF(RQ_LZ4_INCLUDE_DIR MATCHES "${CMAKE_CURRENT_SOURCE_DIR}/external/lz4/lib" OR RQ_LZ4_LIB MATCHES "RQ_LZ4_LIB-NOTFOUND")
MESSAGE(WARNING "We will build our own lz4 library and statically link it.")
SET(RQ_BUILD_LZ4 TRUE)
ENDIF()
ELSE(RQ_LZ4_FOUND)
IF(RQ_LZ4_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find \"lz4\" library."
" Please install the \"lz4\" library"
" or at least run:\n"
" cd ${CMAKE_CURRENT_SOURCE_DIR}\n"
" git submodule init\n"
" git submodule update\n")
ENDIF(RQ_LZ4_FIND_REQUIRED)
ENDIF(RQ_LZ4_FOUND)
MARK_AS_ADVANCED(
CLANG_STD_INCLUDE_DIR
RQ_LZ4_INCLUDE_DIR
)
cmake/Findeigen.cmake
View file @ a394b22a
......@@ -19,31 +19,31 @@
#
FIND_PATH(EIGEN3_INCLUDE_DIR
NAMES Eigen
PATH_SUFFIXES include/ include/eigen3/ eigen3/
PATHS "${PROJECT_SOURCE_DIR}"
${EIGEN3_ROOT}
$ENV{EIGEN3_ROOT}
/usr/
NAMES Eigen
PATH_SUFFIXES include/ include/eigen3/ eigen3/
PATHS
${EIGEN3_ROOT}
$ENV{EIGEN3_ROOT}
/usr/
/usr/local/
${PROJECT_SOURCE_DIR}/external/eigen3
)
message("Found eigen3 at: ${EIGEN3_INCLUDE_DIR}.")
IF(EIGEN3_INCLUDE_DIR)
SET(EIGEN3_FOUND TRUE)
SET(EIGEN3_FOUND TRUE)
ELSE(EIGEN3_INCLUDE_DIR)
SET(EIGEN3_FOUND FALSE)
SET(EIGEN3_FOUND FALSE)
ENDIF(EIGEN3_INCLUDE_DIR)
IF(EIGEN3_FOUND)
MESSAGE(STATUS "Found eigen3 in ${EIGEN3_INCLUDE_DIR}")
MESSAGE(STATUS "Found eigen3 in ${EIGEN3_INCLUDE_DIR}")
ELSE(EIGEN3_FOUND)
IF(EIGEN3_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find \"eigen3\" library")
ENDIF(EIGEN3_FIND_REQUIRED)
IF(EIGEN3_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find \"eigen3\" library")
ENDIF(EIGEN3_FIND_REQUIRED)
ENDIF(EIGEN3_FOUND)
MARK_AS_ADVANCED(
EIGEN3_INCLUDE_DIR
EIGEN3_INCLUDE_DIR
)
cmake/Findgit.cmake
View file @ a394b22a
......@@ -18,29 +18,29 @@
# along with libRaptorQ. If not, see <http://www.gnu.org/licenses/>.
#
FIND_PROGRAM(GIT_EXECUTABLE git
PATHS
${PATH}
"C:/Program Files/Git/bin"
"C:/Program Files (x86)/Git/bin"
DOC "git command line client"
PATHS
${PATH}
"C:/Program Files/Git/bin"
"C:/Program Files (x86)/Git/bin"
DOC "git command line client"
)
IF(GIT_EXECUTABLE)
SET(GIT_FOUND TRUE)
SET(GIT_FOUND TRUE)
ELSE(GIT_EXECUTABLE)
SET(GIT_FOUND FALSE)
SET(GIT_FOUND FALSE)
ENDIF(GIT_EXECUTABLE)
IF(GIT_FOUND)
MESSAGE(STATUS "Found git in ${GIT_EXECUTABLE}")
MESSAGE(STATUS "Found git in ${GIT_EXECUTABLE}")
ELSE(GIT_FOUND)
IF(GIT_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find \"git\" command")
ENDIF(GIT_REQUIRED)
IF(GIT_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find \"git\" command")
ENDIF(GIT_REQUIRED)
ENDIF(GIT_FOUND)
MARK_AS_ADVANCED(
GIT_EXECUTABLE
GIT_EXECUTABLE
)
......
cmake/Findstd_gcc.cmake deleted 100644 → 0
View file @ e5537ce0
#
# Copyright (c) 2015, Luca Fulchir<luca@fulchir.it>, All rights reserved.
#
# This file is part of libRaptorQ.
#
# libRaptorQ is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 3
# of the License, or (at your option) any later version.
#
# libRaptorQ is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# and a copy of the GNU Lesser General Public License
# along with libRaptorQ. If not, see <http://www.gnu.org/licenses/>.
#
file(GLOB sub-dir /usr/include/c++/[0-9].[0-9]*.[0-9]*)
foreach(dir ${sub-dir})
if(IS_DIRECTORY ${dir})
set(GCC_PATH_SUFFIXES ${GCC_PATH_SUFFIXES};${dir})
endif()
endforeach()
FIND_PATH(GCC_STD_INCLUDE_DIR
NAMES unordered_map
PATH_SUFFIXES / ${GCC_PATH_SUFFIXES}
PATHS "${PROJECT_SOURCE_DIR}/../stdlib/"
${GCC_STD_ROOT}
$ENV{GCC_STD_ROOT}
/
)
message("Found gcc stdandard library at: ${GCC_STD_INCLUDE_DIR}.")
IF(GCC_STD_INCLUDE_DIR)
SET(GCC_STD_FOUND TRUE)
ELSE(GCC_STD_INCLUDE_DIR)
SET(GCC_STD_FOUND FALSE)
ENDIF(GCC_STD_INCLUDE_DIR)
IF(GCC_STD_FOUND)
MESSAGE(STATUS "Found gcc standard library in ${GCC_STD_INCLUDE_DIR}")
ELSE(GCC_STD_FOUND)
IF(GCC_STD_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find the gcc standard library")
ENDIF(GCC_STD_FIND_REQUIRED)
ENDIF(GCC_STD_FOUND)
MARK_AS_ADVANCED(
GCC_STD_INCLUDE_DIR
)
cmake/compiler_flags.cmake 0 → 100644
View file @ a394b22a
#
# Copyright (c) 2015-2016, Luca Fulchir<luca@fulchir.it>, All rights reserved.
#
# This file is part of "libRaptorQ".
#
# libRaptorQ is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 3
# of the License, or (at your option) any later version.
#
# libRaptorQ is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# and a copy of the GNU Lesser General Public License
# along with libRaptorQ. If not, see <http://www.gnu.org/licenses/>.
#
include(CheckCCompilerFlag)
include(CheckCXXCompilerFlag)
#string of multiple flags to list
string(REPLACE " " ";" RQ_C_FLAGS "${CMAKE_C_FLAGS}")
string(REPLACE " " ";" RQ_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
##############################################
#### check for UBSAN sanitizing support ######
##############################################
if(CMAKE_BUILD_TYPE MATCHES "Debug")
check_c_compiler_flag("-fsanitize=undefined" RQ_FLAG_C_UBSAN)
check_cxx_compiler_flag("-fsanitize=undefined" RQ_FLAG_CXX_UBSAN)
if (RQ_FLAG_C_UBSAN AND RQ_FLAG_CXX_UBSAN)
set(RQ_ENABLE_UBSAN TRUE)
message(STATUS "UBSAN sanitizing support enabled")
else()
message(STATUS "UBSAN sanitizing support disabled")
endif()
add_definitions(-DRQ_DEBUG=true)
else()
add_definitions(-DRQ_DEBUG=false)
endif()
###################
## Compiler options
###################
#gnu options
set(RQ_GNU_C_OPTIONS ${RQ_DETERMINISTIC} -std=c11
-Wno-unknown-pragmas -Wall -Wextra -pedantic -Wno-padded
-fstack-protector-all -fstrict-aliasing -fwrapv -fvisibility=hidden
-Wduplicated-cond -Wduplicated-branches -Wlogical-op -Wrestrict
-Wnull-dereference -Wdouble-promotion -Wshadow -Wformat=2)
set(RQ_GNU_CXX_OPTIONS ${RQ_DETERMINISTIC} -std=c++11
-fno-rtti -fno-exceptions -Wno-unknown-pragmas -Wall -Wextra
-pedantic -Wno-padded -Wno-unknown-pragmas -fstack-protector-all
-fstrict-aliasing -fwrapv -fvisibility=hidden -fvisibility-inlines-hidden
-Wduplicated-cond -Wduplicated-branches -Wlogical-op -Wrestrict
-Wnull-dereference -Wuseless-cast -Wdouble-promotion -Wshadow -Wformat=2
-Wno-duplicated-branches)
# GCC internal compiler errors with:
# -fsanitize=undefined
# -fsanitize=null
# -fsanitize=signed-integer-overflow
set(RQ_GNU_C_DEBUG -O0 -g -Wno-aggressive-loop-optimizations -ftrapv )
set(RQ_GNU_CXX_DEBUG -O0 -g -Wno-aggressive-loop-optimizations -ftrapv )
set(RQ_GNU_C_DEBUG_SANITIZE -fsanitize=shift -fsanitize=integer-divide-by-zero -fsanitize=vla-bound -fsanitize=return -fisolate-erroneous-paths-dereference -fisolate-erroneous-paths-attribute )
set(RQ_GNU_CXX_DEBUG_SANITIZE -fsanitize=shift -fsanitize=integer-divide-by-zero -fsanitize=vla-bound -fsanitize=return -fisolate-erroneous-paths-dereference -fisolate-erroneous-paths-attribute )
set(RQ_GNU_C_MINSIZEREL -Os )
set(RQ_GNU_CXX_MINSIZEREL -Os )
set(RQ_GNU_C_RELEASE -Ofast -DNDEBUG -fwrapv -ftree-loop-distribution -funroll-loops )
set(RQ_GNU_CXX_RELEASE -Ofast -DNDEBUG -fwrapv -ftree-loop-distribution -funroll-loops )
set(RQ_GNU_C_RELWITHDEBINFO -g -Ofast -fwrapv -ftree-loop-distribution -funroll-loops )
set(RQ_GNU_CXX_RELWITHDEBINFO -g -Ofast -fwrapv -ftree-loop-distribution -funroll-loops )
# clang options
set(RQ_CLANG_C_OPTIONS ${RQ_DETERMINISTIC} -std=c11
-fno-math-errno -Wall -Wextra -pedantic -Weverything -Wno-padded
-fstack-protector-all -fstrict-aliasing -Wformat -Wformat-security
-Wno-disabled-macro-expansion -fvisibility=hidden -fvisibility-inlines-hidden
-Wdouble-promotion -Wshadow -Wformat=2 -Wnull-dereference)
set(RQ_CLANG_CXX_OPTIONS ${RQ_STDLIB_FLAG} ${RQ_DETERMINISTIC}
-std=c++11 -fno-rtti -fno-exceptions -fno-math-errno
-Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat
-Wno-padded -Wno-unknown-pragmas -fstack-protector-all -fstrict-aliasing
-Wformat -Wformat-security -fvisibility=hidden -fvisibility-inlines-hidden
-Wno-documentation -Wno-documentation-unknown-command
-Wdouble-promotion -Wshadow -Wformat=2 -Wnull-dereference)
set(RQ_CLANG_C_DEBUG -O0 -g )
set(RQ_CLANG_CXX_DEBUG -O0 -g )
set(RQ_CLANG_C_DEBUG_SANITIZE -fsanitize=shift -fsanitize=integer-divide-by-zero -fsanitize=vla-bound -fsanitize=return )
set(RQ_CLANG_CXX_DEBUG_SANITIZE -fsanitize=shift -fsanitize=integer-divide-by-zero -fsanitize=vla-bound -fsanitize=return )
set(RQ_CLANG_C_MINSIZEREL -Os )
set(RQ_CLANG_CXX_MINSIZEREL -Os )
set(RQ_CLANG_C_RELEASE -Ofast -DNDEBUG -fwrapv )
set(RQ_CLANG_CXX_RELEASE -Ofast -DNDEBUG -fwrapv )
set(RQ_CLANG_C_RELWITHDEBINFO -g -Ofast -fwrapv )
set(RQ_CLANG_CXX_RELWITHDEBINFO -g -Ofast -fwrapv )
# msvc flags. todo?
set(RQ_MSVC_C_OPTIONS /wd4068)
set(RQ_MSVC_CXX_OPTIONS /wd4068)
###################
# combine the above
###################
# note: generator expansions do not work on "set(..)"
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_CXX_FLAGS} ${RQ_CLANG_CXX_OPTIONS})
if(CMAKE_BUILD_TYPE MATCHES "Debug")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_DEBUG})
if (RQ_ENABLE_UBSAN)
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_DEBUG_SANITIZE})
set(RQ_UBSAN ${RQ_CLANG_CXX_DEBUG_SANITIZE})
endif()
elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_MINSIZEREL})
elseif(CMAKE_BUILD_TYPE MATCHES "Release")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_RELEASE})
elseif(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_RELWITHDEBINFO})
endif()
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_CXX_FLAGS} ${RQ_GNU_CXX_OPTIONS})
if(CMAKE_BUILD_TYPE MATCHES "Debug")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_DEBUG})
if (RQ_ENABLE_UBSAN)
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_DEBUG_SANITIZE})
set(RQ_UBSAN ${RQ_GNU_CXX_DEBUG_SANITIZE})
endif()
elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_MINSIZEREL})
elseif(CMAKE_BUILD_TYPE MATCHES "Release")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_RELEASE})
elseif(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_RELWITHDEBINFO})
endif()
elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_CXX_FLAGS} ${RQ_MSVC_CXX_OPTIONS})
endif()
if(CMAKE_C_COMPILER_ID MATCHES "Clang")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_C_FLAGS} ${RQ_CLANG_C_OPTIONS})
if(CMAKE_BUILD_TYPE MATCHES "Debug")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_CLANG_C_DEBUG})
if (RQ_ENABLE_UBSAN)
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_CLANG_C_DEBUG_SANITIZE})
endif()
elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_CLANG_C_MINSIZEREL})
elseif(CMAKE_BUILD_TYPE MATCHES "Release")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_CLANG_C_RELEASE})
elseif(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_CLANG_C_RELWITHDEBINFO})
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES "GNU")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_C_FLAGS} ${RQ_GNU_C_OPTIONS})
if(CMAKE_BUILD_TYPE MATCHES "Debug")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_GNU_C_DEBUG})
if (RQ_ENABLE_UBSAN)
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_GNU_C_DEBUG_SANITIZE})
endif()
elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_GNU_C_MINSIZEREL})
elseif(CMAKE_BUILD_TYPE MATCHES "Release")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_GNU_C_RELEASE})
elseif(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_LIST_C_COMPILER_FLAGS} ${RQ_GNU_C_RELWITHDEBINFO})
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES "MSVC")
set(RQ_LIST_C_COMPILER_FLAGS ${RQ_C_FLAGS} ${RQ_MSVC_C_OPTIONS})
endif()
########################################
## Test all the previously enabled flags
########################################
set(RQ_C_COMPILER_FLAGS "")
set(RQ_CXX_COMPILER_FLAGS "")
# check each C flag
foreach(flag ${RQ_LIST_C_COMPILER_FLAGS})
string(REPLACE "-" "_" RQ_FLAG2 ${flag})
string(REPLACE "+" "_" RQ_FLAG1 ${RQ_FLAG2})
string(REPLACE "=" "_" RQ_FLAG ${RQ_FLAG1})
check_c_compiler_flag(${flag} RQ_FLAG_C_${RQ_FLAG})
if(RQ_FLAG_C_${RQ_FLAG})
set(C_COMPILER_FLAGS ${C_COMPILER_FLAGS} ${flag})
else()
message(WARNING "flag \"${flag}\" can't be used in your C compiler")
endif()
endforeach()
# check each c++ flag
foreach(flag ${RQ_LIST_CXX_COMPILER_FLAGS})
string(REPLACE "-" "_" RQ_FLAG2 ${flag})
string(REPLACE "+" "_" RQ_FLAG1 ${RQ_FLAG2})
string(REPLACE "=" "_" RQ_FLAG ${RQ_FLAG1})
check_cxx_compiler_flag(${flag} RQ_FLAG_CXX_${RQ_FLAG})
if(RQ_FLAG_CXX_${RQ_FLAG})
set(CXX_COMPILER_FLAGS ${CXX_COMPILER_FLAGS} ${flag})
else()
message(WARNING "flag \"${flag}\" can't be used in your CXX compiler")
endif()
endforeach()
cmake/guess_architecture.cmake
View file @ a394b22a
#
# Copyright (c) 2015, Luca Fulchir<luca@fulchir.it>, All rights reserved.
# Copyright (c) 2015-2018, Luca Fulchir<luca@fulchir.it>, All rights reserved.
#
# This file is part of libRaptorQ.
#
......@@ -57,5 +57,3 @@ ELSE()
ENDIF()
ENDIF()
cppcheck.suppress
View file @ a394b22a
// Performance:
// Octet is a sinble byte. value is faster than reference.
// Octet is a single byte. value is faster than reference.
passedByValue:src/Parameters.hpp:73
passedByValue:src/Parameters.hpp:78
passedByValue:src/Parameters.hpp:83
......@@ -13,6 +13,11 @@ passedByValue:src/Parameters.hpp:132
passedByValue:src/Interleaver.hpp:170
passedByValue:src/Interleaver.hpp:263
passedByValue:src/RaptorQ.hpp:143
passedByValue:src/RaptorQ.hpp:161
passedByValue:src/De_Interleaver.hpp:35
// variable is read by different threads
unreadVariable:test/rfc_test.cpp:569
functionStatic:src/Parameters.hpp:116
noExplicitConstructor:src/Parameters.hpp:69
......@@ -21,5 +26,11 @@ noExplicitConstructor:src/Parameters.hpp:69
// which we do, valgrind never gives us "uninitalized value"
uninitMemberVar:src/Parameters.hpp:68
unusedFunction:src/cRaptorQ.cpp:270
unusedFunction:src/cRaptorQ.cpp:596
unusedFunction:src/cRaptorQ.cpp:452
unusedFunction:src/cRaptorQ.cpp:752
unusedFunction:src/cRaptorQ.cpp:385
missingIncludeSystem
cppcheck_howto
View file @ a394b22a
cppcheck --std=c++11 --enable=all --inconclusive --xml-version=2 --suppressions cppcheck.suppress src/ test/
cppcheck --std=c++11 --enable=all --inconclusive --xml-version=2 --suppressions-list=cppcheck.suppress src/ test/
doc/CMakeLists.txt
View file @ a394b22a
......@@ -23,17 +23,24 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
INCLUDE(../cmake/UseLATEX.cmake)
IF (LATEX_COMPILER AND PDFLATEX_COMPILER AND MAKEINDEX_COMPILER)
ADD_LATEX_DOCUMENT(libRaptorQ.tex
INPUTS GNU_FDL.tex
USE_INDEX
MANGLE_TARGET_NAMES
NO_DEFAULT
)
ADD_LATEX_DOCUMENT(libRaptorQ-RFC6330.tex
INPUTS GNU_FDL.tex
USE_INDEX
MANGLE_TARGET_NAMES
NO_DEFAULT
)
ADD_LATEX_DOCUMENT(libRaptorQ-RaptorQ.tex
INPUTS GNU_FDL.tex
USE_INDEX
MANGLE_TARGET_NAMES
NO_DEFAULT
)
ADD_CUSTOM_TARGET(docs DEPENDS libRaptorQ_pdf)
ADD_CUSTOM_TARGET(docs DEPENDS libRaptorQ-RFC6330_pdf libRaptorQ-RaptorQ_pdf)
ELSE()
ADD_CUSTOM_TARGET(docs)
ENDIF()
INSTALL(FILES ${CMAKE_BINARY_DIR}/doc/libRaptorQ.pdf DESTINATION share/doc/ OPTIONAL)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/doc/libRaptorQ-RFC6330.pdf DESTINATION share/doc/ OPTIONAL)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/doc/libRaptorQ-RaptorQ.pdf DESTINATION share/doc/ OPTIONAL)
doc/libRaptorQ-RFC6330.tex 0 → 100644
View file @ a394b22a
% Copyright (C) 2015-2018, Luca Fulchir<luca@fulchir.it>
% Permission is granted to copy, distribute and/or modify this document
% under the terms of the GNU Free Documentation License, Version 1.3
% or any later version published by the Free Software Foundation;
% with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
% A copy of the license is included in the section entitled "GNU
% Free Documentation License".
\documentclass[11pt,a4paper]{refart}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{hyperref}
\usepackage{makeidx}
\usepackage{verbatimbox}
\usepackage{listings}
\usepackage{color}
\definecolor{mygreen}{rgb}{0,0.6,0}
\definecolor{mygray}{rgb}{0.5,0.5,0.5}
\definecolor{mymauve}{rgb}{0.58,0,0.82}
\lstset{ %
backgroundcolor=\color{white}, % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}
basicstyle=\footnotesize, % the size of the fonts that are used for the code
breakatwhitespace=false, % sets if automatic breaks should only happen at whitespace
breaklines=true, % sets automatic line breaking
captionpos=b, % sets the caption-position to bottom
commentstyle=\color{mygreen}, % comment style
deletekeywords={...}, % if you want to delete keywords from the given language
escapeinside={\%*}{*)}, % if you want to add LaTeX within your code
extendedchars=true, % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
frame=single, % adds a frame around the code
keepspaces=true, % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
keywordstyle=\color{blue}, % keyword style
language=C, % the language of the code
%otherkeywords={*,...}, % if you want to add more keywords to the set
%numbers=left, % where to put the line-numbers; possible values are (none, left, right)
%numbersep=5pt, % how far the line-numbers are from the code
%numberstyle=\tiny\color{mygray}, % the style that is used for the line-numbers
rulecolor=\color{black}, % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
showspaces=false, % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
showstringspaces=false, % underline spaces within strings only
showtabs=false, % show tabs within strings adding particular underscores
stepnumber=2, % the step between two line-numbers. If it's 1, each line will be numbered
stringstyle=\color{mymauve}, % string literal style
tabsize=4, % sets default tabsize to 2 spaces
title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title
}
\title{Using libRaptorQ library: RFC6330 interface}
\date{\today}
\makeindex
\begin{document}
\maketitle
\begin{abstract}
\textbf{libRaptorQ} is a C++11 implementation of the RaptorQ Forward Error Correction. It includes two interfaces, one derived from the \href{https://tools.ietf.org/html/rfc6330}{RFC6330} standard, and one RAW API for a more flexible and easy usage.
This document is only about the RFC6330 interface.
The implementation was started as a university laboratory project, and will be later used and included in \href{https://www.fenrirproject.org}{Fenrir}, the
maintainer's master thesis project.
\end{abstract}
\vfill\hfill
\begin{verbbox}[\tiny]
Copyright (C) 2015-2018, Luca Fulchir<luker@fenrirproject.org>
Permission is granted to copy, distribute and/or modify this document
under the terms of the GNU Free Documentation License, Version 1.3
or any later version published by the Free Software Foundation;
with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
A copy of the license is included in the section entitled "GNU
Free Documentation License".
\end{verbbox}
\theverbbox
\newpage
\tableofcontents
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Contacts}\index{Contacts}
The main development and dicussions on the project, along with bug reporting, happens on
\href{https://fenrirproject.org/Luker/libRaptorQ}{the main website}.
\marginlabel{Mailing Lists} Mailing lists are available at \href{https://fenrirproject.org/lists}{https://fenrirproject.org/lists}\\
The two mailing lists are for development and announcements, but due to the low traffic of the development mailing list, it
can also be used by users for questions on the project.
\marginlabel{IRC} Since there are not many developers for now, the main irc channel is \textbf{\#fenrirproject} on freenode
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Build \& install}
\subsection{Get the source code}
This document follows the 1.0 stable release.
You can get the tarballs from the main website here:\\
\href{https://fenrirproject.org/Luker/libRaptorQ/tags}{https://fenrirproject.org/Luker/libRaptorQ/tags}
Or you can check out the repository (warning: development in progress):
\begin{lstlisting}[language=bash,breaklines=true]
$ git clone --recurse-submodules \
https://fenrirproject.org/Luker/libRaptorQ.git
\end{lstlisting}
You can also get it from github:
\begin{lstlisting}[language=bash,breaklines=true]
$ git clone --recurse-submodules \
https://github.com/LucaFulchir/libRaptorQ.git
\end{lstlisting}
\index{GPG}\marginlabel{GPG verification:}
Once you have cloned it, it's always a good thing to check the repository gpg
signatures, so you can import my key with:
\begin{lstlisting}[language=bash,breaklines=true]
$ gpg --keyserver pgp.mit.edu --recv-key \
7393DAD255BEB5751DBDA04AB11CD823BA278C85
\end{lstlisting}
Now you have the source, and the key, it's enough to check the signature of the
last commit:
\begin{verbatim}
$ git log -n 1 --show-signature
\end{verbatim}
The important part is that you get something like this:
\begin{verbbox}[\footnotesize]
gpg: Signature made Mon 11 Dec 2017 21:55:28 CET
gpg: using RSA key 8F7474044095B405D0F042F0A2CCA134BC7C8572
gpg: Good signature from "Luca Fulchir <luker@fenrirproject.org>" [unknown]
gpg: aka "Luca Fulchir <luca@fulchir.it>" [unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: 7393 DAD2 55BE B575 1DBD A04A B11C D823 BA27 8C85
\end{verbbox}
\theverbbox
And as long as you got the right key, and you find the \textbf{"gpg: Good signature"} string,
you can be sure you have the right code.
The main development happens in the $master$ branch, while we keep one branch for each stable version.
\subsection{Dependencies}\index{dependencies}
libRaptorQ has 4 dependencies:
\begin{description}
\item[\textbf{Eigen3}] This is used for matrix manipulation, which is a big part of RaptorQ.
\item[\textbf{lz4}] Used to compress cached precomputations.
\item[\textbf{git}] This is used not only to get the source, but also by the build system. We get the last git commit id and feed it to clang or gcc as seed for their internal random number generator. This makes it possible to have reproducible builds.
\item[\textbf{optionparser}] Library used to parse the command line easily in C++, without exceptions
\end{description}
All dependencies are included in the sources, so you do not need to download and compile them.\\
Eigen3 is only needed at build time, so there is no need to download it again. we use the 3.3.4 version.\\
LZ4 is included as a git submodule. if you do not have it, run:
\begin{verbatim}
$ git submodule init
$ git submodule update
\end{verbatim}
and LZ4 will be built statically in the library (it will not be installed on the system)
\subsection{Build \& Install}
libRaptorQ uses the cMake build system, so things are fairly standard:
\begin{verbatim}
$ cd libRaptorQ.git
$ mkdir build
$ cd build
$ cmake ../
$ make -j 4
\end{verbatim}
By default, the libRaptorQ project tries to have deterministic builds. This means that if you compile things twice, or with two different computers, the
hash of the resulting library will be the same, provided that the same compiler (clang, gcc 4.8, gcc 4.9 etc) was used. Currently the only exception is the clang
compiler with the \textit{PROFILING} option enabled, and this will not likely be solved.
\index{cMake}
There are lots of options, you can use in cmake. As always, you can change them by adding ``\textbf{-Dcmake\_option=cmake\_value}'' when calling cmake.
You can always use the cmake-gui or ccmake commands to have the list of possible options.
The ones we recognize are:
\begin{description}
\item[LTO] ON/OFF. Default:\textbf{ON}. Enables \textit{Link Time Optimizatios} for clang or gcc. Makes libraries smaller and better optimized.
\item[PROFILING] ON/OFF. Default:\textbf{ON}.
Profiling compiles everything once, then runs a test to see which code paths are used more, and then
recompiles everything again, but making sure that the binary is optimized for those paths. Works with clang and gcc. Provides a slight speedup.
\item[CMAKE\_C\_COMPILER] gcc and clang are directly supported. other should work, too. This is only used if you want to build the C example.
\item[CMAKE\_CXX\_COMPILER] g++, clang++ are directly supported. other should work, too.
\item[CLANG\_STDLIB] ON/OFF. Default:\textbf{OFF}.use clang's libc++ standard library. Only available with clang.
\item[CMAKE\_BUILD\_TYPE] Type of build. you can choose between ``Debug'', ``Release'', ``MinSizeRel'' or ``RelWithDebInfo''
\item[CMAKE\_INSTALL\_PREFIX] Default to \textit{/usr/local}. Change it to fit your distribution guidelines.
\item[RQ\_ENDIANNESS] \texttt{Auto,BigEndian,LittleEndian} Force the build to use one endianness ot the other
\item[RQ\_LINKER] \texttt{Auto,gold,ld,bsd} Force the build to use one linker or the other
\item[USE\_LZ4] ON/OFF. Default:\textbf{ON}. compile with support for lz4
\end{description}
Then you can build everything by running:
\begin{verbatim}
$ make -j 4
\end{verbatim}
Of course, you can configure the number of parallel jobs (the \textit{-j} parameter) to be what you need.
\marginlabel{Optional make targets:}
The following optional targets are also supported:
\begin{verbatim}
$ make docs tests examples
$ make everything
\end{verbatim}
The ``docs'' target builds this document, but you need latex with the refman style.
The tests are only useful to check perfromance of rfc compliance right now. ``examples'' compiles the C and C++ examples, which will not be installed.
\index{Install}\marginlabel{\textbf{Install:}}
The installation process is very simple:
\begin{verbatim}
$ make install DESTDIR=...
\end{verbatim}
You can change the \textit{DESTDIR} parameter to fit your distribution guidelines.
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Working with RaptorQ}
\subsection{Theory (you really need this)}\index{Theory}
To be able to work with liRaptorQ, you must first understand how the RaptorQ algorithms works. We won't go into the details, but just what you need
to be able to tune the algorithm to your needs.
\marginlabel{Fountain codes:}
Fountain codes are a special \textit{Forward-Error-Correcting} code class, which characteristic is simple: if you want to send $K$ packets, you actually
send $K+X$ packets, and the receiver only needs to get any $K$ packets to be able to reconstruct your data. The number $X$ of overhead packets can be as big
as you need (theoretically infinite), so you can tune it to be slightly higher than the expected packet loss.
\marginlabel{Systematic codes:} RaptorQ is also a systematic code. This means that those first $K$ packets are the input \textit{as-is} (\textbf{source symbols}),
and the $X$ packets (\textbf{repair symbols}) have the information needed to recover any of the lost source packets. This has the big advantage of
avoiding any kind of time and memory consuming decoding if there is no packet loss during the transmission.
\marginlabel{Complexity:} The RaptorQ algorithm is often presented as having a linear time encoder and decoder. This is both false and misleading.
Generating the source or repair symbols from the intermediate symbols has linear complexity. Generating the intermediate symbols has cubic complexity on the
number of symbols. Which is a completely different thing.
It is still very quick. On a core i7, 2.4Ghz, you need to wait \textit{0.4ms} for $10$ symbols, \textit{280ms} for 1.000 symbols, but it can take an hour for $27.000$ symbols.
RaptorQ handles up to $56.403$ symbols.
\marginlabel{\textbf{Caching}} libRaptorQ can work with big matrices that take a lot of time to compute. For this reason the matrices can be saved once they have
been computed the first time. libRaptorQ uses a \textbf{local} cache. The matrix can be compressed with LZ4.
\subsection{RFC6330: Blocks \& Symbols}
To understand how to properly use and tune libRaptorQ, you first need to understand how RaptorQ and the \textbf{RFC6330} handles its inputs, outputs, and what the time and memory
constraints are.
\index{Sequencing}\marginlabel{Input sequencing:}
The RC6330 needs to have the whole input you want to send before it can start working.\\
This means that it might be a little more difficult to use in live-streaming contexts, or where you need real-time data, but for that you probably want to use the single-block API in the \textbf{RaptorQ} C++ namespace. There is a second document tracking that interface, this document tracks only the RFC6330 interface.
Once you have the whole input, RFC6330 divides it into \textbf{blocks}. Each block \textit{is encoded and decoded independently} and will be divided into \textbf{symbols}. Each symbol \textit{should}
be transmitted separately in its own packet (if you are on the network).
\index{Sequencing!Sizes}\marginlabel{Sizes:}Each input can have up to \textit{256 blocks}, each block can have up to \textit{$56.403$ symbols}, and each
symbol can have up to \textit{$2^{16}-1$ bytes}long. This gives a maximum files size of almost $881$ GB (946270874880 bytes to be exact)
\index{Interleaving}\marginlabel{Interleaving:}
An other feature of RFC6330 is to automatically provide some interleaving of the input data before transmitting it. This means that one symbol will not
represent one sequential chunk of your input data, but more likely it's the concatenation of different \textbf{sub-symbols}. The size of the subsymbol must thus
be a fraction of the symbol size. \textit{This feature is not used if you set the size of the subsymbol to the size of symbol}.
\marginlabel{Memory and Time:}
Memory and time requirements are to be considered, though, as RFC6330 needs to run a cubic algorithm on matrix of size $K*K$, where $K$ is the number of
symbols in each block.\\
The algorithm needs to keep in memory two of these matrices, although most of the work is done on only one.\\
This is actually a lot. More benchmarks and optimizations will come later, for now remember that with 10 symbols it takes something like 0.4ms on a core i7 2.4GHZ, 280ms with 1000 symbols, and up to an hour with 27.000 symbols.
\subsection{C++ interface}
\index{Interface!C++}
To use the C++ interface you only need to include the \textbf{RaptorQ/RFC6330\_v1.hpp} header, and link \textbf{libRaptorQ} and your threading library (usually
\textit{libpthread}).
You can also use the library as a header-only include, by including \textbf{RaptorQ/RFC6330\_v1\_hdr.hpp}. All your code should keep working even if you switch between linked and header-only at a later date. If you use the header-only version, remember to have Eigen3 in your include path and to include the needed definitions, \texttt{RQ\_LITTLE\_ENDIAN} or \texttt{RQ\_BIG\_ENDIAN}
\marginlabel{Namespace}
Since the libRaptorQ library now has two very different usages, everything was split into two namespaces: \textbf{RFC6330} and \textbf{RaptorQ}. The names should be explanatory: the first namespace (that we cover in this document) is for the RFC interface, while the second namespace is for the low-level interface.
Since we will cover only the \textbf{RFC6330} namespace here, remember that everything we talk about is in this namespace.
Both the RFC6330 and the RaptorQ namespace are versioned so you can keep using old APIs while we change stuff. just add \textbf{\texttt{\_\_v1}} to select the first version of the API. E.g: \textbf{\texttt{RFC6330\_\_v1}}
\newpage
\index{Iterators}\marginlabel{Templates}
There are two main classes you will use:
\begin{verbatim}
template <typename Rnd_It, typename Fwd_It>
class Encoder
template <typename In_It, typename Fwd_It>
class Decoder
\end{verbatim}
As you might guess, the classes for the encoder and decoder take two template parameters.\\
For the \textbf{Encoder}, the first parameter \textit{MUST} be a \textit{random access iterator} to the input data, and the second parameter is an
\textit{forward iterator}. The random access iterator will be used to scan your input data, and perform an interleaving (if you did not set the same size the
symbol and to the subsymbol). The forward iterator will be used to write the data to your structure.\\
The same is done for the \textbf{Decoder}, but we do not need to do any interleaving on the input, so the first iterator can be just an input iterator,
and nothing more.
\subsubsection{Caching precomputations}
\index{Cache!C++'}
libRaptorQ can now cache the most used matrices for a quicker reference. The cache can be scaled up or down dynamically.
\begin{description}
\item[local\_cache\_size] \textbf{const uint64\_t local\_cache}\\
\textbf{return: bool}\\
Set the local cache size. returns false on error.
\item[get\_local\_cache\_size()] \textbf{return: uint64\_t}\\
get the size of our local cache
\item[supported\_compressions()] \textbf{return: Compress} \\
Get the bitmask of all supported compression algorithms. currently only \textbf{Compress::NONE} and \textbf{Compress::LZ4}.
\item[get\_compression()] \textbf{return: Compress} \\
Get the current compression used.
\item [set\_compression] \textbf{const Compress compression} \\
\textbf{return: bool}\\
Try to set a compression algorithm. LZ4 is not a mandatory dependency, so we might not have it
\end{description}
\newpage
\subsubsection{Thread pool}
Both the encoder and the decoder can work with multiple blocks at the same time, so it makes sense to have a thread pool for such work.
\begin{description}
\item[set\_thread\_pool] \textbf{Input: const size\_t threads}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t max\_block\_concurrency}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RaptorQ\_\_v1::Work\_State exit\_type}\\
\textbf{return: bool}\\
set a pool of a certain number of threads. But the decoder is special: if one decoding fails we can always try again once we have at least one other symbol. The second parameter tells the pool how many times we can try to decode the same block concurrently. $1$ is a safe option.\\
The final argument is used when downsizing the queue: what do we do if all the threads are running but you asked to have less threads?
\begin{lstlisting}
enum class Work_State : uint8_t {
KEEP_WORKING
ABORT_COMPUTATION
};
\end{lstlisting}
\end{description}
\newpage
\subsubsection{The Encoder}
\index{Encoder!C++}
The constructor is the following:
\begin{lstlisting}[language=C++]
Encoder (const Rnd_It data_from, const Rnd_It data_to,
const uint16_t min_subsymbol_size,
const uint16_t symbol_size,
const size_t max_sub_block);
\end{lstlisting}
As we said, the RFC will decide by itself how many blocks each input will have, the actual size of the subsymbol, and the size of the blocks. Although needed for RFC compliance, you probably do not care about the \textit{min\_subsymbol\_size}, in which case the safe choice is to put it at the same size of the symbol.\\
The \textit{max\_sub\_block} the the maximum size of a sub-block in bytes. Again, unless you \textit{really} understand the RFC, you should probably ignore this and put it to a high value. This parameter is used to limit the block size used by the RFC, and is the size (in bytes) of the biggest block you allow the RFC to use.\\
\textbf{safe settings for max\_sub\_block:} sub\_symbol\_size * block\_size.
You can instantiate an encoder for example by doing:
\begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output;
...
using T_it = typename std::vector<uint32_t>::iterator;
RFC6330::Encoder<T_it, T_it> enc (input.begin(),
input.end(),
4, 1444, 10000)
\end{lstlisting}
This will create an Encoder that works on vectors of unsigned 32 bit integers for both input and output, that will create symbols of size 1444 bytes, interleaving
your input every 4 bytes, and try to work with a big number of symbols per blocks.
The available methods for the encoder are the following:
\begin{description}
\item[operator bool()] \textbf{return:bool}\\
False if constructor parameters did not make sense. Else true.
\item[OTI\_Common()] \textbf{return: OTI\_Common\_Data}, aka \textit{uint64\_t}.\\
Keeps total \textbf{file size and symbol size}. You need to send this to the receiver, so that it will be able to properly decode the data.
\item[OTI\_Scheme\_Specific\_Data()] \textbf{return: OTI\_Scheme\_Specific\_Data}, aka \textit{uint32\_t}.\\
Keeps number of \textbf{source blocks, sub blocks, and alignment}. As for the OTI\_Common\_Data, you need to send this to the receiver to be able to
properly decode the data.
\item[compute] \textbf{Input: const Compute flags}\\
\textbf{return: std::future<std::pair<Error, uint8\_t> >}\\
For C++11 only (automatically disabled on C++98). Start the computation, return a future, which has a pair of Error and the number of blocks ready.\\
Compute flags:
\begin{lstlisting}[language=C++]
enum class Compute : uint8_t {
NONE
// report every time there is more data, but only
// if that data starts at the beginning.
PARTIAL_FROM_BEGINNING
PARTIAL_ANY
COMPLETE // only when all the data is available
NO_BACKGROUND // only return after the computation is finished
NO_POOL // do not use the thread pool
NO_RETRY // do not retry if we failed, even if there is more data
};
\end{lstlisting}
\item [precompute\_max\_memory] \textbf{return: size\_t}\\
Each precomputation can take a lot of memory, depending on the configuration, so you might want to limit the number of precomputations run in parallel
depending on the memory used. This method returns the amount of memory taken by \textbf{ONE} precomputation.
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t esi, const uint8\_t sbn}.\\
\textbf{return:size\_t}.\\
Take as input the iterators to the data structure into where we have to save the encoded data, the \textbf{Encoding Symbol Id} and the
\textbf{Source Block Number}. As you are writing in C++, you probably want to use the iterators begin/end, though. Returns the number of written
iterators (\textbf{NOT} the bytes)
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t id}.\\
\textbf{return:size\_t}.\\
Exactly as before, but the \textbf{id} contains both the \textit{source block number} and the \textit{encoding symbol id}
\item[begin()] \textbf{return: Block\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the blocks in which the RFC divided the input data. See later to understand how to use it.
\item[end()] \textbf{return: const Block\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the end of the blocks in which the RFC divided the input data. See later to understand how to use it.
\item [free] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: void}\\
Each block takes some memory, (a bit more than $symbols * symbol\_size$), so once you are done sending source and repair symbols for one block,
you might want to free the memory of that block.
\item[blocks()] \textbf{return: uint8\_t} The number of blocks.
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
The block size, in bytes. Each block can have different symbols and thus different size. The last block also might have more padding, and with this you get the real size of that block, without the padding.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
\item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in a specific block. different blocks can have different symbols.
\item[extended\_symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The \textbf{real, internal} number of symbols in a specific block. This can be slightly higher than the \texttt{symbols()} call, and \textit{you probably do not need this} for anything except if you are trying to know more about the RaptorQ algorithm
\item[max\_repair] \textbf{Input: const uint8\_t sbn)}\\
\textbf{return: uint32\_t}\\
The maximum amount of repair symbols that you can generate. Something less than $2^{24}$, but the exact number depends on the number of symbols
in a block
\end{description}
\subsubsection{Blocks}
\index{Blocks!C++}
With the \textit{begin()/end()} calls you get \textit{Input iterators} to the blocks. A Block has the following type:
\begin{lstlisting}[language=C++]
template <typename Rnd_It, typename Fwd_It>
class Block
\end{lstlisting}
and exposes the following methods:
\begin{description}
\item[begin\_source()]\textbf{return: Symbol\_Iterator}
\item[end\_source()]\textbf{return: Symbol\_Iterator}
\item[begin\_repair()]\textbf{return: Symbol\_Iterator}
\item[end\_repair()]\textbf{Input: const uint32\_t max\_repair}\\
\textbf{return: Symbol\_Iterator}
\item[id()]\textbf{return: uint8\_t}
\item[max\_repair()]\textbf{return: uint32\_t}
\item[symbols()]\textbf{return: uint16\_t}
\item[extended\_symbols()]\textbf{return: uint16\_t}\\
As before, this is the amount of internal symbols, that include padding symbols. You probably do not need this unless you are trying
to know more about the RaptorQ algorithm
\item[block\_size()]\textbf{return: uint32\_t}
\end{description}
As the names explain, you will get an iterator to the symbols in the block. As the number of repair symbols can vary, for now you get two separate begin/ends,
so that you can check when you sent the source symbols, and how many repair symbols you send.\\
The other functions are helpers for the details of the block.
\subsubsection{Symbols}
\index{Symbols!C++}
Finally, through the \textit{Symbol\_Iterator} \textit{Input Iterator} we get the \textbf{Symbol} class:
\begin{lstlisting}[language=C++]
template <typename Rnd_It, typename Fwd_It>
class Symbol
\end{lstlisting}
which exposes the following methods we need to get the symbol data:
\begin{description}
\item[operator()]\textbf{Input:Fwd\_It \&start, const Fwd\_It end}\\
\textbf{return: uint64\_t}\\
takes a forward iterator, and fill it with the symbol data. returns the number of written iterators.
\item[id()]\textbf{return: uint32\_t}\\
return the id (\textit{$sbn + esi$}) of this symbol, that you need to include in every packet you send, before the symbols.
\item[block()]\textbf{return: uint8\_t}\\
return the \textit{block} of this symbol.
\item[esi()]\textbf{return: uint32\_t}\\
return the \textit{esi} of this symbol.
\end{description}
\newpage
\subsubsection{The Decoder}
\index{Decoder!C++}
The decoder is a bit simpler than the encoder.
There are two constructors for the Decoder:
\begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output;
using T_it = typename std::vector<uint32_t>::iterator;
RaptorQ::Decoder<T_it, T_it> dec (
const OTI_Common_Data common,
const OTI_Scheme_Specific_Data scheme)
RaptorQ::Decoder<T_it, T_it> dec (uint64_t size,
uint16_t symbol_size,
uint16_t sub_blocks,
uint8_t blocks)
\end{lstlisting}
Which should be pretty self-explanatory, once you understand how the encoder works:
you can either get the \texttt{OTI} parameters from the Encoder or you can provide the raw values by yourself (only if you know what you are doing)
The remaining methods are:
\begin{description}
\item[begin()] \textbf{return: Block\_Iterator<In\_It, Fwd\_It>}\\
This returns an iterator to the blocks in which the RFC divided the input data.
\item[end()] \textbf{return: const Block\_Iterator<In\_It, Fwd\_It>}\\
This returns an iterator to the end of the blocks in which the RFC divided the input data.
\item[operator bool()] \textbf{return: bool}\\
check if the Decoder was initialized properly
\item[compute] \textbf{Input: const Compute flags}\\
\textbf{return: std::future<std::pair<Error, uint8\_t> >}\\
Only enabled if you use C++11. Start the computation as soon as enough data is available.\\
Compute flags:
\begin{lstlisting}[language=C++]
enum class Compute : uint8_t {
NONE
// report every time there is more data, but only
// if that data starts at the beginning.
PARTIAL_FROM_BEGINNING
PARTIAL_ANY
COMPLETE // only when all the data is available
NO_BACKGROUND // only return after the computation is finished
NO_POOL // do not use the thread pool
NO_RETRY // do not retry if we failed, even if there is more data
};
\end{lstlisting}
\item[end\_of\_input] \textbf{Input: const Fill\_With\_Zeros fill}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: std::vector<bool>}\\
Tell the decoder that we know that there will be no more data for this block.
The first parameter is an \texttt{enum}, either \textbf{Fill\_With\_Zeros::YES} or \textbf{Fill\_With\_Zeros::NO}
If you choose \texttt{YES} then the repair symbols will be discarded, the missing symbols will be filled with zeros
and for output you will get the bitmask for which byte was received (true) or was filled with zeros (false).\\
If you choose \textbf{Fill\_With\_Zeros::NO} you will get \textit{an empty vector}
\item[end\_of\_input] \textbf{Input: const Fill\_With\_Zeros fill}\\
\textbf{return: std::vector<bool>}\\
Same as before, but we know that there will be no more inputs for any block.
\item[decode\_symbol]\textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}
\textbf{return: uint64\_t}\\
Decode a single symbol. you need to provide the block number and symbol identifier.
\item[decode\_bytes] \textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: uint64\_t}\\
Decode everything. returns the number of bytes written. You can start writing the output \texttt{skip} bytes after the first iterator in case the parameters you have used do not align (set to $0$ for everyone else).
\item[decode\_bytes] \textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint64\_t}\\
Same as before, but you can specify a \textbf{Single Block Number} instead of the whole output
\newpage
\item[decode\_aligned]\textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: aligned\_res}\\
Same as \texttt{decode\_bytes}, but yu can use this if for some weir reason the data does not align with the type of the output
\begin{lstlisting}[language=C++]
struct aligned_res
{
uint64_t written; // iterators written
uint8_t offset; // bytes written in the last
// iterator if it was not full
};
\end{lstlisting}
\item[decode\_block\_aligned]\textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: aligned\_res}\\
Same as before, but now you can choose the block
\item[add\_symbol]\textbf{Input: In\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const In\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: RFC6330::Error}\\
Add one symbol, while explicitly specifying the symbol id and the block id.
\item[add\_symbol]\textbf{Input: In\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const In\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t id}\\
\textbf{return: RFC6330::Error}\\
Same as before, but extract the block id and the symbol id from the \textit{id} parameter
\item[blocks\_ready()]\textbf{return: uint8\_t}\\
return the number of the blocks that are ready for output
\item[is\_ready()]\textbf{return: bool}\\
Whether everything has been decoded or not.
\item[is\_block\_ready]\textbf{Input:const uint8\_t block}\\
\textbf{return: bool}\\
Test if the specified block has been decoded or not
\item[free]\textbf{Input: const uint8\_t sbn}\\
\textbf{return: void}\\
You might have stopped using a block, but the memory is still there. Free it.
\item[bytes()] \textbf{return: uint64\_t}\\
The total bytes of the output
\item[blocks()] \textbf{return: uint8\_t}\\
The number of blocks.
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
The block size, in bytes. Each block can have different symbols and thus different size.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
\item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in a specific block. different blocks can have different symbols.
\end{description}
\newpage
\subsection{C interface}\index{Interface!C}
The C interface looks a lot like the C++ one.\\
You need to include the \textbf{RaptorQ/RFC6330.h} header, and link the \textit{libRaptorQ} library.\\
\marginlabel{Static linking}\index{Static Linking}
If you are working with the static version of libRaptorQ remember to link the C++ standard library used when
compiling the library (\textit{libstdc++} for gcc or maybe \textit{libc++} for clang), your threding library (usually \textit{libpthread}), and the C math library (\textit{libm}).\\
\marginlabel{\textbf{API versioning}} To better support future changes in the API and to help you with future changes,
there are only 2 API calls in this library:
\begin{description}
\item[RFC6330\_api] \textbf{Input: uint32\_t version}\\
\textbf{return: struct RFC6330\_base\_api*}\\
This will give you a pointer to a struct containing pointers to the calls you will use.
The struct will change depending on which API you request (currently only number $1$ is supported).
Since this pointer works for every api, you have to cast it to the struct you expect, or you will not see
the pointers to the function calls. Then you can use that pointer to access of the functions, like:
\begin{lstlisting}[language=C]
struct RFC6330_v1 *rfc = (struct RFC6330_v1*)
RFC6330_api (1);
rfc->set_compression (RQ_COMPRESS_LZ4);
\end{lstlisting}
\item[RFC6330\_free\_api] \textbf{Input:struct RFC6330\_base\_api **api}\\
\textbf{return: void}\\
As the name implies, this frees the given structure.
\end{description}
Again, we only support API version 1 at the moment.
\index{Cache!C}
\subsubsection{Cache settings}
As for the C++11 interface, you can get and set the local memory cache.
\begin{description}
\item[supported\_compressions] \textbf{return: RFC6330\_Compress}\\
return the bitmask of all supported compressions. currently either \texttt{RQ\_COMPRESS\_NONE} or \texttt{RQ\_COMPRESS\_LZ4}
\item[get\_compression] \textbf{return: RFC6330\_Compress}\\
Which compression are we currently using?
\item[set\_compression] \textbf{Input: const RFC6330\_Compress compression}\\
\textbf{return: bool}\\
Set a compression method or another.
\item[local\_cache\_size] \textbf{Input: const size\_t bytes}\\
\textbf{return: size\_t}\\
Set the maximum amount of cache usable. Default: $0$
\item[get\_local\_cache\_size] \textbf{return: size\_t}\\
get the amount of cache configured
\end{description}
\subsubsection{C Constructors}
The first thing you need is to allocate the encoder or the decoder.
The C interface is just a wrapper around the C++ code, so you still have to specify the same things as before.
A quick glance at the constructors should give you all the information you need:
\begin{lstlisting}[language=C]
typedef enum { RQ_NONE, RQ_ENC_8, RQ_ENC_16,
RQ_ENC_32, RQ_ENC_64, RQ_DEC_8,
RQ_DEC_16, RQ_DEC_32, RQ_DEC_64}
RFC6330_type;
\end{lstlisting}
\begin{description}
\item[Encoder] \textbf{Input:const RFC6330\_type type}\\
.\ \ \ \ \ \ \ \ \ \textbf{void *data\_from}\\
.\ \ \ \ \ \ \ \ \ \textbf{const uint64\_t size}\\
.\ \ \ \ \ \ \ \ \ \textbf{const uint16\_t min\_subsymbol\_size}\\
.\ \ \ \ \ \ \ \ \ \textbf{const uint16\_t symbol\_size}\\
.\ \ \ \ \ \ \ \ \ \textbf{const size\_t max\_sub\_block}\\
\textbf{return: RFC6330\_ptr*}\\
Allocate an encoder. The encoder will work with uint8\_t, uint16\_t, uint32\_t or uint64\_t depending on which \texttt{RFC6330\_type} you choose. The best bet is usually to work with uint8\_t
\item[Decoder] \textbf{Input:const RFC6330\_type type}\\
.\ \ \textbf{const RFC6330\_OTI\_Common\_Data common}\\
.\ \ \textbf{const RFC6330\_OTI\_Scheme\_Specific\_Data scheme}\\
\textbf{return: RFC6330\_ptr*}\\
The \texttt{type} parameter works the same as in the encoder. The other two are parameters you get from the encoder.
\item[Decoder\_raw] \textbf{Input: RFC6330\_type type}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint64\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t symbol\_size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t sub\_blocks}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t blocks}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t alignment}\\
\textbf{return: struct RFC6330\_ptr*}\\
Same as before, but here you can specify the parameters by hand.
\item[initialized]\textbf{Input: const struct RFC6330\_ptr *ptr}\\
\textbf{return: bool}\\
check if the encoder or decoder was initialized properly.
\end{description}
\newpage
\subsubsection{Common functions for (de/en)coding}
These functions are used by both the encoder and the decoder, and will be helpful in tracking how much memory
you will need to allocate, or in general in managing the encoder and decoder.
The \textbf{ptr} must be a valid encoder or decoder.
The names for now are self-explanatory. Blocks can have different symbols, so the size of a block and the number of symbols in a block
depend on which block we are talking about.
\marginlabel{\textbf{symbols, blocks, memory}}\\
\begin{description}
\item[set\_thread\_pool] \textbf{Input: const size\_t threads}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t max\_block\_concurrency}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RFC6330\_Work exit\_type}\\
\textbf{return: bool}\\
Set the size of the thread pool for concurrent work. Since the decoder can fail, but also retry again if there is more data available,
you can also specify how many times libRaptorQ will try to decode the same block concurrently. $1$ is a safe option.\\
The last parameter, \texttt{RFC6330\_Work} specify what to do when you are downsizing the thread pool and some threads are still working. The possible values are \texttt{RQ\_WORK\_KEEP\_WORKING} and \texttt{RQ\_WORK\_ABORT\_COMPUTATION}.
\item[blocks] \textbf{Input: const struct RFC6330\_ptr *ptr}\\
\textbf{return: uint8\_t}\\
The number of blocks in this RFC6330 instance
\item[symbols]\textbf{Input: const struct RFC6330\_ptr *ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in the specified block
\item[extended\_symbols]\textbf{Input: const struct RFC6330\_ptr *ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The the \textbf{real, internal} number of symbols in the specified block. You probably \textit{do not want this} unless you are trying to understand more about the RaptorQ algorithm or you are benchmarking
\item[symbol\_size]\textbf{Input:const struct RFC6330\_ptr *ptr}\\
\textbf{return: size\_t}\\
The size of each symbol
\item[free\_block]\textbf{Input: struct RFC6330\_ptr **ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: void}\\
Free the allocated space for a single block
\item[free]\textbf{Input: struct RFC6330\_ptr **ptr}\\
\textbf{return: void}\\
Free the specified decoder or decoder and all its blocks.
\end{description}
\marginlabel{Computation}\label{computation}\\
\begin{description}
\item[compute] \textbf{Input: const struct RFC6330\_ptr *ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RFC6330\_Compute flags}\\
\textbf{return: RFC6330\_future*}\\
Start the computation as described in the flags. Returns a wrapper to the C++11 futures. You can wait or poll the state of the future in different ways (see right below)\\
\begin{lstlisting}[language=C]
typedef enum {
RQ_COMPUTE_NONE
// report every time there is more data, but only
// if that data starts at the beginning.
RQ_COMPUTE_PARTIAL_FROM_BEGINNING
RQ_COMPUTE_PARTIAL_ANY
RQ_COMPUTE_COMPLETE // only when all the data
// is available
RQ_COMPUTE_NO_BACKGROUND // only return after
// the computation is finished
RQ_COMPUTE_NO_POOL // do not use the thread pool
RQ_COMPUTE_NO_RETRY // do not retry if we failed,
// even if there is more data
} RFC6330_Compute;
\end{lstlisting}
\item[future\_state] \textbf{Input: const struct RFC6330\_future *f}\\
\textbf{return: RFC6330\_Error}\\
poll the state of the future. possible results:
\begin{lstlisting}[language=C]
RQ_ERR_WRONG_INPUT // null pointer?
RQ_ERR_NOT_NEEDED // you have already run
// future_get() on this future
RQ_ERR_WORKING // still working on it
RQ_ERR_NONE // ready to run future_get()
\end{lstlisting}
\newpage
\item[future\_wait\_for] \textbf{Input: const struct RFC6330\_future *f}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint64\_t time}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RFC6330\_Unit\_Time unit}\\
\textbf{return: RFC6330\_Error}\\
Wait for a future to become ready for X amount of time. the unit is specified in the
last parameter:
\begin{lstlisting}[language=C]
typedef enum {
RQ_TIME_NANOSEC,
RQ_TIME_MICROSEC,
RQ_TIME_MILLISEC,
RQ_TIME_SEC,
RQ_TIME_MIN,
RQ_TIME_HOUR
} RFC6330_Unit_Time;
\end{lstlisting}
\item[future\_wait] \textbf{Input: const struct RFC6330\_future *f}\\
\textbf{return: void}\\
Same as before, but wait indefinitely
\item[future\_get] \textbf{Input: struct RFC6330\_future *future}\\
\textbf{return: struct RFC6330\_Result}\\
Returns the state of the future. If the future is not ready it will wait indefinitely.
\begin{lstlisting}[language=C]
struct RAPTORQ_API RFC6330_Result {
RFC6330_Error error;
uint8_t sbn;
};
\end{lstlisting}
The sbn changes meaning depending to be consistent with the flags you passed to the \texttt{compute()} call.
\item[future\_free] \textbf{Input: struct RFC6330\_future **f}\\
\textbf{return: void}\\
Free the memory allocated for the future.
\end{description}
\newpage
\subsubsection{\textbf{Encoding}}
\index{Encoder!C}
Now we will look at the calls specific to the encoder (that is: calls that need an encoder as the first parameter, and don't make sense in a decoder)
\marginlabel{OTI Data}\\
First, we need to tell the receiver all the parameters that the encoder is using, and for that two functions are provided:
\begin{lstlisting}[language=C]
typedef uint64_t RaptorQ_OTI_Common_Data;
typedef uint32_t RaptorQ_OTI_Scheme_Specific_Data;
\end{lstlisting}
\begin{description}
\item[OTI\_Common]\textbf{Input: struct RaptorQ\_ptr *enc}\\
\textbf{return: RFC6330\_OTI\_Common\_Data}\\
Get the OTI\_Common
\item[OTI\_Scheme\_Specific]\textbf{Input: struct RaptorQ\_ptr *enc}\\
\textbf{return: RFC6330\_OTI\_Scheme\_Specific\_Data}\\
Get the OTI\_Scheme\_Specific
\end{description}
\marginlabel{Encoding}
\begin{description}
\item[max\_repair]\textbf{Input: const struct RFC6330\_ptr *enc}\\
.\ \ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
return the max number of repair packets that block can have. This is around $2^{24}$, so you probably always want to be much, much lower than this. The total number of symbols (source+repair) is limited in the RFC, and since the bigger blocks have more source symbols, the max number of repair symbols has to be lower in bigger blocks. Again, this is always in the ballpark of $2^{24}$, you probably don't want to use this many repair symbols.
\item[precompute\_max\_memory]\textbf{Input: const struct RFC6330\_ptr *enc}\\
\textbf{return: size\_t}\\
Get an estimate of how much memory (bytes) will cost the encoding of a single block. Not 100\% accurate, but close enough.
\item[encode\_id]\textbf{Input: const struct RFC6330\_ptr *enc}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t id}\\
\textbf{return: size\_t}\\
Get one encoded symbol, and store it in \texttt{**data}, which holds a pointer to "\texttt{size}" elements of type uint8\_t, uint16\_t, uint32\_t or uint64\_t depending on how the encoder was initialized. See \texttt{RFC6330\_type} in the encoder initialization. Returns the number of elements written.\\
If you request repair symbols before the computation is finished, the call will block until the data is available. This calls works for both source and repair symbols, just increase the id until you finish the source symbols
\item[encode]\textbf{Input: const struct RFC6330\_ptr *enc}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: size\_t}\\
Same as before, but you can specify the symbol and block explicitly.
\item[id]\textbf{Input: const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
combine the esi and the block number to form an id.
\end{description}
\newpage
\subsubsection{\textbf{Decoding}}
As for the encoder, there are functions which are specific for a decoder. Let's look at them
\marginlabel{General status calls}
\begin{description}
\item[bytes]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\textbf{return: uint64\_t}\\
Return the number of bytes that this decoder should output on complete, successful decoding.
\item[blocks\_ready]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\textbf{return: uint8\_t}\\
Return the number of blocks ready to be returned. Useful if you don't want to wait for the whole data.\\
The number of blocks reported can mean either the number of sequential blocks ready, or the number
of blocks regardless of order, depending on how the \texttt{compute} function was called (see "\texttt{Computation}" on \ref{computation})
\item[is\_ready]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\textbf{return: bool}\\
Test if the whole encoder is ready and all data decoded.
\item[is\_block\_ready]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: bool}\\
Test if a single block is ready and its data decoded.
\end{description}
\index{Decoder!C}\marginlabel{Decoding calls}
\begin{description}
\item[end\_of\_input]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \
\textbf{const RFC6330\_Fill\_With\_Zeros fill}\\
\textbf{return: struct RFC6330\_Byte\_Tracker}\\
Tell the decoder that there will be no more input for any block.
The second parameter is an enum, it can be either \textbf{RQ\_NO\_FILL} or \textbf{RQ\_FILL\_WITH\_ZEROS}.\\
If you use \texttt{RQ\_NO\_FILL} the structure returned will be empty, and the decoder will return error if there are not enough repair symbols.\\
If you use \texttt{RQ\_FILL\_WITH\_ZEROS} the decoder will immediately stop decoding and all missing symbols will be filled with zeros. The returned structure will contain a bitmask so you know if each byte comes from the network (true) or it is one of the filled ones (false)
\item[end\_of\_block\_input]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \
\textbf{const RFC6330\_Fill\_With\_Zeros fill}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: struct RFC6330\_Byte\_Tracker}\\
Same as before, but for a single block instead of the whole data.
\item[add\_symbol\_id]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uin32\_t id}\\
\textbf{return: RFC6330\_Error}\\
Add a symbol to the decoder. The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (RQ\_DEC\_8, RQ\_DEC\_16 etc..). Afterwards data will point \textbf{after} the symbol.
\item[add\_symbol\_id]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uin32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uin8\_t sbn}\\
\textbf{return: RFC6330\_Error}\\
Same as with \texttt{add\_symbol\_id}, but now you can specify the esi and the block number manually
\item[decode\_aligned]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: RFC6330\_Dec\_Result}\\
Decode everything as asked by the \texttt{compute()} call (see "\texttt{Computation}" on \ref{computation}).\\
The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (\texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc..). Afterwards data will point \textbf{after} the decoded data. You can skip "skip" bytes at the beginning of **data if the pointer
is not aligned with your data.
\begin{lstlisting}[language=C]
struct RAPTORQ_API RFC6330_Dec_Result {
uint64_t written;
uint8_t offset;
};
\end{lstlisting}
The result is the number of written \textbf{elements} (size depending on \texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc).\\
The \texttt{offset} parameter is the amount of bytes written in the last element if the alignment used in the encoder does not fit the
alignment used in the decoder. Usually you keep those synchronized, so it should be zero (element fully written)
\newpage
\item[decode\_block\_aligned]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: RFC6330\_Dec\_Result}\\
Same as before, but now you can decode a single block.
\item[decode\_symbol]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint64\_t}\\
As you can expect, decode a single symbol. The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (\texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc..). Afterwards data will point \textbf{after} the decoded data.\\
Returns the number of bytes written
\item[decode\_bytes]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: uint64\_t}\\
Decode everything as asked by the \texttt{compute()} call (see "\texttt{Computation}" on \ref{computation}).\\
The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (\texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc..). Afterwards data will point \textbf{after} the decoded data. You can skip "skip" bytes at the beginning of **data if the pointer is not aligned with your data.\\
Returns the number of written bytes.
\item[decode\_block\_bytes]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: uint64\_t}\\
Same as before, but now decode a single block
\end{description}
\input{GNU_FDL.tex}
\printindex
\end{document}
doc/libRaptorQ.tex → doc/libRaptorQ-RaptorQ.tex
View file @ a394b22a
% Copyright (C) 2015, Luca Fulchir<luca@fulchir.it>
% Copyright (C) 2018, Luca Fulchir<luca@fulchir.it>
% Permission is granted to copy, distribute and/or modify this document
% under the terms of the GNU Free Documentation License, Version 1.3
% or any later version published by the Free Software Foundation;
......@@ -10,6 +10,7 @@
\documentclass[11pt,a4paper]{refart}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{hyperref}
\usepackage{makeidx}
\usepackage{verbatimbox}
......@@ -48,7 +49,7 @@
title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title
}
\title{Using libRaptorQ library}
\title{Using libRaptorQ library: RAW interface}
\date{\today}
\makeindex
......@@ -57,20 +58,21 @@
\begin{abstract}
\textbf{libRaptorQ} is a C++11 implementation of the RaptorQ Forward Error Correction, as described in the \href{https://tools.ietf.org/html/rfc6330}{RFC6330} .
\textbf{libRaptorQ} is a C++11
implementation of the RaptorQ Forward Error Correction. It includes two interfaces, one derived from the \href{https://tools.ietf.org/html/rfc6330}{RFC6330} standard, and one RAW API for a more flexible and easy usage.
This document is only about the RAW interface.
The implementation was started as a university laboratory project, and will be later used and included in \href{https://www.fenrirproject.org}{Fenrir}, the
maintainer's master thesis.
maintainer's master thesis project.
This implementation is quite short (the core is $\sim3k$ lines), thanks to the chosen language and the use of external libraries for matrix handling (eigen3).
libRaptorQ is the only RaptorQ implementation in C++, include C hooks, and it is the only free (\textbf{LGPL3}) implementation of the rfc, except for the (apache2)
java implementation, OpenRQ , which is much bigger ($\sim 46k$) and slower.
\end{abstract}
\vfill\hfill
\begin{verbbox}[\tiny]
Copyright (C) 2015, Luca Fulchir<luker@fenrirproject.org>
Copyright (C) 2015-2018, Luca Fulchir<luker@fenrirproject.org>
Permission is granted to copy, distribute and/or modify this document
under the terms of the GNU Free Documentation License, Version 1.3
or any later version published by the Free Software Foundation;
......@@ -90,10 +92,10 @@ Free Documentation License".
\section{Contacts}\index{Contacts}
The main development and dicussions on the project, along with bug reporting, happens on
\href{https://www.fenrirproject.org/projects/libraptorq/}{the main website}.
\href{https://fenrirproject.org/Luker/libRaptorQ}{the main website}.
\marginlabel{Mailing Lists} Mailing lists are available at \href{https://www.fenrirproject.org/lists}{https://www.fenrirproject.org/lists}\\
The two mailing lists are for development and announcments, but due to the low traffic of the development mailing list, it
\marginlabel{Mailing Lists} Mailing lists are available at \href{https://fenrirproject.org/lists}{https://fenrirproject.org/lists}\\
The two mailing lists are for development and announcements, but due to the low traffic of the development mailing list, it
can also be used by users for questions on the project.
\marginlabel{IRC} Since there are not many developers for now, the main irc channel is \textbf{\#fenrirproject} on freenode
......@@ -103,40 +105,34 @@ can also be used by users for questions on the project.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Build \& install}
\subsection{Get the source code}
Although things seems to work, no stable release has been released yet, as of \today.
This document follows the 1.0 stable release.
This means you can only check this out with git.
You can get the tarballs from the main website here:\\
\href{https://fenrirproject.org/Luker/libRaptorQ/tags}{https://fenrirproject.org/Luker/libRaptorQ/tags}
To check out the repository:
Or you can check out the repository (warning: development in progress):
\begin{verbatim}
$ git clone https://github.com/LucaFulchir/libRaptorQ.git
\end{verbatim}
\begin{lstlisting}[language=bash,breaklines=true]
$ git clone --recurse-submodules \
https://fenrirproject.org/Luker/libRaptorQ.git
\end{lstlisting}
You can also get it from our main server:
You can also get it from github:
\begin{verbatim}
$ git clone https://www.fenrirproject.org/Luker/libRaptorQ.git
\end{verbatim}
\begin{lstlisting}[language=bash,breaklines=true]
$ git clone --recurse-submodules \
https://github.com/LucaFulchir/libRaptorQ.git
\end{lstlisting}
\index{GPG}\marginlabel{GPG verification:}
Once you have cloned it, it's always a good thing to check the repository gpg
signatures, so you can import my key with:
\begin{verbatim}
$ gpg --keyserver pgp.mit.edu --recv-key D42DDF0A
\end{verbatim}
please check the full fingerprint, it should be this:
\begin{lstlisting}[language=bash,breaklines=true]
$ gpg --keyserver pgp.mit.edu --recv-key \
7393DAD255BEB5751DBDA04AB11CD823BA278C85
\end{lstlisting}
\begin{verbbox}[\footnotesize]
$ gpg2 --fingerprint D42DDF0A
pub rsa3072/D42DDF0A 2015-01-01 [expires: 2016-01-01]
Key fingerprint = AB35 E45F 5CA5 E35B 8B55 818F 0157 D133 D42D DF0A
uid [ unknown] Luca Fulchir (2015 key) <luker@fenrirproject.org>
\end{verbbox}
\theverbbox
Now you have the source, and the key, it's enough to check the signature of the
last commit:
......@@ -148,28 +144,43 @@ $ git log -n 1 --show-signature
The important part is that you get something like this:
\begin{verbbox}[\footnotesize]
gpg: Signature made Fri 27 Mar 2015 20:59:59 CET using RSA key ID D42DDF0A
gpg: Good signature from "Luca Fulchir (2015 key) <luker@fenrirproject.org>"
[unknown]
gpg: Signature made Mon 11 Dec 2017 21:55:28 CET
gpg: using RSA key 8F7474044095B405D0F042F0A2CCA134BC7C8572
gpg: Good signature from "Luca Fulchir <luker@fenrirproject.org>" [unknown]
gpg: aka "Luca Fulchir <luca@fulchir.it>" [unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: AB35 E45F 5CA5 E35B 8B55 818F 0157 D133 D42D DF0A
Author: Luca Fulchir <luker@fenrirproject.org>
Primary key fingerprint: 7393 DAD2 55BE B575 1DBD A04A B11C D823 BA27 8C85
\end{verbbox}
\theverbbox
And as long as you got the right key, and you find the \textbf{"gpg: Good signature"} string,
you can be sure you have the right code.
The main development happens in the $master$ branch, while we keep one branch for each stable version.
\subsection{Dependencies}\index{dependencies}
libRaptorQ has only 2 dependencies:
libRaptorQ has 4 dependencies:
\begin{description}
\item[\textbf{Eigen3}] This is used for matrix manipulation, which is a big part of RaptorQ.
\item[\textbf{git}] This is used not only to get the source, but also by the build system. We get the last git commit id and feed it to clang or gcc as seed for their
internal random number generator. This makes it possible to have reproducible builds.
\item[\textbf{lz4}] Used to compress cached precomputations.
\item[\textbf{git}] This is used not only to get the source, but also by the build system. We get the last git commit id and feed it to clang or gcc as seed for their internal random number generator. This makes it possible to have reproducible builds.
\item[\textbf{optionparser}] Library used to parse the command line easily in C++, without exceptions.
\end{description}
All dependencies are included in the sources, so you do not need to download and compile them.\\
Eigen3 is only needed at build time, so there is no need to download it again. we use the 3.3.4 version.\\
LZ4 is included as a git submodule. if you do not have it, run:
\begin{verbatim}
$ git submodule init
$ git submodule update
\end{verbatim}
and LZ4 w
ill be built
statically in the library (it will not be installed on the system)
\subsection{Build \& Install}
libRaptorQ uses the cMake build system, so things are fairly standard:
......@@ -177,6 +188,7 @@ libRaptorQ uses the cMake build system, so things are fairly standard:
\begin{verbatim}
$ cd libRaptorQ.git
$ mkdir build
$ cd build
$ cmake ../
$ make -j 4
\end{verbatim}
......@@ -193,16 +205,18 @@ You can always use the cmake-gui or ccmake commands to have the list of possible
The ones we recognize are:
\begin{description}
\item[LTO] yes/no. Default:\textbf{yes}. Enable \textit{Link Time Optimizatios} for clang or gcc. Makes libraries smaller and better optimized.
\item[PROFILING] yes/no. Default:\textbf{yes}. \textit{WARN: breaks deterministic builds with clang.}
\item[LTO] ON/OFF. Default:\textbf{ON}. Enables \textit{Link Time Optimizatios} for clang or gcc. Makes libraries smaller and better optimized.
\item[PROFILING] ON/OFF. Default:\textbf{ON}.
Profiling compiles everything once, then runs a test to see which code paths are used more, and then
recompiles everything again, but making sure that the binary is optimized for those paths. Works with clang and gcc. Provides a slight speedup.
\item[CMAKE\_C\_COMPILER] gcc and clang are directly supported. other should work, too. This is only used if you want to build the C example.
\item[CMAKE\_CXX\_COMPILER] g++, clang++ are directly supported. other should work, too.
\item[STDLIB] change the c++ standard library. ``libstdc++'' for the (default) gcc one, ``libc++'' for the clang/llvm one. Note that it seems you can't use libc++ with gcc yet.
\item[CMAKE\_CXX\_FLAGS] Additional compiler flags you might want to pass.
\item[CLANG\_STDLIB] ON/OFF. Default:\textbf{OFF}.use clang's libc++ standard library. Only available with clang.
\item[CMAKE\_BUILD\_TYPE] Type of build. you can choose between ``Debug'', ``Release'', ``MinSizeRel'' or ``RelWithDebInfo''
\item[CMAKE\_INSTALL\_PREFIX] Default to \textit{/usr/local}. Change it to fit your distribution guidelines.
\item[RQ\_ENDIANNESS] \texttt{Auto,BigEndian,LittleEndian} Force the build to use one endianness ot the other
\item[RQ\_LINKER] \texttt{Auto,gold,ld,bsd} Force the build to use one linker or the other
\item[USE\_LZ4] ON/OFF. Default:\textbf{ON}. compile with support for lz4
\end{description}
Then you can build everything by running:
......@@ -256,42 +270,51 @@ number of symbols. Which is a completely different thing.
It is still very quick. On a core i7, 2.4Ghz, you need to wait \textit{0.4ms} for $10$ symbols, \textit{280ms} for 1.000 symbols, but it can take an hour for $27.000$ symbols.
RaptorQ handles up to $56.403$ symbols.
\subsection{Blocks \& Symbols}
\marginlabel{\textbf{Caching}} libRaptorQ can work with big matrices that take a lot of time to compute. For this reason the matrices can be saved once they have
been computed the first time. libRaptorQ uses a \textbf{local} cache. The matrix can be compressed with LZ4.
\subsection{RaptorQ: Blocks \& Symbols}
To understand how to properly use and tune libRaptorQ, you first need to understand how RaptorQ handles its inputs, outputs, and what the time and memory
constraints are.
\index{Sequencing}\marginlabel{Input sequencing:}
RaptorQ needs to have the whole input you want to send before it can start working.\\
This means that it might be a little more difficult to use in live-streaming contexts, or where you need real-time data, but libRaptorQ will have options to
facilitate usage even in those contexts.
The RC6330 API needs to have the whole input you want to send before it can start working.\\
This means that You should buffer the data you are sending, and then when you have a block big enough you pass it to libRaptorQ, which will start generating symbols.
Once you have the whole input, RaptorQ divides it into \textbf{blocks}. Each block \textit{is encoded and decoded independently} and will be divided into \textbf{symbols}. Each symbol \textit{should}
be transmitted separately in its own packet (if you are on the network).
Contrary to the RFC6330, the RAW api is about a single block of data. You will have to divide your input in multiple blocks by hand, and decide by yourself how to send the encoded symbols. This will increase your flexibility, at the expense of not being conformant to RFC6330.
Note that a RaptorQ block does not need to be completely filled since the block is automatically padded with zeros, but you need to somehow tell the receiver how much data you have in that block so that it can know where the padding starts. RFC6330 does this with the \texttt{OTI} parameters. \textbf{with the RAW API you will have to do it by yourself}.
-
\index{Sequencing!Sizes}\marginlabel{Sizes:}Each input can have up to \textit{256 blocks}, each block can have up to \textit{$56.403$ symbols}, and each
symbol can have up to \textit{$2^{16}-1$ bytes}long. This gives a maximum files size of almost $881$ GB (946270874880 bytes to be exact)
\index{Sequencing!Sizes}\marginlabel{Sizes:} Each block can have up to \textit{$56.403$ symbols}, and each
symbol can have up to \textit{size\_t bytes} long ($2^{32}$ on 32 bit machines, $2^{
64}$ on 64 bits
). Please remember the Complexity discussion before using high values.
\index{Interleaving}\marginlabel{Interleaving:}
An other feature of RaptorQ is to automatically provide some interleaving of the input data before transmitting it. This means that one symbol will not
represent one sequential chunk of your input data, but more likely it's the concatenation of different \textbf{sub-symbols}. The size of the subsymbol must thus
be a fraction of the symbol size. This feature is not used if you set the size of the subsymbol to the size of symbol.
\index{Interleaving}\marginlabel{Interleaving:}
Contrary to the RFC6330 the RAW api does not include any kind of interleaving
\marginlabel{Memory and Time:}
Memory and time requirements are to be considered, though, as RaptorQ needs to run a cubic algorithm on matrix of size $K*K$, where $K$ is the number of
symbols in each block.\\
Memory and time requirements are to be considered since RaptorQ needs to run a cubic algorithm on matrix of size $K*K$, where $K$ is the number of symbols in each block.\\
The algorithm needs to keep in memory two of these matrices, although most of the work is done on only one.\\
This is actually a lot. More benchmarks and optimizations will come later, for now remember that with 10 symbols it takes something like 0.4ms on a core i7 2.4GHZ, 280ms with 1000 symbols, and up to an hour with 27.000 symbols.
This is actually a lot. More benchmarks and optimizations will come later, for now remember that with 10 symbols it takes something like 0.4ms on a core i7 2.4GHZ, 280ms with 1000 symbols, and up to an hour with 27.000 symbols. YMMV.
\subsection{C++ interface}
\index{Interface!C++}
To use the C++ interface you only need to include the \textbf{RaptorQ.hpp} header, and link \textbf{libRaptorQ} and your threading library (usually
\textit{libpthread}).
To use the C++ interface you only need to include the \textbf{RaptorQ/RaptorQ\_v1.hpp} header, and link \textbf{libRaptorQ} and your threading library (usually \textit{libpthread}).
You can also use the library as a header-only include, by including \textbf{RaptorQ/RaptorQ\_v1\_hdr.hpp}. All your code should keep working even if you switch between linked and header-only at a later date. If you use the header-only version, remember to have Eigen3 in your include path, and to include the needed definitions, \texttt{RQ\_LITTLE\_ENDIAN} or \texttt{RQ\_BIG\_ENDIAN}
\marginlabel{Namespace}
Since the libRaptorQ library now has two very different usages, everything was split into two namespaces: \textbf{RFC6330} and \textbf{RaptorQ}. The names should be explanatory: the first namespace is for the RFC interface, while the second namespace is for the low-level interface.
Since we will cover only the \textbf{RaptorQ} namespace here, remember that everything we talk about is in this namespace.
To provide grater flexibility, the whole library uses iterators to read your data, and to write the data onto your data structures.\\
This means that a big part of the library is a template, which adapts to the alignment of the data in the data structures you use.
Both the RFC6330 and the RaptorQ namespace are versioned so you can keep using old APIs while we change stuff. just add \textbf{\texttt{\_\_v1}} to select the first version of the API. E.g: \textbf{\texttt{RaptorQ\_\_v1}}
\newpage
\index{Iterators}\marginlabel{Templates}
There are two main classes you will use:
......@@ -304,28 +327,66 @@ class Decoder
\end{verbatim}
As you might guess, the classes for the encoder and decoder take two template parameters.\\
For the \textbf{Encoder}, the first parameter \textit{MUST} be a \textit{random access iterator} to the input data, and the second parameter is an
\textit{forward iterator}. The random access iterator will be used to scan your input data, and perform an interleaving (if you did not set the same size the
symbol and to the subsymbol). The forward iterator will be used to write the data to your structure.\\
The same is done for the \textbf{Decoder}, but we do not need to do any interleaving on the input, so the first iterator can be just an input iterator,
For the \textbf{Encoder}, the first parameter \textit{MUST} be a \textit{random access iterator} to the input data, and the second parameter is a
\textit{forward iterator}. The random access iterator will be used to scan your input data. The forward iterator will be used to write the data to your output.\\
The same is done for the \textbf{Decoder}, but the first iterator can be just an input iterator,
and nothing more.
\subsubsection{Caching precomputations}
\index{Cache!C++'}
libRaptorQ can now cache the most used matrices for a quicker reference. The cache can be scaled up or down dynamically.
\begin{description}
\item[local\_cache\_size] \textbf{const uint64\_t local\_cache}\\
\textbf{return: bool}\\
Set the local cache size. returns false on error.
\item[get\_local\_cache\_size()] \textbf{return: uint64\_t}\\
get the size of our local cache
\item[supported\_compressions()] \textbf{return: Compress} \\
Get the bitmask of all supported compression algorithms. currently only \textbf{Compress::NONE} and \textbf{Compress::LZ4}.
\item[get\_compression()] \textbf{return: Compress} \\
Get the current compression used.
\item [set\_compression] \textbf{const Compress compression} \\
\textbf{return: bool}\\
Try to set a compression algorithm. LZ4 is not a mandatory dependency, so we might not have it
\end{description}
\newpage
\subsubsection{The Encoder}
\index{Encoder!C++}
The constructor is the following:
\begin{lstlisting}[language=C++]
Encoder (const Block_Size symbols, const size_t symbol_size);
\end{lstlisting}
Which is really simple. The first is an Enum that will help you get the correct block size, and the second is your arbitrary symbol size.
Since the RaptorQ algorithm only works with specific block sizes, You can get all the possible block sizes from \texttt{RaptorQ\_\_v1::*blocks}, which is a pointer to an array of 477 possible sizes, from $10$ to $56403$. You can cast each element in this array to \texttt{uint16\_t} to verify the actual value, or directly name them, as in \texttt{RaptorQ\_\_v1::Block\_Size::Block\_42}.
If you pass unsafe values the encoder (or decoder) will refuse to work.
You can instantiate an encoder for example by doing:
\begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output;
...
using T_it = typename std::vector<uint32_t>::iterator;
RaptorQ::Encoder<T_it, T_it> enc (input.begin(),
input.end(),
4, 1444, 10000)
using RaptorQ = RaptorQ__v1;
RaptorQ::Encoder<T_it, T_it> enc (
RaptorQ::Block_Size::Block_42,
1500);
\end{lstlisting}
This will create an Encoder that works on vectors of unsigned 32 bit integers for both input and output, that will create symbols of size 1444 bytes, interleaving
your input every 4 bytes, and try to work with big number of symbols per blocks (\textbf{TODO: explain memory requirements})
This will create an Encoder that works on vectors of unsigned 32 bit integers for both input and output, that will be fed to a block of size 42
symbols of size 1500 bytes.
The available methods for the encoder are the following:
......@@ -333,326 +394,202 @@ The available methods for the encoder are the following:
\item[operator bool()] \textbf{return:bool}\\
False if constructor parameters did not make sense. Else true.
\item[OTI\_Common()] \textbf{return: OTI\_Common\_Data}, aka \textit{uint64\_t}.\\
Keeps total \textbf{file size and symbol size}. You need to send this to the receiver, so that it will be able to properly decode the data.
\item[OTI\_Scheme\_Specific\_Data()] \textbf{return: OTI\_Scheme\_Specific\_Data}, aka \textit{uint32\_t}.\\
Keeps number of \textbf{source blocks, sub blocks, and alignment}. As for the OTI\_Common\_Data, you need to send this to the receiver to be able to
properly decode the data.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t esi, const uint8\_t sbn}.\\
\textbf{return:uint64\_t}.\\
Take as input the iterators to the data structure into where we have to save the encoded data, the \textbf{Encoding Symbol Id} and the
\textbf{Source Block Number}. As you are writing in C++, you probably want to use the iterators begin/end, though. Returns the number of written
iterators (\textbf{NOT} the bytes)
\item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in a specific block. different blocks can have different symbols.
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t id}.\\
\textbf{return:uint64\_t}.\\
Exactly as before, but the \textbf{id} contains both the \textit{source block number} and the \textit{encoding symbol id}
\item[max\_repair] \textbf{Input: const uint8\_t sbn)}\\
\textbf{return: uint32\_t}\\
The maximum amount of repair symbols that you can generate. Something less than $2^{24}$, but the exact number depends on the number of symbols in a block
\item[begin()] \textbf{return: Block\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the blocks in which RaptorQ divided the input data. See later to understand how to use it.
\item[end()] \textbf{return: const Block\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the end of the blocks in which RaptorQ divided the input data. See later to understand how to use it.
\item[begin\_source()] \textbf{return: Symbol\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the symbols in this block. Source symbols only
\item[end\_source()] \textbf{return: const Symbol.\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the end of the source symbols.
\item[begin\_repair()] \textbf{return: Symbol\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the symbols in this block. Source symbols only
\item[end\_repair] \textbf{Input: const uint32\_t max\_repair}\\
\textbf{return: const Symbol.\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the end of the source symbols.
\item[precompute] \textbf{Input:const uint8\_t threads, const bool background}\\
\textbf{return: void}\\
Do the work of computing all different blocks in multithread. If \textit{background} is true, then return immediately, else return only when the job is done.\\
If \textit{threads} is $0$, try to guess the maximum threads from the number of available cpus.
\item[has\_data()] \textbf{return: bool}\\
Did you feed this encoder the data? That is: did you use \texttt{set\_data()} without calling \texttt{clear\_data()} later?
\item [precompute\_max\_memory] \textbf{return: size\_t}\\
Each precomputation can take a lot of memory, depending on the configuration, so you might want to limit the number of precomputations run in parallel
depending on the memory used. This method returns the amount of memory taken by \textbf{ONE} precomputation.
\item[set\_data] \textbf{Input: const Rnd\_It \&from}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Rnd\_It \&to}\\
\textbf{return: size\_t}\\
Set the iterators from which we load all the data.
\item [free] \textbf{Input: const uint8\_t sbn)}\\
\textbf{return: void}\\
Each block takes some memory, (a bit more than $symbols * symbol_size$), so once you are done sending source and repair symbols for one block,
you might want to free the memory of that block.
\item[clear\_data]\textbf{return: void}\\
clear all data, without deallocating memory so that things might be slightly faster next time.
\item[blocks()] \textbf{return: uint8\_t} The number of blocks.
\item[ready()]\textbf{return: bool}\\
Is everything decoded?
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
The block size, in bytes. Each block can have different symbols and thus different size.
\item[stop()]\textbf{return void}\\
Stop any current decoding and return error.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
\item[precompute\_sync()]\textbf{return: bool}\\
Start a precomputation so that next computations might be slightly faster. Only return when the precomputation is done or stop() was called.
\item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in a specific block. different blocks can have different symbols.
\item[compute\_sync()]\textbf{return: bool}\\
Compute internal symbols from the input. Only return when the computation is done or stop() was called.
\item[max\_repair] \textbf{Input: const uint8\_t sbn)}\\
\textbf{return: uint32\_t}\\
The maximum amount of repair symbols that you can generate. Something less than $2^{24}$, but the exact number depends on the number of symbols
in a block
\end{description}
\item[precompute()] \textbf{return: std::shared\_future<Error>}\\
Start a precomputation so that next computations might be slightly faster. Returns immediately, but the user can wait or poll on the returned future.
~\\
\item[compute()] \textbf{return: std::shared\_future<Error>}\\
Start a computation on the data. Returns immediately, but the user can wait or poll on the returned future.
\subsubsection{Blocks}
\index{Blocks!C++}
With the \textit{begin()/end()} calls you get \textit{Input iterators} to the blocks. a Block is has the following type:
\begin{lstlisting}[language=C++]
template <typename Rnd_It, typename Fwd_It>
class Block
\end{lstlisting}
\item[encode]\textbf{Input:Fwd\_It \&output}\\
.\ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \textbf{const uint32\_t id}\\
\textbf{return: size\_t}\\
Once the computation is finished, you can get the encoded symbols. The first \texttt{symbols()} are source symbols, and the next are repair symbols. Returns the number of iterators written into the \texttt{output} iterator, which will point to the first non-written element after the symbol.
and exposes the 4 following methods:
\begin{description}
\item[begin\_source]\textbf{return: Symbol\_Iterator}
\item[end\_source]\textbf{return: Symbol\_Iterator}
\item[begin\_repair]\textbf{return: Symbol\_Iterator}
\item[end\_repair]\textbf{Input: const uint32\_t max\_repair}\\
\textbf{return: Symbol\_Iterator}
\item[max\_repair]\textbf{return:uint32\_t}
\item[symbols]\textbf{return: uint16\_t}
\item[block\_size]\textbf{return: uint32\_t}
\end{description}
As the names explain, you will get an iterator to the symbols in the block. As the number of repair symbols can vary, for now you get two separate begin/ends,
so that you can check when you sent the source symbols, and how many repair symbols you send.\\
The other functions are helpers for the details of the block.
\subsubsection{Symbols}
\index{Symbols!C++}
Finally, through the \textit{Symbol\_Iterator} \textit{Input Iterator} we get the \textbf{Symbol} class:
With the \textit{begin\_source()/end\_source()/begin\_repair()/end\_repair(..)} calls you get \textit{input iterators} to the symbols. A symbol has the following type:
\begin{lstlisting}[language=C++]
template <typename Rnd_It, typename Fwd_It>
class Symbol
\end{lstlisting}
which exposes the 2 methods we need to get the symbol data:
and exposes the following methods:
\begin{description}
\item[operator*]\textbf{Input:Fwd\_It \&start, const Fwd\_It end}\\
\textbf{return: uint64\_t}\\
takes a forward iterator, and fill it with the symbol data. returns the number of written iterators.
\item[id()]\textbf{return: uint32\_t}\\
return the id (\textit{$sbn + esi$}) of this symbol, that you need to include in every packet you send, before the symbols.
\item[id()]\textbf{return: uint32\_t}
\item[operator()]\textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
\textbf{return: uint32\_t}\\
Write the symbol into the start-end iterators provided by the user. Afterwards the start iterator will point to the first non-written element after the symbol
\end{description}
\newpage
\subsubsection{The Decoder}
\index{Decoder!C++}
The decoder is a bit simpler than the encoder.
Theere are two constructors for the Decoder:
There are two constructors for the Decoder:
\begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output;
using T_it = typename std::vector<uint32_t>::iterator;
RaptorQ::Decoder<T_it, T_it> dec (
const OTI_Common_Data common,
const OTI_Scheme_Specific_Data scheme)
RaptorQ::Decoder<T_it, T_it> dec (uint64_t size,
uint16_t symbol_size,
uint16_t sub_blocks,
uint8_t blocks)
\end{lstlisting}
Which should be pretty self-explanatory, once you understand how the encoder works.
The remaining methods are:
\begin{description}
\item[decode]\textbf{Input: Fwd\_It \&start, const Fwd\_It end}\\
\textbf{return:uint64\_t}\\
Write \textbf{all} the blocks into the iterator. refuses to write if the input has not been completely received. Return the number of iterators written.
\item[decode]\textbf{Input: Fwd\_It \&start, const Fwd\_It end, const uint8\_t sbn}\\
\textbf{return:uint64\_t}\\
Write a specific block into the iterator. Refuses to write if the input for that block has not been completely received.
\item[add\_symbol]\textbf{In\_It \&start, const In\_It end, const uint32\_t esi, const uint8\_t sbn}\\
\textbf{return: bool}\\
Add one symbol, while explicitly specifying the symbol id and the block id.
\item[add\_symbol]\textbf{In\_It \&start, const In\_It end, const uint32\_t id}\\
\textbf{return: bool}\\
Same as before, but extract the block id and the symbol id from the \textit{id} parameter
\item[free]\textbf{Input: const uint8\_t sbn}\\
\textbf{return: void}\\
You might have stopped using a block, but the memory is still there. free it.
\item[blocks()] \textbf{return: uint8\_t} The number of blocks.
RaptorQ__v1::Decoder<T_it, T_it> dec (
const Block_Size symbols,
const size_t symbol_size,
const Report type);
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
The block size, in bytes. Each block can have different symbols and thus different size.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
\item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in a specific block. different blocks can have different symbols.
\end{description}
\newpage
\subsection{C interface}\index{Interface!C}
The C interface looks a lot like the C++ one.\\
You need to include the \textbf{cRaptorQ.h} header, and link the \textit{libRaptorQ} library.\\
\marginlabel{Static linking}\index{Static Linking}
If you are working with the static version of libRaptorQ remember to link the C++ standard library used when
compiling the library (\textit{libstdc++} for gcc or maybe \textit{libc++} for clang), your threding library (usually \textit{libpthread}), and the C math library (\textit{libm}).\\
First of all, you need to build the encoder or the decoder.
The C interface is just a wrapper around the C++ code, so you still have to specify the same things as before.
A quick glance at the constructors should give you all the information you need:
\marginlabel{C Constructors}
\begin{lstlisting}[language=C]
typedef enum { NONE = 0,
ENC_8 = 1, ENC_16 = 2, ENC_32 = 3, ENC_64 = 4,
DEC_8 = 5, DEC_16 = 6, DEC_32 = 7, DEC_64 = 8}
RaptorQ_type;
struct RAPTORQ_LOCAL RaptorQ_ptr
{
void *ptr = nullptr;
const RaptorQ_type type;
};
RaptorQ_ptr* RaptorQ_Enc (const RaptorQ_type type,
void *data,
const uint64_t size,
const uint16_t min_subsymbol_size,
const uint16_t symbol_size,
const size_t max_memory);
RaptorQ_ptr* RaptorQ_Dec (const RaptorQ_type type,
const RaptorQ_OTI_Common_Data common,
const RaptorQ_OTI_Scheme_Specific_Data
scheme);
\end{lstlisting}
The encoder and decoder must have a specific alignment. in C++ you can also have different
alignments for the input and output, while in C things are a bit more strict as we have to enumerate
all the possible cases. So you only get the same data alignment for both input and output.\\
Still, you don't lose anything in performance.\\
\newpage
\subsubsection{Common functions for (de/en)coding}
These functions are used by both the decoder and the decoder, and will be helpful in tracking how much memory
you will need to allocate, or in general in managing the encoder and decoder.
The only parameter that is not self-explanatory is the Report type: it regulates when the decoder will tell you that there is available data and will let you extract it.\\
The possible values are:
The \textbf{ptr} must be a valid encoder or decoder.
The names for now are self-explanatory. Blocks can have different symbols, so the size of a block and the number of symbols in a block
depend on which block we are talking about.
\begin{itemize}
\item \textbf{Report::PARTIAL\_FROM\_BEGINNING} report the data as it become available, but the data will be sequential
\item \textbf{Report::PARTIAL\_ANY} report any symbol as it becomes available. Could be out-of-order, and you can get symbols before the decoding starts.
\item \textbf{Report::COMPLETE} wait until the whole block has been decoded.
\end{itemize}
\marginlabel{symbols, blocks, memory}
\begin{lstlisting}[language=C]
uint16_t RaptorQ_symbol_size (struct RaptorQ_ptr *ptr);
uint8_t RaptorQ_blocks (struct RaptorQ_ptr *ptr);
uint32_t RaptorQ_block_size (struct RaptorQ_ptr *ptr,
const uint8_t sbn);
uint16_t RaptorQ_symbols (struct RaptorQ_ptr *ptr,
const uint8_t sbn);
void RaptorQ_free (struct RaptorQ_ptr **ptr);
void RaptorQ_free_block (struct RaptorQ_ptr *ptr,
const uint8_t sbn);
\end{lstlisting}
Finally, when you are done working with a block, you can free the memory associated with the single block
and just free the whole (en/de)coder when you are done. Freeing the whole (en/de)coder will obviously free
also all the blocks.
The decoding methods are:
\newpage
\subsubsection{Encoding}
\marginlabel{OTI Data}
First, we need to tell the receiver all the parameters that the encoder is using, and for that two functions are provided:
\begin{description}
\item[symbols()]\textbf{return: uint16\_t}\\
The number of symbols in this block
\item[symbol\_size()]\textbf{return: size\_t}\\
The bytes in each symbol
\item[can\_decode()]\textbf{return: bool}\\
Whether we have enough data to start decoding or not
\item[ready()]\textbf{return: bool}\\
Whether everything has been decoded or not.
\item[needed\_symbols()]\textbf{uint16\_t}\\
How many more symbols we need before we try to decode again. If this is 0, you can decode.
\item[clear\_data()]\textbf{return: void}\\
clear all data, without deallocating memory so that things might be slightly faster next time.
\item[stop()]\textbf{return: void}\\
Stop any decoding done in the background.
\item[begin()] \textbf{return: Symbol\_Iterator<In\_It, Fwd\_It>}\\
This returns an iterator to the symbols in this block.
\item[end()] \textbf{return: const Symbol\_Iterator<In\_It, Fwd\_It>}\\
This returns an iterator to the end of the symbols in this block
\item[add\_symbols] \textbf{Input: In\_it \&from}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const In\_it \&to}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
\textbf{return: RaptorQ\_\_v1::Error}\\
Add one symbol to this block. If the symbol is less than \texttt{symbol\_size()} the symbol will be padded with zeros
\item[end\_of\_input] \textbf{Input: const Fill\_With\_Zeros fill}\\
\textbf{return: std::vector<bool>}\\
Tell the decoder that we know that there will be no more data for this block.
The first parameter is an \texttt{enum}, either \textbf{Fill\_With\_Zeros::YES} or \textbf{Fill\_With\_Zeros::NO}
If you choose \texttt{YES} then the repair symbols will be discarded, the missing symbols will be filled with zeros
and for output you will get the bitmask for which byte was received (true) or was filled with zeros (false).\\
If you choose \textbf{Fill\_With\_Zeros::NO} you will get \textit{an empty vector}, and the decoding will end with error if it was waiting for more input.
\item[set\_max\_concurrency] \textbf{Input: const uint16\_t max\_threads}\\
\textbf{return: void}\\
The decoder can try to decode the same block multiple times, while more repair symbols arrive.
Unless you really need it, leave the default, which is $1$, so no decoding concurrency or the same block.
\begin{lstlisting}[language=C]
typedef uint64_t RaptorQ_OTI_Common_Data;
typedef uint32_t RaptorQ_OTI_Scheme_Specific_Data;
\item[decode\_once()] \textbf{return: RaptorQ\_\_v1::Decoder\_Result}\\
Try to decode the block, only return once the decoding is finished, do not try again even if more repair symbols arrived.
RaptorQ_OTI_Common_Data RaptorQ_OTI_Common (
struct RaptorQ_ptr *enc);
RaptorQ_OTI_Scheme_Specific_Data RaptorQ_OTI_Scheme (
struct RaptorQ_ptr *enc);
\item[poll()] \textbf{return: RaptorQ\_\_v1::Decoder\_wait\_res}\\
\begin{lstlisting}[language=c++]
struct Decoder_wait_res {
Error error;
uint16_t symbol;
};
\end{lstlisting}
\index{Encoder!C}\marginlabel{Encoding}
\begin{lstlisting}[language=C]
// maximum number of repair symbol in a block
uint32_t RaptorQ_max_repair (RaptorQ_ptr *enc,
const uint8_t sbn);
//estimate bytes of ram used in the precomputation
// of one block
size_t RaptorQ_precompute_max_memory (
struct RaptorQ_ptr *enc);
// do the precomputation.
void RaptorQ_precompute (struct RaptorQ_ptr *enc,
const uint8_t threads,
const bool background);
// encode one symbol. source block number and
// symbol id are in the ``id'' field.
// returns number of alignments written.
uint64_t RaptorQ_encode_id (struct RaptorQ_ptr *enc,
void **data,
const uint64_t size,
const uint32_t id);
// encode one symbol. same as before
uint64_t RaptorQ_encode (struct RaptorQ_ptr *enc,
void **data,
const uint64_t size,
const uint32_t esi,
const uint8_t sbn);
// build an ``id'' field out of an esi and sbn field.
uint32_t RaptorQ_id (const uint32_t esi,
const uint8_t sbn);
In general, the \texttt{error} can be \textbf{Error::WORKING}, \texttt{Error::NEED\_DATA}, \texttt{Error::NONE}. The last one signals that the block (or symbol, see later) has been decoded.
The \texttt{symbol} parameter changes depending on the \textbf{RaptorQ\_\_v1::Report} type that you set when you constructed the decoder:
\begin{itemize}
\item \textbf{Report::PARTIAL\_FROM\_BEGINNING}: \texttt{symbols} will be the number of symbols available, but counting only the sequential ones starting from the beginning
\item \textbf{Report::PARTIAL\_ANY}: \texttt{symbols} will be the first symbol available (if \texttt{error} == \texttt{Error::NONE}). If the block has been decoded \texttt{symbols} == \texttt{decoder.symbols()}
\item \textbf{Report::COMPLETE} \texttt{symbols} is not used.
\end{itemize}
\item[wait\_sync()] \textbf{return: RaptorQ\_\_v1::Decoder\_wait\_res}\\
Start the decoding, and keep trying or waiting for new data to arrive until either we successfully decode the block or \texttt{stop()} is called. Also stops waiting for data after \texttt{end\_of\_input} is called (but will keep retrying if there already is enough).\\
The result is the same as above
\item[wait()] \textbf{return: std::future<RaptorQ\_\_v1::Decoder\_wait\_res>}\\
Same as before, but returns a future immediately. This call is enabled only if you are compiling with C++11 or later
\item[decode\_symbol()] \textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t esi} \\
\textbf{return: RaptorQ\_\_v1::Error}\\
Decode a single symbol. If $end-start$ is less then the symbol size, then you will get a partial symbol.
\textbf{return: RaptorQ\_\_v1::Error}\\
\item[decode\_bytes] \textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t from\_byte}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t skip}\\
\textbf{return: Decoder\_written} \\
Decode a specific range of the block. The \texttt{skip} parameter is the offset (in bytes) to skip from the \texttt{size} iterator. This is done to allow to work with iterators whose data is bigger than 1 byte. Although we recommend to work with uint8\_t data.\\
Returns the number of bytes written, offset of the data in the last iterator (0 if it is aligned to the iterator)
\begin{lstlisting}[language=c++]
struct RAPTORQ_API Decoder_written {
size_t written;
size_t offset;
};
\end{lstlisting}
As for the C++ version, everything is thread-safe.
You can start the precomputation in background and not worry about it.\\
If you request repair symbols before the computation is finished, the call will block until the data is available.
The \textbf{encode} functions are the same, and will encode \textbf{one} symbol.
They work for both source symbols and repair symbols, just keep increasing the \textit{esi} field
\end{description}
\newpage
\subsubsection{Decoding}
\index{Decoder!C}\marginlabel{Decoding}
\begin{lstlisting}[language=C]
// return the total size of the data that will be
// decoded
uint64_t RaptorQ_bytes (struct RaptorQ_ptr *dec);
// decode all blocks.
// returns the number of written alignments.
// returns 0 if decoding of *everything* is not possible
uint64_t RaptorQ_decode (struct RaptorQ_ptr *dec,
void **data,
const size_t size);
// decode only one block (if possible)
// returns 0 if decoding of the block is not possible.
uint64_t RaptorQ_decode_block (struct RaptorQ_ptr *dec,
void **data,
const size_t size,
const uint8_t sbn);
// add a received symbol to the structure.
// either by using the `id' field
bool RaptorQ_add_symbol_id (struct RaptorQ_ptr *dec,
void **data,
const uint32_t size,
const uint32_t id);
// or by explicitely declaring esi and sbn
bool RaptorQ_add_symbol (struct RaptorQ_ptr *dec,
void **data,
const uint32_t size,
const uint32_t esi,
const uint8_t sbn);
\end{lstlisting}
......
examples/example_cpp_raw.cpp 0 → 100644
View file @ a394b22a
/*
* Copyright (c) 2015-2018, Luca Fulchir<luker@fenrirproject.org>,
* All rights reserved.
*
* This file is part of "libRaptorQ".
*
* libRaptorQ is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3
* of the License, or (at your option) any later version.
*
* libRaptorQ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* and a copy of the GNU Lesser General Public License
* along with libRaptorQ. If not, see <http://www.gnu.org/licenses/>.
*/
#include "../src/RaptorQ/RaptorQ_v1_hdr.hpp"
#include <algorithm>
#include <fstream>
#include <iostream>
#include <limits>
#include <random>
#include <stdlib.h>
#include <vector>
// Demonstration of how to use the C++ RAW interface
// it's pretty simple, we generate some input,
// then encode, drop some packets (source and repair)
// and finally decode everything.
// rename the main namespace for ease of use
namespace RaptorQ = RaptorQ__v1;
// mysize is bytes.
bool test_rq (const uint32_t mysize, std::mt19937_64 &rnd,
float drop_probability,
const uint8_t overhead);
// the "overhead" variable tells us how many symbols more than the
// minimum we will generate. RaptorQ can not always decode a block,
// but there is a small probability that it will fail.
// More overhead => less probability of failure
// overhead 0 => 1% failures
// overhead 1 => 0.01% failures
// overhead 2 => 0.0001% failures
// etc... as you can see, it make little sense to work with more than 3-4
// overhead symbols, but at least one should be considered
bool test_rq (const uint32_t mysize, std::mt19937_64 &rnd,
float drop_probability,
const uint8_t overhead)
{
// the actual input.
std::vector<uint8_t> input;
input.reserve (mysize);
// initialize vector with random data
// distr should be "uint8_t". But visual studio does not like it, so
// we use uint16_t
// generate a random distribution between all values of uint8_t
std::uniform_int_distribution<int16_t> distr (0,
std::numeric_limits<uint8_t>::max());
// fill our input with random data
for (size_t idx = 0; idx < mysize; ++idx) {
input.push_back (static_cast<uint8_t> (distr(rnd)));
}
// the input will be divided in blocks of 4 bytes.
// it's a bit low, but this is just an example.
// NOTE: the symbol size must be a multiple of the container size.
// since sizeof(uint8_t) == 1 and 4 is a multiple of 1, we are safe.
const uint16_t symbol_size = 4; // bytes
// how many symbols do we need to encode all our input in a single block?
auto min_symbols = (input.size() * sizeof(uint8_t)) / symbol_size;
if ((input.size() * sizeof(uint8_t)) % symbol_size != 0)
++min_symbols;
// convert "symbols" to a typesafe equivalent, RaptorQ::Block_Size
// This is needed becouse not all numbers are valid block sizes, and this
// helps you choose the right block size
RaptorQ::Block_Size block = RaptorQ::Block_Size::Block_10;
for (auto blk : *RaptorQ::blocks) {
// RaptorQ::blocks is a pointer to an array, just scan it to find your
// block.
if (static_cast<uint16_t> (blk) >= min_symbols) {
block = blk;
break;
}
}
// now initialize the encoder.
// the input for the encoder is std::vector<uint8_t>
// the output for the encoder is std::vector<uint8_t>
// yes, you can have different types, but most of the time you will
// want to work with uint8_t
RaptorQ::Encoder<typename std::vector<uint8_t>::iterator,
typename std::vector<uint8_t>::iterator> enc (
block, symbol_size);
// give the input to the encoder. the encoder answers with the size of what
// it can use
if (enc.set_data (input.begin(), input.end()) != mysize) {
std::cout << "Could not give data to the encoder :(\n";
return false;
}
// actual symbols. you could just use static_cast<uint16_t> (blok)
// but this way you can actually query the encoder.
uint16_t _symbols = enc.symbols();
// print some stuff in output
std::cout << "Size: " << mysize << " symbols: " <<
static_cast<uint32_t> (_symbols) <<
" symbol size: " <<
static_cast<int32_t>(enc.symbol_size()) << "\n";
// RQ need to do its magic on the input before you can ask the symbols.
// multiple ways to do this are available.
// The simplest is to run the computation and block everything until
// the work has been done. Not a problem for small sizes (<200),
// but big sizes will take **a lot** of time, consider running this with the
// asynchronous calls
if (!enc.compute_sync()) {
// if this happens it's a bug in the library.
// the **Decoder** can fail, but the **Encoder** can never fail.
std::cout << "Enc-RaptorQ failure! really bad!\n";
return false;
}
// the probability that a symbol will be dropped.
if (drop_probability > static_cast<float> (90.0))
drop_probability = 90.0; // this is still too high probably.
// we will store here all encoded and transmitted symbols
// std::pair<symbol id (esi), symbol data>
using symbol_id = uint32_t; // just a better name
std::vector<std::pair<symbol_id, std::vector<uint8_t>>> received;
{
// in this block we will generate the symbols that will be sent to
// the decoder.
// a block of size X will need at least X symbols to be decoded.
// we will randomly drop some symbols, but we will keep generating
// repari symbols until we have the required number of symbols.
std::uniform_real_distribution<float> drop_rnd (0.0, 100.0);
uint32_t received_tot = 0;
// Now get the source symbols.
// source symbols are specials because they contain the input data
// as-is, so if you get all of these, you don't need repair symbols
// to make sure that we are using the decoder, drop the first
// source symbol.
auto source_sym_it = enc.begin_source();
++source_sym_it; // ignore the first soure symbol (=> drop it)
source_sym_it++;
for (; source_sym_it != enc.end_source(); ++source_sym_it) {
// we save the symbol here:
// make sure the vector has enough space for the symbol:
// fill it with zeros for the size of the symbol
std::vector<uint8_t> source_sym_data (symbol_size, 0);
// save the data of the symbol into our vector
auto it = source_sym_data.begin();
auto written = (*source_sym_it) (it, source_sym_data.end());
if (written != symbol_size) {
// this can only happen if "source_sym_data" did not have
// enough space for a symbol (here: never)
std::cout << written << "-vs-" << symbol_size <<
" Could not get the whole source symbol!\n";
return false;
}
// can we keep this symbol or do we randomly drop it?
float dropped = drop_rnd (rnd);
if (dropped <= drop_probability) {
continue; // start the cycle again
}
// good, the symbol was received.
++received_tot;
// add it to the vector of received symbols
symbol_id tmp_id = (*source_sym_it).id();
received.emplace_back (tmp_id, std::move(source_sym_data));
}
std::cout << "Source Packet lost: " << enc.symbols() - received.size()
<< "\n";
//--------------------------------------------
// we finished working with the source symbols.
// now we need to transmit the repair symbols.
auto repair_sym_it = enc.begin_repair();
auto max_repair = enc.max_repair(); // RaptorQ can theoretically handle
// infinite repair symbols
// but computers are not so infinite
// we need to have at least enc.symbols() + overhead symbols.
for (; received.size() < (enc.symbols() + overhead) &&
repair_sym_it != enc.end_repair (max_repair);
++repair_sym_it) {
// we save the symbol here:
// make sure the vector has enough space for the symbol:
// fill it with zeros for the size of the symbol
std::vector<uint8_t> repair_sym_data (symbol_size, 0);
// save the data of the symbol into our vector
auto it = repair_sym_data.begin();
auto written = (*repair_sym_it) (it, repair_sym_data.end());
if (written != symbol_size) {
// this can only happen if "repair_sym_data" did not have
// enough space for a symbol (here: never)
std::cout << written << "-vs-" << symbol_size <<
" Could not get the whole repair symbol!\n";
return false;
}
// can we keep this symbol or do we randomly drop it?
float dropped = drop_rnd (rnd);
if (dropped <= drop_probability) {
continue; // start the cycle again
}
// good, the symbol was received.
++received_tot;
// add it to the vector of received symbols
symbol_id tmp_id = (*repair_sym_it).id();
received.emplace_back (tmp_id, std::move(repair_sym_data));
}
if (repair_sym_it == enc.end_repair (enc.max_repair())) {
// we dropped waaaay too many symbols!
// should never happen in real life. it means that we do not
// have enough repair symbols.
// at this point you can actually start to retransmit the
// repair symbols from enc.begin_repair(), but we don't care in
// this example
std::cout << "Maybe losing " << drop_probability << "% is too much?\n";
return false;
}
}
// Now we all the source and repair symbols are in "received".
// we will use those to start decoding:
// define "Decoder_type" to write less afterwards
using Decoder_type = RaptorQ::Decoder<
typename std::vector<uint8_t>::iterator,
typename std::vector<uint8_t>::iterator>;
Decoder_type dec (block, symbol_size, Decoder_type::Report::COMPLETE);
// "Decoder_type::Report::COMPLETE" means that the decoder will not
// give us any output until we have decoded all the data.
// there are modes to extract the data symbol by symbol in an ordered
// an unordered fashion, but let's keep this simple.
// we will store the output of the decoder here:
// note: the output need to have at least "mysize" bytes, and
// we fill it with zeros
std::vector<uint8_t> output (mysize, 0);
// now push every received symbol into the decoder
for (auto &rec_sym : received) {
// as a reminder:
// rec_sym.first = symbol_id (uint32_t)
// rec_sym.second = std::vector<uint8_t> symbol_data
symbol_id tmp_id = rec_sym.first;
auto it = rec_sym.second.begin();
auto err = dec.add_symbol (it, rec_sym.second.end(), tmp_id);
if (err != RaptorQ::Error::NONE && err != RaptorQ::Error::NOT_NEEDED) {
// When you add a symbol, you can get:
// NONE: no error
// NOT_NEEDED: libRaptorQ ignored it because everything is
// already decoded
// INITIALIZATION: wrong parameters to the decoder contructor
// WRONG_INPUT: not enough data on the symbol?
// some_other_error: errors in the library
std::cout << "error adding?\n";
return false;
}
}
// by now we now there will be no more input, so we tell this to the
// decoder. You can skip this call, but if the decoder does not have
// enough data it sill wait forever (or until you call .stop())
dec.end_of_input (RaptorQ::Fill_With_Zeros::NO);
// optional if you want partial decoding without using the repair
// symbols
// std::vector<bool> symbols_bitmask = dec.end_of_input (
// RaptorQ::Fill_With_Zeros::YES);
// decode, and do not return until the computation is finished.
auto res = dec.wait_sync();
if (res.error != RaptorQ::Error::NONE) {
std::cout << "Couldn't decode.\n";
return false;
}
// now save the decoded data in our output
size_t decode_from_byte = 0;
size_t skip_bytes_at_begining_of_output = 0;
auto out_it = output.begin();
auto decoded = dec.decode_bytes (out_it, output.end(), decode_from_byte,
skip_bytes_at_begining_of_output);
// "decode_from_byte" can be used to have only a part of the output.
// it can be used in advanced setups where you ask only a part
// of the block at a time.
// "skip_bytes_at_begining_of_output" is used when dealing with containers
// which size does not align with the output. For really advanced usage only
// Both should be zero for most setups.
if (decoded.written != mysize) {
if (decoded.written == 0) {
// we were really unlucky and the RQ algorithm needed
// more symbols!
std::cout << "Couldn't decode, RaptorQ Algorithm failure. "
"Can't Retry.\n";
} else {
// probably a library error
std::cout << "Partial Decoding? This should not have happened: " <<
decoded.written << " vs " << mysize << "\n";
}
return false;
} else {
std::cout << "Decoded: " << mysize << "\n";
}
// byte-wise check: did we actually decode everything the right way?
for (uint64_t i = 0; i < mysize; ++i) {
if (input[i] != output[i]) {
// this is a bug in the library, please report
std::cout << "The output does not correspond to the input!\n";
return false;
}
}
return true;
}
int main (void)
{
// get a random number generator
std::mt19937_64 rnd;
std::ifstream rand("/dev/urandom");
uint64_t seed = 0;
rand.read (reinterpret_cast<char *> (&seed), sizeof(seed));
rand.close ();
rnd.seed (seed);
// keep some computation in memory. If you use only one block size it
// will make things faster on bigger block size.
// allocate 5Mb
RaptorQ__v1::local_cache_size (5000000);
// for our test, we use an input of random size, between 100 and 10.000
// bytes.
std::uniform_int_distribution<uint32_t> distr(100, 10000);
uint32_t input_size = distr (rnd);
if (!test_rq (input_size, rnd, 20.0, 4))
return -1;
std::cout << "The example completed successfully\n";
return 0;
}