Skip to content
Commits on Source (86)
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ a394b22a
doc/libRaptorQ.aux /doc/*.aux
doc/libRaptorQ.bbl /doc/*.bbl
doc/libRaptorQ.blg /doc/*.blg
doc/libRaptorQ.idx /doc/*.idx
doc/libRaptorQ.ilg /doc/*.ilg
doc/libRaptorQ.ind /doc/*.ind
doc/libRaptorQ.log /doc/*.log
doc/libRaptorQ.out /doc/*.out
doc/libRaptorQ.pdf /doc/*.pdf
doc/libRaptorQ.synctex.gz /doc/*.synctex.gz
doc/libRaptorQ.toc /doc/*.toc
CMakeFiles/ /CMakeFiles/
CMakeCache.txt /CMakeCache.txt
CMakeLists.txt.user /CMakeLists.txt.user
cmake_install.cmake /cmake_install.cmake
Makefile /Makefile
*~ *~
build* /build*
.gitlab-ci.yml
View file @ a394b22a
...@@ -5,12 +5,12 @@ stages: ...@@ -5,12 +5,12 @@ stages:
job1: job1:
stage: makedoc stage: makedoc
script: mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=Release ../; make docs; cp doc/libRaptorQ.pdf ~/ script: mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=Release ../; make docs; cp doc/libRaptorQ-RFC6330.pdf ~/; cp doc/libRaptorQ-RaptorQ.pdf ~/
job2: job2:
stage: deploydoc stage: deploydoc
script: git diff-tree --no-commit-id --name-only -r HEAD | grep ^doc/ || exit 0; cd /var/lib/gitlab-runner/libRaptorQ.wiki; git pull; cp ~/libRaptorQ.pdf .; git add libRaptorQ.pdf; git commit -m '(gitlab-runner) updated pdf'; git push; script: git diff-tree --no-commit-id --name-only -r HEAD | grep ^doc/ || exit 0; cd /var/lib/gitlab-runner/libRaptorQ.wiki; git pull; cp ~/libRaptorQ-RFC6330.pdf .; cp ~/libRaptorQ-RaptorQ.pdf .; git add libRaptorQ-RFC6330.pdf; git add libRaptorQ-RaptorQ.pdf; git commit -m '(gitlab-runner) updated pdf'; git push;
job3: job3:
stage: build stage: build
......
CHANGELOG
View file @ a394b22a
v1.0.0 (prealpha): v1.0.0 (beta):
* C++98 compatibility (untested) * C++98 compatibility (untested)
* we are now a header-only library that can be compiled and linked to C/C++98, with RFC and RAW API * we are now a header-only library that can be compiled and linked to C/C++98, with RFC and RAW API
* RAW API block size is now typesafe. * RAW API block size is now typesafe.
* RAW API does not the interleaving code. * RAW API does not use the interleaving code.
* API reworked to easy manage old/new API in the future * API reworked to easy manage old/new API in the future
* new, simpler RAW API (single-block) * new, simpler RAW API (single-block)
* computations can be stopped * computations can be stopped
* precomputation caching * precomputation caching
* cached result can be compressed (raw/LZ4) * cached result can be compressed (raw/LZ4)
* RFC API: global multithreading instead of per-class * RFC API: global multithreading instead of per-class
* RFC compliance
* CLI tool for raw API * CLI tool for raw API
* OSX/BSD support * OSX/BSD support
v0.1.10 (unreleased): v0.1.10 (unreleased):
* backported MAC/FreeBSD support (and incomplete windows) * backported MAC/FreeBSD support (and incomplete windows)
* backported RFC compliance
v0.1.9: v0.1.9:
* remember to bump up version in the cmake -.-' * remember to bump up version in the cmake -.-'
* squashed warnings for gcc & clang in 32 bit mode. * squashed warnings for gcc & clang in 32 bit mode.
......
CMakeLists.txt
View file @ a394b22a
...@@ -23,13 +23,13 @@ enable_language(CXX) ...@@ -23,13 +23,13 @@ enable_language(CXX)
enable_language(C) enable_language(C)
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake; ${CMAKE_MODULE_PATH}) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake; ${CMAKE_MODULE_PATH})
set(RQ_VERSION 1.0.0-prealpha) set(RQ_VERSION 1.0.0-rc2)
set(RQ_ABI 1) set(RQ_ABI 1)
message(STATUS "libRaptorQ version ${RQ_VERSION}") message(STATUS "libRaptorQ version ${RQ_VERSION}")
add_definitions(-DRQ_VERSION="${RQ_VERSION}") add_definitions(-DRQ_VERSION="${RQ_VERSION}")
add_definitions(-DEIGEN_MPL2_ONLY)
# default values if (NOT CMAKE_BUILD_TYPE)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release") set(CMAKE_BUILD_TYPE "Release")
endif() endif()
...@@ -39,12 +39,41 @@ include(GNUInstallDirs) ...@@ -39,12 +39,41 @@ include(GNUInstallDirs)
option(DYNAMIC_LIB "Build dynamic library" ON) option(DYNAMIC_LIB "Build dynamic library" ON)
option(STATIC_LIB "Build static library" ON) option(STATIC_LIB "Build static library" ON)
option(CLANG_STDLIB "Use clang's libc++" OFF) option(CLANG_STDLIB "Use clang's libc++" OFF)
option(USE_LZ4 "Use LZ4 compression for result caching" ON)
option(CLI "BUild CLI tools" ON) option(CLI "BUild CLI tools" ON)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build Type")
set(USE_LZ4 "ON" CACHE STRING "Use LZ4 compression for result caching")
set(RQ_LINKER CACHE STRING "linker to use (auto/gold/ld/bsd)") set(RQ_LINKER CACHE STRING "linker to use (auto/gold/ld/bsd)")
set(RQ_ENDIANNESS CACHE STRING "endianness of your system")
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS Debug Release RelWithDebInfo MinSizeRel)
set_property(CACHE USE_LZ4 PROPERTY STRINGS ON BUNDLED OFF)
set_property(CACHE RQ_LINKER PROPERTY STRINGS GOLD LD BSD) set_property(CACHE RQ_LINKER PROPERTY STRINGS GOLD LD BSD)
mark_as_advanced(FORCE RQ_LINKER) set_property(CACHE RQ_ENDIANNESS PROPERTY STRINGS Auto BigEndian LittleEndian)
mark_as_advanced(FORCE RQ_LINKER RQ_ENDIANNESS RQ_GOLD RQ_LZ4_LIB)
if (USE_LZ4 MATCHES "BUNDLED")
set(USE_LZ4 "ON")
set(RQ_LZ4_USE_OWN TRUE)
endif()
if (NOT RQ_ENDIANNESS)
set(RQ_ENDIANNESS Auto)
endif()
if (RQ_ENDIANNESS MATCHES "Auto")
include(TestBigEndian)
TEST_BIG_ENDIAN(IS_BIG_ENDIAN)
if (IS_BIG_ENDIAN)
add_definitions(-DRQ_BIG_ENDIAN)
message (STATUS "Detected big endian machine")
else()
add_definitions(-DRQ_LITTLE_ENDIAN)
message (STATUS "Detected little endian machine")
endif()
elseif (RQ_ENDIANNESS MATCHES "BigEndian")
add_definitions(-DRQ_BIG_ENDIAN)
message (STATUS "Forced to big endian")
elseif (RQ_ENDIANNESS MATCHES "LittleEndian")
add_definitions(-DRQ_LITTLE_ENDIAN)
message (STATUS "Forced to little endian")
endif()
# supported linkers: gold, linux standard, bsd # supported linkers: gold, linux standard, bsd
if(NOT RQ_LINKER) if(NOT RQ_LINKER)
...@@ -70,7 +99,7 @@ else() ...@@ -70,7 +99,7 @@ else()
set(RQ_LINKER BSD) set(RQ_LINKER BSD)
endif() endif()
# defaults: only enable LTO/PROFILING with known compielrs # defaults: only enable LTO/PROFILING with known compilers
# exception: osx/BSD. llvm linker/profiling not default? # exception: osx/BSD. llvm linker/profiling not default?
if ((CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") AND if ((CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") AND
(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") AND (NOT ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")) (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") AND (NOT ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD"))
...@@ -158,10 +187,11 @@ endif() ...@@ -158,10 +187,11 @@ endif()
find_package(Threads REQUIRED) find_package(Threads REQUIRED)
find_package(git) find_package(git)
find_package(eigen REQUIRED) find_package(eigen REQUIRED)
include_directories(SYSTEM ${EIGEN3_INCLUDE_DIR}) include_directories(SYSTEM PUBLIC ${EIGEN3_INCLUDE_DIR})
include_directories(SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/external/optionparser-1.4")
if (USE_LZ4 MATCHES "ON") if (USE_LZ4 MATCHES "ON")
find_package(RQ_LZ4 REQUIRED) find_package(RQ_LZ4 REQUIRED)
include_directories(SYSTEM ${RQ_LZ4_INCLUDE_DIR}) include_directories(SYSTEM PUBLIC ${RQ_LZ4_INCLUDE_DIR})
add_definitions(-DRQ_USE_LZ4) add_definitions(-DRQ_USE_LZ4)
endif() endif()
#lz4 build if necessary #lz4 build if necessary
...@@ -192,8 +222,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows") ...@@ -192,8 +222,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows")
message(WARNING, "RaptorQ builds are not deterministic in Windows yet") message(WARNING, "RaptorQ builds are not deterministic in Windows yet")
else() else()
add_definitions(-DRQ_UNIX) add_definitions(-DRQ_UNIX)
# used by ftok to get a key. Track both the file and the version
add_definitions(-DRQ_SHMPATH="${CMAKE_INSTALL_LIBDIR}/libRaptorQ.so-${RQ_VERSION}")
endif() endif()
...@@ -202,11 +230,13 @@ SET(SOURCES ...@@ -202,11 +230,13 @@ SET(SOURCES
src/RaptorQ/v1/wrapper/C_RFC_API.cpp src/RaptorQ/v1/wrapper/C_RFC_API.cpp
src/RaptorQ/v1/wrapper/CPP_RAW_API_void.cpp src/RaptorQ/v1/wrapper/CPP_RAW_API_void.cpp
src/RaptorQ/v1/wrapper/CPP_RFC_API_void.cpp src/RaptorQ/v1/wrapper/CPP_RFC_API_void.cpp
src/RaptorQ/v1/wrapper/CPP_caches.cpp
) )
SET(HEADERS SET(HEADERS
src/RaptorQ/v1/block_sizes.hpp src/RaptorQ/v1/block_sizes.hpp
src/RaptorQ/v1/caches.hpp src/RaptorQ/v1/caches.hpp
src/RaptorQ/v1/caches.ipp
src/RaptorQ/v1/common.hpp src/RaptorQ/v1/common.hpp
src/RaptorQ/v1/De_Interleaver.hpp src/RaptorQ/v1/De_Interleaver.hpp
src/RaptorQ/v1/Decoder.hpp src/RaptorQ/v1/Decoder.hpp
...@@ -230,7 +260,11 @@ SET(HEADERS ...@@ -230,7 +260,11 @@ SET(HEADERS
src/RaptorQ/v1/Thread_Pool.hpp src/RaptorQ/v1/Thread_Pool.hpp
src/RaptorQ/v1/util/Bitmask.hpp src/RaptorQ/v1/util/Bitmask.hpp
src/RaptorQ/v1/util/div.hpp src/RaptorQ/v1/util/div.hpp
src/RaptorQ/v1/util/endianess.hpp
src/RaptorQ/v1/util/Graph.hpp src/RaptorQ/v1/util/Graph.hpp
)
SET(HEADERS_LINKED
src/RaptorQ/RaptorQ.h src/RaptorQ/RaptorQ.h
src/RaptorQ/RaptorQ_v1.hpp src/RaptorQ/RaptorQ_v1.hpp
src/RaptorQ/RFC6330.h src/RaptorQ/RFC6330.h
...@@ -244,6 +278,17 @@ SET(HEADERS ...@@ -244,6 +278,17 @@ SET(HEADERS
src/RaptorQ/v1/wrapper/CPP_RAW_API_void.hpp src/RaptorQ/v1/wrapper/CPP_RAW_API_void.hpp
) )
SET(HEADERS_ONLY
src/RaptorQ/RaptorQ_v1_hdr.hpp
src/RaptorQ/RFC6330_v1_hdr.hpp
)
if(CMAKE_SYSTEM_NAME MATCHES "Windows")
# windows-only placeholder
else()
# linux-only placeholder
endif()
if (USE_LZ4 MATCHES "ON") if (USE_LZ4 MATCHES "ON")
SET (HEADERS ${HEADERS} SET (HEADERS ${HEADERS}
src/RaptorQ/v1/Shared_Computation/LZ4_Wrapper.hpp) src/RaptorQ/v1/Shared_Computation/LZ4_Wrapper.hpp)
...@@ -296,7 +341,7 @@ CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/API_Wrapper_C.cpp.gcda) ...@@ -296,7 +341,7 @@ CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/API_Wrapper_C.cpp.gcda)
endif() endif()
# PRE-run: build a library, generate the profile. # PRE-run: build a library, generate the profile.
add_library(RaptorQ_Static_Profiling STATIC ${SOURCES} ${HEADERS}) add_library(RaptorQ_Static_Profiling STATIC ${SOURCES} ${HEADERS} ${HEADERS_LINKED})
target_link_libraries(RaptorQ_Static_Profiling ${STDLIB} ${RQ_LZ4_DEP} ${RQ_UBSAN}) target_link_libraries(RaptorQ_Static_Profiling ${STDLIB} ${RQ_LZ4_DEP} ${RQ_UBSAN})
add_dependencies(RaptorQ_Static_Profiling LZ4) add_dependencies(RaptorQ_Static_Profiling LZ4)
target_compile_options( target_compile_options(
...@@ -315,9 +360,9 @@ CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/API_Wrapper_C.cpp.gcda) ...@@ -315,9 +360,9 @@ CMakeFiles/RaptorQ_Static_Profiling.dir/src/RaptorQ/v1/API_Wrapper_C.cpp.gcda)
${PROFILE_GET} ${PROFILE_GET}
) )
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
target_link_libraries(test_c_profiled RaptorQ_Static_Profiling ${STDLIB} m ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(test_c_profiled RaptorQ_Static_Profiling ${STDLIB} m ${CMAKE_THREAD_LIBS_INIT} ${RQ_UBSAN})
else() else()
target_link_libraries(test_c_profiled RaptorQ_Static_Profiling ${STDLIB} ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(test_c_profiled RaptorQ_Static_Profiling ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_UBSAN})
endif() endif()
set_target_properties(test_c_profiled PROPERTIES LINK_FLAGS ${PROFILE_GET}) set_target_properties(test_c_profiled PROPERTIES LINK_FLAGS ${PROFILE_GET})
...@@ -358,7 +403,7 @@ endif() ...@@ -358,7 +403,7 @@ endif()
# build the static library # build the static library
if(STATIC_LIB MATCHES "ON") if(STATIC_LIB MATCHES "ON")
add_library(RaptorQ_Static STATIC ${SOURCES} ${HEADERS}) add_library(RaptorQ_Static STATIC ${SOURCES} ${HEADERS} ${HEADERS_LINKED})
add_dependencies(RaptorQ_Static LZ4) add_dependencies(RaptorQ_Static LZ4)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
set_target_properties(RaptorQ_Static PROPERTIES OUTPUT_NAME RaptorQ.${RQ_ABI}) set_target_properties(RaptorQ_Static PROPERTIES OUTPUT_NAME RaptorQ.${RQ_ABI})
...@@ -378,7 +423,7 @@ if(STATIC_LIB MATCHES "ON") ...@@ -378,7 +423,7 @@ if(STATIC_LIB MATCHES "ON")
set_target_properties ( set_target_properties (
RaptorQ_Static RaptorQ_Static
PROPERTIES PROPERTIES
LINK_FLAGS "-Wl,-z,now,-z,relro -pie ${LD_OPT}" LINK_FLAGS "-Wl,-z,now,-z,relro${RQ_LZ4_EXCLUDE_SYM} -pie ${LD_OPT}"
) )
endif() endif()
set_property(TARGET RaptorQ_Static PROPERTY ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib") set_property(TARGET RaptorQ_Static PROPERTY ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib")
...@@ -416,9 +461,9 @@ if(DYNAMIC_LIB MATCHES "ON") ...@@ -416,9 +461,9 @@ if(DYNAMIC_LIB MATCHES "ON")
add_definitions(-DRAPTORQ_DLL) add_definitions(-DRAPTORQ_DLL)
add_definitions(-DRAPTORQ_DLL_EXPORTS) add_definitions(-DRAPTORQ_DLL_EXPORTS)
add_library(RaptorQ SHARED ${SOURCES} ${HEADERS}) add_library(RaptorQ SHARED ${SOURCES} ${HEADERS} ${HEADERS_LINKED})
add_dependencies(RaptorQ LZ4) add_dependencies(RaptorQ LZ4)
target_link_libraries(RaptorQ ${RQ_LZ4_DEP}) target_link_libraries(RaptorQ ${RQ_LZ4_DEP} ${RQ_UBSAN})
target_compile_options( target_compile_options(
RaptorQ PRIVATE RaptorQ PRIVATE
${CXX_COMPILER_FLAGS} ${CXX_COMPILER_FLAGS}
...@@ -435,7 +480,7 @@ if(DYNAMIC_LIB MATCHES "ON") ...@@ -435,7 +480,7 @@ if(DYNAMIC_LIB MATCHES "ON")
# with the wrong number of parameters in the next set_target_properties # with the wrong number of parameters in the next set_target_properties
set(RQ_LINK_LIB "${LD_OPT} ") set(RQ_LINK_LIB "${LD_OPT} ")
else() else()
set(RQ_LINK_LIB "-Wl,-z,now,-z,relro ${LD_OPT}") set(RQ_LINK_LIB "-Wl,-z,now,-z,relro${RQ_LZ4_EXCLUDE_SYM} ${LD_OPT}")
endif() endif()
set_target_properties ( set_target_properties (
RaptorQ RaptorQ
...@@ -449,7 +494,7 @@ endif() ...@@ -449,7 +494,7 @@ endif()
# benchamrks (header only) # benchamrks (header only)
add_executable(libRaptorQ-test EXCLUDE_FROM_ALL test/rfc_test.cpp) add_executable(libRaptorQ-test EXCLUDE_FROM_ALL test/rfc_test.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options( target_compile_options(
libRaptorQ-test PRIVATE libRaptorQ-test PRIVATE
${CXX_COMPILER_FLAGS} ${CXX_COMPILER_FLAGS}
...@@ -463,6 +508,7 @@ target_compile_options( ...@@ -463,6 +508,7 @@ target_compile_options(
test_c PRIVATE test_c PRIVATE
${C_COMPILER_FLAGS} ${C_COMPILER_FLAGS}
) )
add_dependencies(test_c RaptorQ) add_dependencies(test_c RaptorQ)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
# Linux, *BSD # Linux, *BSD
...@@ -473,24 +519,41 @@ else() ...@@ -473,24 +519,41 @@ else()
endif() endif()
# CPP interface - RFC interface (header only) # CPP interface - RFC interface (header only)
add_executable(test_cpp_rfc EXCLUDE_FROM_ALL test/test_cpp_rfc.cpp) add_executable(test_cpp_rfc EXCLUDE_FROM_ALL test/test_cpp_rfc.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options( target_compile_options(
test_cpp_rfc PRIVATE test_cpp_rfc PRIVATE
${CXX_COMPILER_FLAGS} ${CXX_COMPILER_FLAGS} "-DTEST_HDR_ONLY"
) )
target_link_libraries(test_cpp_rfc ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP}) target_link_libraries(test_cpp_rfc ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# CPP interface - RFC interface (linked version)
add_executable(test_cpp_rfc_linked EXCLUDE_FROM_ALL test/test_cpp_rfc.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options(
test_cpp_rfc_linked PRIVATE
${CXX_COMPILER_FLAGS}
)
add_dependencies(test_cpp_rfc_linked RaptorQ)
target_link_libraries(test_cpp_rfc_linked RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# CPP interface - RAW interface (header only) # CPP interface - RAW interface (header only)
add_executable(test_cpp_raw EXCLUDE_FROM_ALL test/test_cpp_raw.cpp) add_executable(test_cpp_raw EXCLUDE_FROM_ALL test/test_cpp_raw.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options( target_compile_options(
test_cpp_raw PRIVATE test_cpp_raw PRIVATE
${CXX_COMPILER_FLAGS} "-DTEST_HDR_ONLY"
)
target_link_libraries(test_cpp_raw RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# CPP interface - RAW interface (linked version)
add_executable(test_cpp_raw_linked EXCLUDE_FROM_ALL test/test_cpp_raw.cpp)
target_compile_options(
test_cpp_raw_linked PRIVATE
${CXX_COMPILER_FLAGS} ${CXX_COMPILER_FLAGS}
) )
target_link_libraries(test_cpp_raw ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP}) add_dependencies(test_cpp_raw_linked RaptorQ)
add_custom_target(examples DEPENDS test_c test_cpp_rfc test_cpp_raw libRaptorQ-test) target_link_libraries(test_cpp_raw_linked RaptorQ ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
# CLI tool - RAW API interface (header only) # CLI tool - RAW API interface (header only)
set(CLI_raw_sources src/cli/RaptorQ.cpp external/optionparser-1.4/optionparser.h) set(CLI_raw_sources src/cli/RaptorQ.cpp external/optionparser-1.4/optionparser.h ${HEADERS} ${HEADERS_ONLY})
if(CLI MATCHES "ON") if(CLI MATCHES "ON")
add_executable(CLI_raw ${CLI_raw_sources}) add_executable(CLI_raw ${CLI_raw_sources})
add_custom_target(CLI_tools DEPENDS CLI_raw) add_custom_target(CLI_tools DEPENDS CLI_raw)
...@@ -506,10 +569,30 @@ set_target_properties(CLI_raw PROPERTIES OUTPUT_NAME RaptorQ ...@@ -506,10 +569,30 @@ set_target_properties(CLI_raw PROPERTIES OUTPUT_NAME RaptorQ
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin") RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
target_link_libraries(CLI_raw ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP}) target_link_libraries(CLI_raw ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
#### EXAMPLES
# CPP interface - RAW interface (header only)
add_executable(example_cpp_raw EXCLUDE_FROM_ALL examples/example_cpp_raw.cpp ${HEADERS_ONLY} ${HEADERS})
target_compile_options(
example_cpp_raw PRIVATE
${CXX_COMPILER_FLAGS}
)
target_link_libraries(example_cpp_raw ${RQ_UBSAN} ${STDLIB} ${CMAKE_THREAD_LIBS_INIT} ${RQ_LZ4_DEP})
add_custom_target(examples DEPENDS test_c test_cpp_rfc test_cpp_rfc_linked test_cpp_raw test_cpp_raw_linked libRaptorQ-test example_cpp_raw)
add_custom_target(everything DEPENDS make_static_deterministic examples docs CLI_tools) add_custom_target(everything DEPENDS make_static_deterministic examples docs CLI_tools)
install(FILES ${HEADERS} DESTINATION include/RaptorQ/) foreach (hdr ${HEADERS} ${HEADERS_ONLY} ${HEADERS_LINKED})
get_filename_component (full_dir_of_file ${hdr} PATH)
# remove the initial "src/":
string (SUBSTRING ${full_dir_of_file} 4 -1 dir_of_file)
install (FILES ${hdr} DESTINATION "include/${dir_of_file}")
endforeach()
install(TARGETS RaptorQ RaptorQ_Static install(TARGETS RaptorQ RaptorQ_Static
LIBRARY DESTINATION lib LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib COMPONENT libraries) ARCHIVE DESTINATION lib COMPONENT libraries)
......
CONTRIBUTING.md
View file @ a394b22a
...@@ -66,7 +66,7 @@ developer accepted the CLA and is the author of the commit. ...@@ -66,7 +66,7 @@ developer accepted the CLA and is the author of the commit.
* Always sign your commits with the "-s" option, to add the "Signed-off-by" line in the commit message * Always sign your commits with the "-s" option, to add the "Signed-off-by" line in the commit message
* Only the main developers are required to GPG-sign the commits. * Only the main developers are required to GPG-sign the commits.
* Run the tests to make sure you have not broken something by mistake * Run the tests to make sure you have not broken something by mistake
* Please keep commits short and to the point. Nobody wants to review 2000+ lines of code that touch everything.
## Documentation ## Documentation
......
README.md
View file @ a394b22a
...@@ -2,12 +2,15 @@ ...@@ -2,12 +2,15 @@
[![build status](https://www.fenrirproject.org/Luker/libRaptorQ/badges/master/build.svg)](https://www.fenrirproject.org/Luker/libRaptorQ/commits/master) [![build status](https://www.fenrirproject.org/Luker/libRaptorQ/badges/master/build.svg)](https://www.fenrirproject.org/Luker/libRaptorQ/commits/master)
[![Donate](https://img.shields.io/badge/Donate-PayPal-blue.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=79L8P7TXWFQK4)
[![Donate](https://img.shields.io/badge/Donate-Patreon-orange.svg)](https://patreon.com/Luker)
[Homepage](https://www.fenrirproject.org/Luker/libRaptorQ/wikis/home) [Homepage](https://www.fenrirproject.org/Luker/libRaptorQ/wikis/home)
stable release: **v0.1.9**
current release: **v1.0.0-prealpha** old release: **v0.1.10**
**CURRENTLY BREAKING APIs, stick with the stable release** > Note: v0.1.10 is not compatible with prefivous versions, which were not RFC compliant!
UPDATE: **almost finished the brakage, things should be stable enough**
current release: **v1.0.0-rc2** (use this in new projects)
RaptorQ is a **Forward Error Correction** algorithm designed to deliver your data RaptorQ is a **Forward Error Correction** algorithm designed to deliver your data
efficiently and without retransmissions for lost packets. efficiently and without retransmissions for lost packets.
...@@ -37,6 +40,16 @@ Currently it's only been tested under Linux, but should work well under ...@@ -37,6 +40,16 @@ Currently it's only been tested under Linux, but should work well under
Unfortunately Windows is not supported. support for <future> is broken, Unfortunately Windows is not supported. support for <future> is broken,
so you can not compile it with Visual Studio 2015. so you can not compile it with Visual Studio 2015.
## Features
- RFC6330 API (complex, not recommended)
- RAW API (simpler, recommended)
- multi language:
- header only C++11
- compiled, C
- compiled, C++98
- cached precomputations (configurable)
## Developers ## ## Developers ##
...@@ -62,45 +75,23 @@ you can also get it from our main server: ...@@ -62,45 +75,23 @@ you can also get it from our main server:
Once you have cloned it, it's always a good thing to check the repository gpg Once you have cloned it, it's always a good thing to check the repository gpg
signatures, so you can import my key with: signatures, so you can import my key with:
new long term key: **7393 DAD2 55BE B575 1DBD A04A B11C D823 BA27 8C85**
``$ gpg --keyserver pgp.mit.edu --recv-key 7393DAD255BEB5751DBDA04AB11CD823BA278C85``
2016 key: 2016 key:
``$ gpg --keyserver pgp.mit.edu --recv-key F61F6137`` ``$ gpg --keyserver pgp.mit.edu --recv-key F61F6137``
2015 key: 2015 key:
``$ gpg --keyserver pgp.mit.edu --recv-key D42DDF0A`` ``$ gpg --keyserver pgp.mit.edu --recv-key D42DDF0A``
please check the full fingerprint, it should be like this: please check the full fingerprint.
```bash
$ gpg2 --fingerprint F61F6137
pub rsa2048/F61F6137 2016-01-11 [expires: 2017-01-25]
Key fingerprint = 95EB 6FA4 03D7 29A7 13C2 FC9F F85A DA1C F61F 6137
uid [ultimate] Luca Fulchir (2016 key) <luker@fenrirproject.org>
full 2015 fingerprint:
Key fingerprint = AB35 E45F 5CA5 E35B 8B55 818F 0157 D133 D42D DF0A
```
Now you have the source, and the key, it's enough to check the signature of the Now you have the source, and the key, it's enough to check the signature of the
last commit: last commit:
``$ git log -n 1 --show-signature`` ``$ git log -n 1 --show-signature``
The important part is that you get something like this: As long as you got the right key, and you find the "gpg: Goog signature",
```bash
gpg: Signature made Mon 11 Jan 2016 14:19:21 CET using RSA key ID F61F6137
gpg: Good signature from "Luca Fulchir (2016 key) <luker@fenrirproject.org>"
[unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: 95EB 6FA4 03D7 29A7 13C2 FC9F F85A DA1C F61F 6137
Author: Luca Fulchir <luker@fenrirproject.org>
```
And as long as you got the right key, and you find the "gpg: Goog signature",
you can be sure you have the right code. you can be sure you have the right code.
TDB: The repository is using the maintainer private key for now,
it will start using a dedicated key in the (near) future.
## Install ## ## Install ##
...@@ -175,7 +166,21 @@ CMAKE_INSTALL_PREFIX Default: /usr/local ...@@ -175,7 +166,21 @@ CMAKE_INSTALL_PREFIX Default: /usr/local
## Using libRaptorQ ## ## Using libRaptorQ ##
C and C++11 interfaces are available. For the C++11, header-only version, you can include:
- "RaptorQ/RaptorQ_v1_hdr.hpp"
- "RaptorQ/RFC6330_v1_hdr.hpp"
For the linked, C+98/C++11 API:
- "RaptorQ/RaptorQ_v1.hpp"
- "RaptorQ/RFC6330_v1.hpp"
For the linked, C API:
- "RaptorQ/RaptorQ.h"
- "RaptorQ/RFC6330.h"
The C++ api is completely in sync between the linked and header-only version,
so you can switch between the two just by changing the included header file.
You can compile a PDF of the documentation by doing: You can compile a PDF of the documentation by doing:
``$ make docs`` ``$ make docs``
......
cmake/FindRQ_LZ4.cmake
View file @ a394b22a
...@@ -46,6 +46,10 @@ ELSE(RQ_LZ4_INCLUDE_DIR) ...@@ -46,6 +46,10 @@ ELSE(RQ_LZ4_INCLUDE_DIR)
ENDIF(RQ_LZ4_INCLUDE_DIR) ENDIF(RQ_LZ4_INCLUDE_DIR)
IF(RQ_LZ4_FOUND) IF(RQ_LZ4_FOUND)
IF(RQ_LZ4_USE_OWN)
# force our own lz4 libary
SET(RQ_LZ4_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/lz4/lib)
ENDIF()
MESSAGE(STATUS "Found lz4 in ${RQ_LZ4_INCLUDE_DIR}") MESSAGE(STATUS "Found lz4 in ${RQ_LZ4_INCLUDE_DIR}")
# we need both headers and library, or we use our own # we need both headers and library, or we use our own
IF(RQ_LZ4_INCLUDE_DIR MATCHES "${CMAKE_CURRENT_SOURCE_DIR}/external/lz4/lib" OR RQ_LZ4_LIB MATCHES "RQ_LZ4_LIB-NOTFOUND") IF(RQ_LZ4_INCLUDE_DIR MATCHES "${CMAKE_CURRENT_SOURCE_DIR}/external/lz4/lib" OR RQ_LZ4_LIB MATCHES "RQ_LZ4_LIB-NOTFOUND")
......
cmake/compiler_flags.cmake
View file @ a394b22a
...@@ -33,7 +33,6 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug") ...@@ -33,7 +33,6 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug")
check_cxx_compiler_flag("-fsanitize=undefined" RQ_FLAG_CXX_UBSAN) check_cxx_compiler_flag("-fsanitize=undefined" RQ_FLAG_CXX_UBSAN)
if (RQ_FLAG_C_UBSAN AND RQ_FLAG_CXX_UBSAN) if (RQ_FLAG_C_UBSAN AND RQ_FLAG_CXX_UBSAN)
set(RQ_ENABLE_UBSAN TRUE) set(RQ_ENABLE_UBSAN TRUE)
set(RQ_UBSAN ubsan)
message(STATUS "UBSAN sanitizing support enabled") message(STATUS "UBSAN sanitizing support enabled")
else() else()
message(STATUS "UBSAN sanitizing support disabled") message(STATUS "UBSAN sanitizing support disabled")
...@@ -48,8 +47,19 @@ endif() ...@@ -48,8 +47,19 @@ endif()
################### ###################
#gnu options #gnu options
set(RQ_GNU_C_OPTIONS ${RQ_DETERMINISTIC} -std=c11 -ffast-math -Wno-unknown-pragmas -Wall -Wextra -pedantic -Wno-padded -fstack-protector-all -fstrict-aliasing -fwrapv -fvisibility=hidden ) set(RQ_GNU_C_OPTIONS ${RQ_DETERMINISTIC} -std=c11
set(RQ_GNU_CXX_OPTIONS ${RQ_DETERMINISTIC} -std=c++11 -ffast-math -fno-rtti -fno-exceptions -Wno-unknown-pragmas -Wall -Wextra -pedantic -Wno-padded -Wno-unknown-pragmas -fstack-protector-all -fstrict-aliasing -fwrapv -fvisibility=hidden -fvisibility-inlines-hidden ) -Wno-unknown-pragmas -Wall -Wextra -pedantic -Wno-padded
-fstack-protector-all -fstrict-aliasing -fwrapv -fvisibility=hidden
-Wduplicated-cond -Wduplicated-branches -Wlogical-op -Wrestrict
-Wnull-dereference -Wdouble-promotion -Wshadow -Wformat=2)
set(RQ_GNU_CXX_OPTIONS ${RQ_DETERMINISTIC} -std=c++11
-fno-rtti -fno-exceptions -Wno-unknown-pragmas -Wall -Wextra
-pedantic -Wno-padded -Wno-unknown-pragmas -fstack-protector-all
-fstrict-aliasing -fwrapv -fvisibility=hidden -fvisibility-inlines-hidden
-Wduplicated-cond -Wduplicated-branches -Wlogical-op -Wrestrict
-Wnull-dereference -Wuseless-cast -Wdouble-promotion -Wshadow -Wformat=2
-Wno-duplicated-branches)
# GCC internal compiler errors with: # GCC internal compiler errors with:
# -fsanitize=undefined # -fsanitize=undefined
...@@ -67,8 +77,18 @@ set(RQ_GNU_C_RELWITHDEBINFO -g -Ofast -fwrapv -ftree-loop-distribution -funrol ...@@ -67,8 +77,18 @@ set(RQ_GNU_C_RELWITHDEBINFO -g -Ofast -fwrapv -ftree-loop-distribution -funrol
set(RQ_GNU_CXX_RELWITHDEBINFO -g -Ofast -fwrapv -ftree-loop-distribution -funroll-loops ) set(RQ_GNU_CXX_RELWITHDEBINFO -g -Ofast -fwrapv -ftree-loop-distribution -funroll-loops )
# clang options # clang options
set(RQ_CLANG_C_OPTIONS ${RQ_DETERMINISTIC} -std=c11 -ffast-math -fno-math-errno -Wall -pedantic -Weverything -Wno-padded -fstack-protector-all -fstrict-aliasing -Wformat -Wformat-security -Wno-disabled-macro-expansion -fvisibility=hidden -fvisibility-inlines-hidden) set(RQ_CLANG_C_OPTIONS ${RQ_DETERMINISTIC} -std=c11
set(RQ_CLANG_CXX_OPTIONS ${RQ_STDLIB_FLAG} ${RQ_DETERMINISTIC} -std=c++11 -fno-rtti -fno-exceptions -ffast-math -fno-math-errno -Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat -Wno-padded -Wno-unknown-pragmas -fstack-protector-all -fstrict-aliasing -Wformat -Wformat-security -fvisibility=hidden -fvisibility-inlines-hidden) -fno-math-errno -Wall -Wextra -pedantic -Weverything -Wno-padded
-fstack-protector-all -fstrict-aliasing -Wformat -Wformat-security
-Wno-disabled-macro-expansion -fvisibility=hidden -fvisibility-inlines-hidden
-Wdouble-promotion -Wshadow -Wformat=2 -Wnull-dereference)
set(RQ_CLANG_CXX_OPTIONS ${RQ_STDLIB_FLAG} ${RQ_DETERMINISTIC}
-std=c++11 -fno-rtti -fno-exceptions -fno-math-errno
-Wall -pedantic -Weverything -Wno-c++98-compat-pedantic -Wno-c++98-compat
-Wno-padded -Wno-unknown-pragmas -fstack-protector-all -fstrict-aliasing
-Wformat -Wformat-security -fvisibility=hidden -fvisibility-inlines-hidden
-Wno-documentation -Wno-documentation-unknown-command
-Wdouble-promotion -Wshadow -Wformat=2 -Wnull-dereference)
set(RQ_CLANG_C_DEBUG -O0 -g ) set(RQ_CLANG_C_DEBUG -O0 -g )
set(RQ_CLANG_CXX_DEBUG -O0 -g ) set(RQ_CLANG_CXX_DEBUG -O0 -g )
...@@ -96,6 +116,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") ...@@ -96,6 +116,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_DEBUG}) set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_DEBUG})
if (RQ_ENABLE_UBSAN) if (RQ_ENABLE_UBSAN)
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_DEBUG_SANITIZE}) set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_DEBUG_SANITIZE})
set(RQ_UBSAN ${RQ_CLANG_CXX_DEBUG_SANITIZE})
endif() endif()
elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel") elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_MINSIZEREL}) set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_CLANG_CXX_MINSIZEREL})
...@@ -110,6 +131,7 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") ...@@ -110,6 +131,7 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_DEBUG}) set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_DEBUG})
if (RQ_ENABLE_UBSAN) if (RQ_ENABLE_UBSAN)
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_DEBUG_SANITIZE}) set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_DEBUG_SANITIZE})
set(RQ_UBSAN ${RQ_GNU_CXX_DEBUG_SANITIZE})
endif() endif()
elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel") elseif(CMAKE_BUILD_TYPE MATCHES "MinSizeRel")
set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_MINSIZEREL}) set(RQ_LIST_CXX_COMPILER_FLAGS ${RQ_LIST_CXX_COMPILER_FLAGS} ${RQ_GNU_CXX_MINSIZEREL})
......
cmake/guess_architecture.cmake
View file @ a394b22a
# #
# Copyright (c) 2015, Luca Fulchir<luca@fulchir.it>, All rights reserved. # Copyright (c) 2015-2018, Luca Fulchir<luca@fulchir.it>, All rights reserved.
# #
# This file is part of libRaptorQ. # This file is part of libRaptorQ.
# #
...@@ -31,31 +31,29 @@ SET(ARCH "X86_64" CACHE STRING "Processor architecture") # defult, most common ...@@ -31,31 +31,29 @@ SET(ARCH "X86_64" CACHE STRING "Processor architecture") # defult, most common
SET_PROPERTY(CACHE ARCH PROPERTY STRINGS X86_64 X86 ARM) SET_PROPERTY(CACHE ARCH PROPERTY STRINGS X86_64 X86 ARM)
IF(ARCH) IF(ARCH)
# you provided us the right architecture? good ^^ # you provided us the right architecture? good ^^
ELSE() ELSE()
#we need to guess it somehow... #we need to guess it somehow...
IF(WIN32) IF(WIN32)
SET(ARCH "X86_64") SET(ARCH "X86_64")
ENDIF() ENDIF()
IF(APPLE) IF(APPLE)
EXECUTE_PROCESS(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCH ) EXECUTE_PROCESS(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCH )
ENDIF() ENDIF()
IF(UNIX) IF(UNIX)
EXECUTE_PROCESS(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCH ) EXECUTE_PROCESS(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCH )
IF(${ARCH} STREQUAL "x86_64" OR ${ARCH} STREQUAL "amd64" OR ${ARCH} STREQUAL "ia64") IF(${ARCH} STREQUAL "x86_64" OR ${ARCH} STREQUAL "amd64" OR ${ARCH} STREQUAL "ia64")
SET(ARCH, "X86_64") SET(ARCH, "X86_64")
ELSE() ELSE()
IF(${ARCH} STREQUAL "x86" OR ${ARCH} STREQUAL "i686") IF(${ARCH} STREQUAL "x86" OR ${ARCH} STREQUAL "i686")
SET(ARCH, "X86") SET(ARCH, "X86")
ELSE() ELSE()
IF(${ARCH} STREQUAL "arm" OR ${ARCH} STREQUAL "armeb" OR ${ARCH} STREQUAL "armel") IF(${ARCH} STREQUAL "arm" OR ${ARCH} STREQUAL "armeb" OR ${ARCH} STREQUAL "armel")
SET(ARCH, "ARM") SET(ARCH, "ARM")
ELSE() ELSE()
ENDIF() ENDIF()
ENDIF() ENDIF()
ENDIF() ENDIF()
ENDIF() ENDIF()
ENDIF() ENDIF()
doc/CMakeLists.txt
View file @ a394b22a
...@@ -23,17 +23,24 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) ...@@ -23,17 +23,24 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
INCLUDE(../cmake/UseLATEX.cmake) INCLUDE(../cmake/UseLATEX.cmake)
IF (LATEX_COMPILER AND PDFLATEX_COMPILER AND MAKEINDEX_COMPILER) IF (LATEX_COMPILER AND PDFLATEX_COMPILER AND MAKEINDEX_COMPILER)
ADD_LATEX_DOCUMENT(libRaptorQ.tex ADD_LATEX_DOCUMENT(libRaptorQ-RFC6330.tex
INPUTS GNU_FDL.tex INPUTS GNU_FDL.tex
USE_INDEX USE_INDEX
MANGLE_TARGET_NAMES MANGLE_TARGET_NAMES
NO_DEFAULT NO_DEFAULT
) )
ADD_LATEX_DOCUMENT(libRaptorQ-RaptorQ.tex
INPUTS GNU_FDL.tex
USE_INDEX
MANGLE_TARGET_NAMES
NO_DEFAULT
)
ADD_CUSTOM_TARGET(docs DEPENDS libRaptorQ_pdf) ADD_CUSTOM_TARGET(docs DEPENDS libRaptorQ-RFC6330_pdf libRaptorQ-RaptorQ_pdf)
ELSE() ELSE()
ADD_CUSTOM_TARGET(docs) ADD_CUSTOM_TARGET(docs)
ENDIF() ENDIF()
INSTALL(FILES ${CMAKE_BINARY_DIR}/doc/libRaptorQ.pdf DESTINATION share/doc/ OPTIONAL) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/doc/libRaptorQ-RFC6330.pdf DESTINATION share/doc/ OPTIONAL)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/doc/libRaptorQ-RaptorQ.pdf DESTINATION share/doc/ OPTIONAL)
doc/libRaptorQ-RFC6330.tex 0 → 100644
View file @ a394b22a
% Copyright (C) 2015-2018, Luca Fulchir<luca@fulchir.it>
% Permission is granted to copy, distribute and/or modify this document
% under the terms of the GNU Free Documentation License, Version 1.3
% or any later version published by the Free Software Foundation;
% with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
% A copy of the license is included in the section entitled "GNU
% Free Documentation License".
\documentclass[11pt,a4paper]{refart}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{hyperref}
\usepackage{makeidx}
\usepackage{verbatimbox}
\usepackage{listings}
\usepackage{color}
\definecolor{mygreen}{rgb}{0,0.6,0}
\definecolor{mygray}{rgb}{0.5,0.5,0.5}
\definecolor{mymauve}{rgb}{0.58,0,0.82}
\lstset{ %
backgroundcolor=\color{white}, % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}
basicstyle=\footnotesize, % the size of the fonts that are used for the code
breakatwhitespace=false, % sets if automatic breaks should only happen at whitespace
breaklines=true, % sets automatic line breaking
captionpos=b, % sets the caption-position to bottom
commentstyle=\color{mygreen}, % comment style
deletekeywords={...}, % if you want to delete keywords from the given language
escapeinside={\%*}{*)}, % if you want to add LaTeX within your code
extendedchars=true, % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
frame=single, % adds a frame around the code
keepspaces=true, % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
keywordstyle=\color{blue}, % keyword style
language=C, % the language of the code
%otherkeywords={*,...}, % if you want to add more keywords to the set
%numbers=left, % where to put the line-numbers; possible values are (none, left, right)
%numbersep=5pt, % how far the line-numbers are from the code
%numberstyle=\tiny\color{mygray}, % the style that is used for the line-numbers
rulecolor=\color{black}, % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
showspaces=false, % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
showstringspaces=false, % underline spaces within strings only
showtabs=false, % show tabs within strings adding particular underscores
stepnumber=2, % the step between two line-numbers. If it's 1, each line will be numbered
stringstyle=\color{mymauve}, % string literal style
tabsize=4, % sets default tabsize to 2 spaces
title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title
}
\title{Using libRaptorQ library: RFC6330 interface}
\date{\today}
\makeindex
\begin{document}
\maketitle
\begin{abstract}
\textbf{libRaptorQ} is a C++11 implementation of the RaptorQ Forward Error Correction. It includes two interfaces, one derived from the \href{https://tools.ietf.org/html/rfc6330}{RFC6330} standard, and one RAW API for a more flexible and easy usage.
This document is only about the RFC6330 interface.
The implementation was started as a university laboratory project, and will be later used and included in \href{https://www.fenrirproject.org}{Fenrir}, the
maintainer's master thesis project.
\end{abstract}
\vfill\hfill
\begin{verbbox}[\tiny]
Copyright (C) 2015-2018, Luca Fulchir<luker@fenrirproject.org>
Permission is granted to copy, distribute and/or modify this document
under the terms of the GNU Free Documentation License, Version 1.3
or any later version published by the Free Software Foundation;
with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
A copy of the license is included in the section entitled "GNU
Free Documentation License".
\end{verbbox}
\theverbbox
\newpage
\tableofcontents
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Contacts}\index{Contacts}
The main development and dicussions on the project, along with bug reporting, happens on
\href{https://fenrirproject.org/Luker/libRaptorQ}{the main website}.
\marginlabel{Mailing Lists} Mailing lists are available at \href{https://fenrirproject.org/lists}{https://fenrirproject.org/lists}\\
The two mailing lists are for development and announcements, but due to the low traffic of the development mailing list, it
can also be used by users for questions on the project.
\marginlabel{IRC} Since there are not many developers for now, the main irc channel is \textbf{\#fenrirproject} on freenode
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Build \& install}
\subsection{Get the source code}
This document follows the 1.0 stable release.
You can get the tarballs from the main website here:\\
\href{https://fenrirproject.org/Luker/libRaptorQ/tags}{https://fenrirproject.org/Luker/libRaptorQ/tags}
Or you can check out the repository (warning: development in progress):
\begin{lstlisting}[language=bash,breaklines=true]
$ git clone --recurse-submodules \
https://fenrirproject.org/Luker/libRaptorQ.git
\end{lstlisting}
You can also get it from github:
\begin{lstlisting}[language=bash,breaklines=true]
$ git clone --recurse-submodules \
https://github.com/LucaFulchir/libRaptorQ.git
\end{lstlisting}
\index{GPG}\marginlabel{GPG verification:}
Once you have cloned it, it's always a good thing to check the repository gpg
signatures, so you can import my key with:
\begin{lstlisting}[language=bash,breaklines=true]
$ gpg --keyserver pgp.mit.edu --recv-key \
7393DAD255BEB5751DBDA04AB11CD823BA278C85
\end{lstlisting}
Now you have the source, and the key, it's enough to check the signature of the
last commit:
\begin{verbatim}
$ git log -n 1 --show-signature
\end{verbatim}
The important part is that you get something like this:
\begin{verbbox}[\footnotesize]
gpg: Signature made Mon 11 Dec 2017 21:55:28 CET
gpg: using RSA key 8F7474044095B405D0F042F0A2CCA134BC7C8572
gpg: Good signature from "Luca Fulchir <luker@fenrirproject.org>" [unknown]
gpg: aka "Luca Fulchir <luca@fulchir.it>" [unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: 7393 DAD2 55BE B575 1DBD A04A B11C D823 BA27 8C85
\end{verbbox}
\theverbbox
And as long as you got the right key, and you find the \textbf{"gpg: Good signature"} string,
you can be sure you have the right code.
The main development happens in the $master$ branch, while we keep one branch for each stable version.
\subsection{Dependencies}\index{dependencies}
libRaptorQ has 4 dependencies:
\begin{description}
\item[\textbf{Eigen3}] This is used for matrix manipulation, which is a big part of RaptorQ.
\item[\textbf{lz4}] Used to compress cached precomputations.
\item[\textbf{git}] This is used not only to get the source, but also by the build system. We get the last git commit id and feed it to clang or gcc as seed for their internal random number generator. This makes it possible to have reproducible builds.
\item[\textbf{optionparser}] Library used to parse the command line easily in C++, without exceptions
\end{description}
All dependencies are included in the sources, so you do not need to download and compile them.\\
Eigen3 is only needed at build time, so there is no need to download it again. we use the 3.3.4 version.\\
LZ4 is included as a git submodule. if you do not have it, run:
\begin{verbatim}
$ git submodule init
$ git submodule update
\end{verbatim}
and LZ4 will be built statically in the library (it will not be installed on the system)
\subsection{Build \& Install}
libRaptorQ uses the cMake build system, so things are fairly standard:
\begin{verbatim}
$ cd libRaptorQ.git
$ mkdir build
$ cd build
$ cmake ../
$ make -j 4
\end{verbatim}
By default, the libRaptorQ project tries to have deterministic builds. This means that if you compile things twice, or with two different computers, the
hash of the resulting library will be the same, provided that the same compiler (clang, gcc 4.8, gcc 4.9 etc) was used. Currently the only exception is the clang
compiler with the \textit{PROFILING} option enabled, and this will not likely be solved.
\index{cMake}
There are lots of options, you can use in cmake. As always, you can change them by adding ``\textbf{-Dcmake\_option=cmake\_value}'' when calling cmake.
You can always use the cmake-gui or ccmake commands to have the list of possible options.
The ones we recognize are:
\begin{description}
\item[LTO] ON/OFF. Default:\textbf{ON}. Enables \textit{Link Time Optimizatios} for clang or gcc. Makes libraries smaller and better optimized.
\item[PROFILING] ON/OFF. Default:\textbf{ON}.
Profiling compiles everything once, then runs a test to see which code paths are used more, and then
recompiles everything again, but making sure that the binary is optimized for those paths. Works with clang and gcc. Provides a slight speedup.
\item[CMAKE\_C\_COMPILER] gcc and clang are directly supported. other should work, too. This is only used if you want to build the C example.
\item[CMAKE\_CXX\_COMPILER] g++, clang++ are directly supported. other should work, too.
\item[CLANG\_STDLIB] ON/OFF. Default:\textbf{OFF}.use clang's libc++ standard library. Only available with clang.
\item[CMAKE\_BUILD\_TYPE] Type of build. you can choose between ``Debug'', ``Release'', ``MinSizeRel'' or ``RelWithDebInfo''
\item[CMAKE\_INSTALL\_PREFIX] Default to \textit{/usr/local}. Change it to fit your distribution guidelines.
\item[RQ\_ENDIANNESS] \texttt{Auto,BigEndian,LittleEndian} Force the build to use one endianness ot the other
\item[RQ\_LINKER] \texttt{Auto,gold,ld,bsd} Force the build to use one linker or the other
\item[USE\_LZ4] ON/OFF. Default:\textbf{ON}. compile with support for lz4
\end{description}
Then you can build everything by running:
\begin{verbatim}
$ make -j 4
\end{verbatim}
Of course, you can configure the number of parallel jobs (the \textit{-j} parameter) to be what you need.
\marginlabel{Optional make targets:}
The following optional targets are also supported:
\begin{verbatim}
$ make docs tests examples
$ make everything
\end{verbatim}
The ``docs'' target builds this document, but you need latex with the refman style.
The tests are only useful to check perfromance of rfc compliance right now. ``examples'' compiles the C and C++ examples, which will not be installed.
\index{Install}\marginlabel{\textbf{Install:}}
The installation process is very simple:
\begin{verbatim}
$ make install DESTDIR=...
\end{verbatim}
You can change the \textit{DESTDIR} parameter to fit your distribution guidelines.
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Working with RaptorQ}
\subsection{Theory (you really need this)}\index{Theory}
To be able to work with liRaptorQ, you must first understand how the RaptorQ algorithms works. We won't go into the details, but just what you need
to be able to tune the algorithm to your needs.
\marginlabel{Fountain codes:}
Fountain codes are a special \textit{Forward-Error-Correcting} code class, which characteristic is simple: if you want to send $K$ packets, you actually
send $K+X$ packets, and the receiver only needs to get any $K$ packets to be able to reconstruct your data. The number $X$ of overhead packets can be as big
as you need (theoretically infinite), so you can tune it to be slightly higher than the expected packet loss.
\marginlabel{Systematic codes:} RaptorQ is also a systematic code. This means that those first $K$ packets are the input \textit{as-is} (\textbf{source symbols}),
and the $X$ packets (\textbf{repair symbols}) have the information needed to recover any of the lost source packets. This has the big advantage of
avoiding any kind of time and memory consuming decoding if there is no packet loss during the transmission.
\marginlabel{Complexity:} The RaptorQ algorithm is often presented as having a linear time encoder and decoder. This is both false and misleading.
Generating the source or repair symbols from the intermediate symbols has linear complexity. Generating the intermediate symbols has cubic complexity on the
number of symbols. Which is a completely different thing.
It is still very quick. On a core i7, 2.4Ghz, you need to wait \textit{0.4ms} for $10$ symbols, \textit{280ms} for 1.000 symbols, but it can take an hour for $27.000$ symbols.
RaptorQ handles up to $56.403$ symbols.
\marginlabel{\textbf{Caching}} libRaptorQ can work with big matrices that take a lot of time to compute. For this reason the matrices can be saved once they have
been computed the first time. libRaptorQ uses a \textbf{local} cache. The matrix can be compressed with LZ4.
\subsection{RFC6330: Blocks \& Symbols}
To understand how to properly use and tune libRaptorQ, you first need to understand how RaptorQ and the \textbf{RFC6330} handles its inputs, outputs, and what the time and memory
constraints are.
\index{Sequencing}\marginlabel{Input sequencing:}
The RC6330 needs to have the whole input you want to send before it can start working.\\
This means that it might be a little more difficult to use in live-streaming contexts, or where you need real-time data, but for that you probably want to use the single-block API in the \textbf{RaptorQ} C++ namespace. There is a second document tracking that interface, this document tracks only the RFC6330 interface.
Once you have the whole input, RFC6330 divides it into \textbf{blocks}. Each block \textit{is encoded and decoded independently} and will be divided into \textbf{symbols}. Each symbol \textit{should}
be transmitted separately in its own packet (if you are on the network).
\index{Sequencing!Sizes}\marginlabel{Sizes:}Each input can have up to \textit{256 blocks}, each block can have up to \textit{$56.403$ symbols}, and each
symbol can have up to \textit{$2^{16}-1$ bytes}long. This gives a maximum files size of almost $881$ GB (946270874880 bytes to be exact)
\index{Interleaving}\marginlabel{Interleaving:}
An other feature of RFC6330 is to automatically provide some interleaving of the input data before transmitting it. This means that one symbol will not
represent one sequential chunk of your input data, but more likely it's the concatenation of different \textbf{sub-symbols}. The size of the subsymbol must thus
be a fraction of the symbol size. \textit{This feature is not used if you set the size of the subsymbol to the size of symbol}.
\marginlabel{Memory and Time:}
Memory and time requirements are to be considered, though, as RFC6330 needs to run a cubic algorithm on matrix of size $K*K$, where $K$ is the number of
symbols in each block.\\
The algorithm needs to keep in memory two of these matrices, although most of the work is done on only one.\\
This is actually a lot. More benchmarks and optimizations will come later, for now remember that with 10 symbols it takes something like 0.4ms on a core i7 2.4GHZ, 280ms with 1000 symbols, and up to an hour with 27.000 symbols.
\subsection{C++ interface}
\index{Interface!C++}
To use the C++ interface you only need to include the \textbf{RaptorQ/RFC6330\_v1.hpp} header, and link \textbf{libRaptorQ} and your threading library (usually
\textit{libpthread}).
You can also use the library as a header-only include, by including \textbf{RaptorQ/RFC6330\_v1\_hdr.hpp}. All your code should keep working even if you switch between linked and header-only at a later date. If you use the header-only version, remember to have Eigen3 in your include path and to include the needed definitions, \texttt{RQ\_LITTLE\_ENDIAN} or \texttt{RQ\_BIG\_ENDIAN}
\marginlabel{Namespace}
Since the libRaptorQ library now has two very different usages, everything was split into two namespaces: \textbf{RFC6330} and \textbf{RaptorQ}. The names should be explanatory: the first namespace (that we cover in this document) is for the RFC interface, while the second namespace is for the low-level interface.
Since we will cover only the \textbf{RFC6330} namespace here, remember that everything we talk about is in this namespace.
Both the RFC6330 and the RaptorQ namespace are versioned so you can keep using old APIs while we change stuff. just add \textbf{\texttt{\_\_v1}} to select the first version of the API. E.g: \textbf{\texttt{RFC6330\_\_v1}}
\newpage
\index{Iterators}\marginlabel{Templates}
There are two main classes you will use:
\begin{verbatim}
template <typename Rnd_It, typename Fwd_It>
class Encoder
template <typename In_It, typename Fwd_It>
class Decoder
\end{verbatim}
As you might guess, the classes for the encoder and decoder take two template parameters.\\
For the \textbf{Encoder}, the first parameter \textit{MUST} be a \textit{random access iterator} to the input data, and the second parameter is an
\textit{forward iterator}. The random access iterator will be used to scan your input data, and perform an interleaving (if you did not set the same size the
symbol and to the subsymbol). The forward iterator will be used to write the data to your structure.\\
The same is done for the \textbf{Decoder}, but we do not need to do any interleaving on the input, so the first iterator can be just an input iterator,
and nothing more.
\subsubsection{Caching precomputations}
\index{Cache!C++'}
libRaptorQ can now cache the most used matrices for a quicker reference. The cache can be scaled up or down dynamically.
\begin{description}
\item[local\_cache\_size] \textbf{const uint64\_t local\_cache}\\
\textbf{return: bool}\\
Set the local cache size. returns false on error.
\item[get\_local\_cache\_size()] \textbf{return: uint64\_t}\\
get the size of our local cache
\item[supported\_compressions()] \textbf{return: Compress} \\
Get the bitmask of all supported compression algorithms. currently only \textbf{Compress::NONE} and \textbf{Compress::LZ4}.
\item[get\_compression()] \textbf{return: Compress} \\
Get the current compression used.
\item [set\_compression] \textbf{const Compress compression} \\
\textbf{return: bool}\\
Try to set a compression algorithm. LZ4 is not a mandatory dependency, so we might not have it
\end{description}
\newpage
\subsubsection{Thread pool}
Both the encoder and the decoder can work with multiple blocks at the same time, so it makes sense to have a thread pool for such work.
\begin{description}
\item[set\_thread\_pool] \textbf{Input: const size\_t threads}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t max\_block\_concurrency}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RaptorQ\_\_v1::Work\_State exit\_type}\\
\textbf{return: bool}\\
set a pool of a certain number of threads. But the decoder is special: if one decoding fails we can always try again once we have at least one other symbol. The second parameter tells the pool how many times we can try to decode the same block concurrently. $1$ is a safe option.\\
The final argument is used when downsizing the queue: what do we do if all the threads are running but you asked to have less threads?
\begin{lstlisting}
enum class Work_State : uint8_t {
KEEP_WORKING
ABORT_COMPUTATION
};
\end{lstlisting}
\end{description}
\newpage
\subsubsection{The Encoder}
\index{Encoder!C++}
The constructor is the following:
\begin{lstlisting}[language=C++]
Encoder (const Rnd_It data_from, const Rnd_It data_to,
const uint16_t min_subsymbol_size,
const uint16_t symbol_size,
const size_t max_sub_block);
\end{lstlisting}
As we said, the RFC will decide by itself how many blocks each input will have, the actual size of the subsymbol, and the size of the blocks. Although needed for RFC compliance, you probably do not care about the \textit{min\_subsymbol\_size}, in which case the safe choice is to put it at the same size of the symbol.\\
The \textit{max\_sub\_block} the the maximum size of a sub-block in bytes. Again, unless you \textit{really} understand the RFC, you should probably ignore this and put it to a high value. This parameter is used to limit the block size used by the RFC, and is the size (in bytes) of the biggest block you allow the RFC to use.\\
\textbf{safe settings for max\_sub\_block:} sub\_symbol\_size * block\_size.
You can instantiate an encoder for example by doing:
\begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output;
...
using T_it = typename std::vector<uint32_t>::iterator;
RFC6330::Encoder<T_it, T_it> enc (input.begin(),
input.end(),
4, 1444, 10000)
\end{lstlisting}
This will create an Encoder that works on vectors of unsigned 32 bit integers for both input and output, that will create symbols of size 1444 bytes, interleaving
your input every 4 bytes, and try to work with a big number of symbols per blocks.
The available methods for the encoder are the following:
\begin{description}
\item[operator bool()] \textbf{return:bool}\\
False if constructor parameters did not make sense. Else true.
\item[OTI\_Common()] \textbf{return: OTI\_Common\_Data}, aka \textit{uint64\_t}.\\
Keeps total \textbf{file size and symbol size}. You need to send this to the receiver, so that it will be able to properly decode the data.
\item[OTI\_Scheme\_Specific\_Data()] \textbf{return: OTI\_Scheme\_Specific\_Data}, aka \textit{uint32\_t}.\\
Keeps number of \textbf{source blocks, sub blocks, and alignment}. As for the OTI\_Common\_Data, you need to send this to the receiver to be able to
properly decode the data.
\item[compute] \textbf{Input: const Compute flags}\\
\textbf{return: std::future<std::pair<Error, uint8\_t> >}\\
For C++11 only (automatically disabled on C++98). Start the computation, return a future, which has a pair of Error and the number of blocks ready.\\
Compute flags:
\begin{lstlisting}[language=C++]
enum class Compute : uint8_t {
NONE
// report every time there is more data, but only
// if that data starts at the beginning.
PARTIAL_FROM_BEGINNING
PARTIAL_ANY
COMPLETE // only when all the data is available
NO_BACKGROUND // only return after the computation is finished
NO_POOL // do not use the thread pool
NO_RETRY // do not retry if we failed, even if there is more data
};
\end{lstlisting}
\item [precompute\_max\_memory] \textbf{return: size\_t}\\
Each precomputation can take a lot of memory, depending on the configuration, so you might want to limit the number of precomputations run in parallel
depending on the memory used. This method returns the amount of memory taken by \textbf{ONE} precomputation.
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t esi, const uint8\_t sbn}.\\
\textbf{return:size\_t}.\\
Take as input the iterators to the data structure into where we have to save the encoded data, the \textbf{Encoding Symbol Id} and the
\textbf{Source Block Number}. As you are writing in C++, you probably want to use the iterators begin/end, though. Returns the number of written
iterators (\textbf{NOT} the bytes)
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t id}.\\
\textbf{return:size\_t}.\\
Exactly as before, but the \textbf{id} contains both the \textit{source block number} and the \textit{encoding symbol id}
\item[begin()] \textbf{return: Block\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the blocks in which the RFC divided the input data. See later to understand how to use it.
\item[end()] \textbf{return: const Block\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the end of the blocks in which the RFC divided the input data. See later to understand how to use it.
\item [free] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: void}\\
Each block takes some memory, (a bit more than $symbols * symbol\_size$), so once you are done sending source and repair symbols for one block,
you might want to free the memory of that block.
\item[blocks()] \textbf{return: uint8\_t} The number of blocks.
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
The block size, in bytes. Each block can have different symbols and thus different size. The last block also might have more padding, and with this you get the real size of that block, without the padding.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
\item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in a specific block. different blocks can have different symbols.
\item[extended\_symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The \textbf{real, internal} number of symbols in a specific block. This can be slightly higher than the \texttt{symbols()} call, and \textit{you probably do not need this} for anything except if you are trying to know more about the RaptorQ algorithm
\item[max\_repair] \textbf{Input: const uint8\_t sbn)}\\
\textbf{return: uint32\_t}\\
The maximum amount of repair symbols that you can generate. Something less than $2^{24}$, but the exact number depends on the number of symbols
in a block
\end{description}
\subsubsection{Blocks}
\index{Blocks!C++}
With the \textit{begin()/end()} calls you get \textit{Input iterators} to the blocks. A Block has the following type:
\begin{lstlisting}[language=C++]
template <typename Rnd_It, typename Fwd_It>
class Block
\end{lstlisting}
and exposes the following methods:
\begin{description}
\item[begin\_source()]\textbf{return: Symbol\_Iterator}
\item[end\_source()]\textbf{return: Symbol\_Iterator}
\item[begin\_repair()]\textbf{return: Symbol\_Iterator}
\item[end\_repair()]\textbf{Input: const uint32\_t max\_repair}\\
\textbf{return: Symbol\_Iterator}
\item[id()]\textbf{return: uint8\_t}
\item[max\_repair()]\textbf{return: uint32\_t}
\item[symbols()]\textbf{return: uint16\_t}
\item[extended\_symbols()]\textbf{return: uint16\_t}\\
As before, this is the amount of internal symbols, that include padding symbols. You probably do not need this unless you are trying
to know more about the RaptorQ algorithm
\item[block\_size()]\textbf{return: uint32\_t}
\end{description}
As the names explain, you will get an iterator to the symbols in the block. As the number of repair symbols can vary, for now you get two separate begin/ends,
so that you can check when you sent the source symbols, and how many repair symbols you send.\\
The other functions are helpers for the details of the block.
\subsubsection{Symbols}
\index{Symbols!C++}
Finally, through the \textit{Symbol\_Iterator} \textit{Input Iterator} we get the \textbf{Symbol} class:
\begin{lstlisting}[language=C++]
template <typename Rnd_It, typename Fwd_It>
class Symbol
\end{lstlisting}
which exposes the following methods we need to get the symbol data:
\begin{description}
\item[operator()]\textbf{Input:Fwd\_It \&start, const Fwd\_It end}\\
\textbf{return: uint64\_t}\\
takes a forward iterator, and fill it with the symbol data. returns the number of written iterators.
\item[id()]\textbf{return: uint32\_t}\\
return the id (\textit{$sbn + esi$}) of this symbol, that you need to include in every packet you send, before the symbols.
\item[block()]\textbf{return: uint8\_t}\\
return the \textit{block} of this symbol.
\item[esi()]\textbf{return: uint32\_t}\\
return the \textit{esi} of this symbol.
\end{description}
\newpage
\subsubsection{The Decoder}
\index{Decoder!C++}
The decoder is a bit simpler than the encoder.
There are two constructors for the Decoder:
\begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output;
using T_it = typename std::vector<uint32_t>::iterator;
RaptorQ::Decoder<T_it, T_it> dec (
const OTI_Common_Data common,
const OTI_Scheme_Specific_Data scheme)
RaptorQ::Decoder<T_it, T_it> dec (uint64_t size,
uint16_t symbol_size,
uint16_t sub_blocks,
uint8_t blocks)
\end{lstlisting}
Which should be pretty self-explanatory, once you understand how the encoder works:
you can either get the \texttt{OTI} parameters from the Encoder or you can provide the raw values by yourself (only if you know what you are doing)
The remaining methods are:
\begin{description}
\item[begin()] \textbf{return: Block\_Iterator<In\_It, Fwd\_It>}\\
This returns an iterator to the blocks in which the RFC divided the input data.
\item[end()] \textbf{return: const Block\_Iterator<In\_It, Fwd\_It>}\\
This returns an iterator to the end of the blocks in which the RFC divided the input data.
\item[operator bool()] \textbf{return: bool}\\
check if the Decoder was initialized properly
\item[compute] \textbf{Input: const Compute flags}\\
\textbf{return: std::future<std::pair<Error, uint8\_t> >}\\
Only enabled if you use C++11. Start the computation as soon as enough data is available.\\
Compute flags:
\begin{lstlisting}[language=C++]
enum class Compute : uint8_t {
NONE
// report every time there is more data, but only
// if that data starts at the beginning.
PARTIAL_FROM_BEGINNING
PARTIAL_ANY
COMPLETE // only when all the data is available
NO_BACKGROUND // only return after the computation is finished
NO_POOL // do not use the thread pool
NO_RETRY // do not retry if we failed, even if there is more data
};
\end{lstlisting}
\item[end\_of\_input] \textbf{Input: const Fill\_With\_Zeros fill}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: std::vector<bool>}\\
Tell the decoder that we know that there will be no more data for this block.
The first parameter is an \texttt{enum}, either \textbf{Fill\_With\_Zeros::YES} or \textbf{Fill\_With\_Zeros::NO}
If you choose \texttt{YES} then the repair symbols will be discarded, the missing symbols will be filled with zeros
and for output you will get the bitmask for which byte was received (true) or was filled with zeros (false).\\
If you choose \textbf{Fill\_With\_Zeros::NO} you will get \textit{an empty vector}
\item[end\_of\_input] \textbf{Input: const Fill\_With\_Zeros fill}\\
\textbf{return: std::vector<bool>}\\
Same as before, but we know that there will be no more inputs for any block.
\item[decode\_symbol]\textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}
\textbf{return: uint64\_t}\\
Decode a single symbol. you need to provide the block number and symbol identifier.
\item[decode\_bytes] \textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: uint64\_t}\\
Decode everything. returns the number of bytes written. You can start writing the output \texttt{skip} bytes after the first iterator in case the parameters you have used do not align (set to $0$ for everyone else).
\item[decode\_bytes] \textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint64\_t}\\
Same as before, but you can specify a \textbf{Single Block Number} instead of the whole output
\newpage
\item[decode\_aligned]\textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: aligned\_res}\\
Same as \texttt{decode\_bytes}, but yu can use this if for some weir reason the data does not align with the type of the output
\begin{lstlisting}[language=C++]
struct aligned_res
{
uint64_t written; // iterators written
uint8_t offset; // bytes written in the last
// iterator if it was not full
};
\end{lstlisting}
\item[decode\_block\_aligned]\textbf{Input: Fwd\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: aligned\_res}\\
Same as before, but now you can choose the block
\item[add\_symbol]\textbf{Input: In\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const In\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: RFC6330::Error}\\
Add one symbol, while explicitly specifying the symbol id and the block id.
\item[add\_symbol]\textbf{Input: In\_It \&start}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const In\_It end}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t id}\\
\textbf{return: RFC6330::Error}\\
Same as before, but extract the block id and the symbol id from the \textit{id} parameter
\item[blocks\_ready()]\textbf{return: uint8\_t}\\
return the number of the blocks that are ready for output
\item[is\_ready()]\textbf{return: bool}\\
Whether everything has been decoded or not.
\item[is\_block\_ready]\textbf{Input:const uint8\_t block}\\
\textbf{return: bool}\\
Test if the specified block has been decoded or not
\item[free]\textbf{Input: const uint8\_t sbn}\\
\textbf{return: void}\\
You might have stopped using a block, but the memory is still there. Free it.
\item[bytes()] \textbf{return: uint64\_t}\\
The total bytes of the output
\item[blocks()] \textbf{return: uint8\_t}\\
The number of blocks.
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
The block size, in bytes. Each block can have different symbols and thus different size.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
\item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in a specific block. different blocks can have different symbols.
\end{description}
\newpage
\subsection{C interface}\index{Interface!C}
The C interface looks a lot like the C++ one.\\
You need to include the \textbf{RaptorQ/RFC6330.h} header, and link the \textit{libRaptorQ} library.\\
\marginlabel{Static linking}\index{Static Linking}
If you are working with the static version of libRaptorQ remember to link the C++ standard library used when
compiling the library (\textit{libstdc++} for gcc or maybe \textit{libc++} for clang), your threding library (usually \textit{libpthread}), and the C math library (\textit{libm}).\\
\marginlabel{\textbf{API versioning}} To better support future changes in the API and to help you with future changes,
there are only 2 API calls in this library:
\begin{description}
\item[RFC6330\_api] \textbf{Input: uint32\_t version}\\
\textbf{return: struct RFC6330\_base\_api*}\\
This will give you a pointer to a struct containing pointers to the calls you will use.
The struct will change depending on which API you request (currently only number $1$ is supported).
Since this pointer works for every api, you have to cast it to the struct you expect, or you will not see
the pointers to the function calls. Then you can use that pointer to access of the functions, like:
\begin{lstlisting}[language=C]
struct RFC6330_v1 *rfc = (struct RFC6330_v1*)
RFC6330_api (1);
rfc->set_compression (RQ_COMPRESS_LZ4);
\end{lstlisting}
\item[RFC6330\_free\_api] \textbf{Input:struct RFC6330\_base\_api **api}\\
\textbf{return: void}\\
As the name implies, this frees the given structure.
\end{description}
Again, we only support API version 1 at the moment.
\index{Cache!C}
\subsubsection{Cache settings}
As for the C++11 interface, you can get and set the local memory cache.
\begin{description}
\item[supported\_compressions] \textbf{return: RFC6330\_Compress}\\
return the bitmask of all supported compressions. currently either \texttt{RQ\_COMPRESS\_NONE} or \texttt{RQ\_COMPRESS\_LZ4}
\item[get\_compression] \textbf{return: RFC6330\_Compress}\\
Which compression are we currently using?
\item[set\_compression] \textbf{Input: const RFC6330\_Compress compression}\\
\textbf{return: bool}\\
Set a compression method or another.
\item[local\_cache\_size] \textbf{Input: const size\_t bytes}\\
\textbf{return: size\_t}\\
Set the maximum amount of cache usable. Default: $0$
\item[get\_local\_cache\_size] \textbf{return: size\_t}\\
get the amount of cache configured
\end{description}
\subsubsection{C Constructors}
The first thing you need is to allocate the encoder or the decoder.
The C interface is just a wrapper around the C++ code, so you still have to specify the same things as before.
A quick glance at the constructors should give you all the information you need:
\begin{lstlisting}[language=C]
typedef enum { RQ_NONE, RQ_ENC_8, RQ_ENC_16,
RQ_ENC_32, RQ_ENC_64, RQ_DEC_8,
RQ_DEC_16, RQ_DEC_32, RQ_DEC_64}
RFC6330_type;
\end{lstlisting}
\begin{description}
\item[Encoder] \textbf{Input:const RFC6330\_type type}\\
.\ \ \ \ \ \ \ \ \ \textbf{void *data\_from}\\
.\ \ \ \ \ \ \ \ \ \textbf{const uint64\_t size}\\
.\ \ \ \ \ \ \ \ \ \textbf{const uint16\_t min\_subsymbol\_size}\\
.\ \ \ \ \ \ \ \ \ \textbf{const uint16\_t symbol\_size}\\
.\ \ \ \ \ \ \ \ \ \textbf{const size\_t max\_sub\_block}\\
\textbf{return: RFC6330\_ptr*}\\
Allocate an encoder. The encoder will work with uint8\_t, uint16\_t, uint32\_t or uint64\_t depending on which \texttt{RFC6330\_type} you choose. The best bet is usually to work with uint8\_t
\item[Decoder] \textbf{Input:const RFC6330\_type type}\\
.\ \ \textbf{const RFC6330\_OTI\_Common\_Data common}\\
.\ \ \textbf{const RFC6330\_OTI\_Scheme\_Specific\_Data scheme}\\
\textbf{return: RFC6330\_ptr*}\\
The \texttt{type} parameter works the same as in the encoder. The other two are parameters you get from the encoder.
\item[Decoder\_raw] \textbf{Input: RFC6330\_type type}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint64\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t symbol\_size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t sub\_blocks}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t blocks}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t alignment}\\
\textbf{return: struct RFC6330\_ptr*}\\
Same as before, but here you can specify the parameters by hand.
\item[initialized]\textbf{Input: const struct RFC6330\_ptr *ptr}\\
\textbf{return: bool}\\
check if the encoder or decoder was initialized properly.
\end{description}
\newpage
\subsubsection{Common functions for (de/en)coding}
These functions are used by both the encoder and the decoder, and will be helpful in tracking how much memory
you will need to allocate, or in general in managing the encoder and decoder.
The \textbf{ptr} must be a valid encoder or decoder.
The names for now are self-explanatory. Blocks can have different symbols, so the size of a block and the number of symbols in a block
depend on which block we are talking about.
\marginlabel{\textbf{symbols, blocks, memory}}\\
\begin{description}
\item[set\_thread\_pool] \textbf{Input: const size\_t threads}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t max\_block\_concurrency}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RFC6330\_Work exit\_type}\\
\textbf{return: bool}\\
Set the size of the thread pool for concurrent work. Since the decoder can fail, but also retry again if there is more data available,
you can also specify how many times libRaptorQ will try to decode the same block concurrently. $1$ is a safe option.\\
The last parameter, \texttt{RFC6330\_Work} specify what to do when you are downsizing the thread pool and some threads are still working. The possible values are \texttt{RQ\_WORK\_KEEP\_WORKING} and \texttt{RQ\_WORK\_ABORT\_COMPUTATION}.
\item[blocks] \textbf{Input: const struct RFC6330\_ptr *ptr}\\
\textbf{return: uint8\_t}\\
The number of blocks in this RFC6330 instance
\item[symbols]\textbf{Input: const struct RFC6330\_ptr *ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The number of symbols in the specified block
\item[extended\_symbols]\textbf{Input: const struct RFC6330\_ptr *ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint16\_t}\\
The the \textbf{real, internal} number of symbols in the specified block. You probably \textit{do not want this} unless you are trying to understand more about the RaptorQ algorithm or you are benchmarking
\item[symbol\_size]\textbf{Input:const struct RFC6330\_ptr *ptr}\\
\textbf{return: size\_t}\\
The size of each symbol
\item[free\_block]\textbf{Input: struct RFC6330\_ptr **ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: void}\\
Free the allocated space for a single block
\item[free]\textbf{Input: struct RFC6330\_ptr **ptr}\\
\textbf{return: void}\\
Free the specified decoder or decoder and all its blocks.
\end{description}
\marginlabel{Computation}\label{computation}\\
\begin{description}
\item[compute] \textbf{Input: const struct RFC6330\_ptr *ptr}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RFC6330\_Compute flags}\\
\textbf{return: RFC6330\_future*}\\
Start the computation as described in the flags. Returns a wrapper to the C++11 futures. You can wait or poll the state of the future in different ways (see right below)\\
\begin{lstlisting}[language=C]
typedef enum {
RQ_COMPUTE_NONE
// report every time there is more data, but only
// if that data starts at the beginning.
RQ_COMPUTE_PARTIAL_FROM_BEGINNING
RQ_COMPUTE_PARTIAL_ANY
RQ_COMPUTE_COMPLETE // only when all the data
// is available
RQ_COMPUTE_NO_BACKGROUND // only return after
// the computation is finished
RQ_COMPUTE_NO_POOL // do not use the thread pool
RQ_COMPUTE_NO_RETRY // do not retry if we failed,
// even if there is more data
} RFC6330_Compute;
\end{lstlisting}
\item[future\_state] \textbf{Input: const struct RFC6330\_future *f}\\
\textbf{return: RFC6330\_Error}\\
poll the state of the future. possible results:
\begin{lstlisting}[language=C]
RQ_ERR_WRONG_INPUT // null pointer?
RQ_ERR_NOT_NEEDED // you have already run
// future_get() on this future
RQ_ERR_WORKING // still working on it
RQ_ERR_NONE // ready to run future_get()
\end{lstlisting}
\newpage
\item[future\_wait\_for] \textbf{Input: const struct RFC6330\_future *f}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint64\_t time}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const RFC6330\_Unit\_Time unit}\\
\textbf{return: RFC6330\_Error}\\
Wait for a future to become ready for X amount of time. the unit is specified in the
last parameter:
\begin{lstlisting}[language=C]
typedef enum {
RQ_TIME_NANOSEC,
RQ_TIME_MICROSEC,
RQ_TIME_MILLISEC,
RQ_TIME_SEC,
RQ_TIME_MIN,
RQ_TIME_HOUR
} RFC6330_Unit_Time;
\end{lstlisting}
\item[future\_wait] \textbf{Input: const struct RFC6330\_future *f}\\
\textbf{return: void}\\
Same as before, but wait indefinitely
\item[future\_get] \textbf{Input: struct RFC6330\_future *future}\\
\textbf{return: struct RFC6330\_Result}\\
Returns the state of the future. If the future is not ready it will wait indefinitely.
\begin{lstlisting}[language=C]
struct RAPTORQ_API RFC6330_Result {
RFC6330_Error error;
uint8_t sbn;
};
\end{lstlisting}
The sbn changes meaning depending to be consistent with the flags you passed to the \texttt{compute()} call.
\item[future\_free] \textbf{Input: struct RFC6330\_future **f}\\
\textbf{return: void}\\
Free the memory allocated for the future.
\end{description}
\newpage
\subsubsection{\textbf{Encoding}}
\index{Encoder!C}
Now we will look at the calls specific to the encoder (that is: calls that need an encoder as the first parameter, and don't make sense in a decoder)
\marginlabel{OTI Data}\\
First, we need to tell the receiver all the parameters that the encoder is using, and for that two functions are provided:
\begin{lstlisting}[language=C]
typedef uint64_t RaptorQ_OTI_Common_Data;
typedef uint32_t RaptorQ_OTI_Scheme_Specific_Data;
\end{lstlisting}
\begin{description}
\item[OTI\_Common]\textbf{Input: struct RaptorQ\_ptr *enc}\\
\textbf{return: RFC6330\_OTI\_Common\_Data}\\
Get the OTI\_Common
\item[OTI\_Scheme\_Specific]\textbf{Input: struct RaptorQ\_ptr *enc}\\
\textbf{return: RFC6330\_OTI\_Scheme\_Specific\_Data}\\
Get the OTI\_Scheme\_Specific
\end{description}
\marginlabel{Encoding}
\begin{description}
\item[max\_repair]\textbf{Input: const struct RFC6330\_ptr *enc}\\
.\ \ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
return the max number of repair packets that block can have. This is around $2^{24}$, so you probably always want to be much, much lower than this. The total number of symbols (source+repair) is limited in the RFC, and since the bigger blocks have more source symbols, the max number of repair symbols has to be lower in bigger blocks. Again, this is always in the ballpark of $2^{24}$, you probably don't want to use this many repair symbols.
\item[precompute\_max\_memory]\textbf{Input: const struct RFC6330\_ptr *enc}\\
\textbf{return: size\_t}\\
Get an estimate of how much memory (bytes) will cost the encoding of a single block. Not 100\% accurate, but close enough.
\item[encode\_id]\textbf{Input: const struct RFC6330\_ptr *enc}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t id}\\
\textbf{return: size\_t}\\
Get one encoded symbol, and store it in \texttt{**data}, which holds a pointer to "\texttt{size}" elements of type uint8\_t, uint16\_t, uint32\_t or uint64\_t depending on how the encoder was initialized. See \texttt{RFC6330\_type} in the encoder initialization. Returns the number of elements written.\\
If you request repair symbols before the computation is finished, the call will block until the data is available. This calls works for both source and repair symbols, just increase the id until you finish the source symbols
\item[encode]\textbf{Input: const struct RFC6330\_ptr *enc}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: size\_t}\\
Same as before, but you can specify the symbol and block explicitly.
\item[id]\textbf{Input: const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
combine the esi and the block number to form an id.
\end{description}
\newpage
\subsubsection{\textbf{Decoding}}
As for the encoder, there are functions which are specific for a decoder. Let's look at them
\marginlabel{General status calls}
\begin{description}
\item[bytes]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\textbf{return: uint64\_t}\\
Return the number of bytes that this decoder should output on complete, successful decoding.
\item[blocks\_ready]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\textbf{return: uint8\_t}\\
Return the number of blocks ready to be returned. Useful if you don't want to wait for the whole data.\\
The number of blocks reported can mean either the number of sequential blocks ready, or the number
of blocks regardless of order, depending on how the \texttt{compute} function was called (see "\texttt{Computation}" on \ref{computation})
\item[is\_ready]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\textbf{return: bool}\\
Test if the whole encoder is ready and all data decoded.
\item[is\_block\_ready]\textbf{Input: const struct RFC6330\_ptr *dec}\\
\.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: bool}\\
Test if a single block is ready and its data decoded.
\end{description}
\index{Decoder!C}\marginlabel{Decoding calls}
\begin{description}
\item[end\_of\_input]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \
\textbf{const RFC6330\_Fill\_With\_Zeros fill}\\
\textbf{return: struct RFC6330\_Byte\_Tracker}\\
Tell the decoder that there will be no more input for any block.
The second parameter is an enum, it can be either \textbf{RQ\_NO\_FILL} or \textbf{RQ\_FILL\_WITH\_ZEROS}.\\
If you use \texttt{RQ\_NO\_FILL} the structure returned will be empty, and the decoder will return error if there are not enough repair symbols.\\
If you use \texttt{RQ\_FILL\_WITH\_ZEROS} the decoder will immediately stop decoding and all missing symbols will be filled with zeros. The returned structure will contain a bitmask so you know if each byte comes from the network (true) or it is one of the filled ones (false)
\item[end\_of\_block\_input]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \
\textbf{const RFC6330\_Fill\_With\_Zeros fill}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: struct RFC6330\_Byte\_Tracker}\\
Same as before, but for a single block instead of the whole data.
\item[add\_symbol\_id]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uin32\_t id}\\
\textbf{return: RFC6330\_Error}\\
Add a symbol to the decoder. The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (RQ\_DEC\_8, RQ\_DEC\_16 etc..). Afterwards data will point \textbf{after} the symbol.
\item[add\_symbol\_id]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uin32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uin8\_t sbn}\\
\textbf{return: RFC6330\_Error}\\
Same as with \texttt{add\_symbol\_id}, but now you can specify the esi and the block number manually
\item[decode\_aligned]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: RFC6330\_Dec\_Result}\\
Decode everything as asked by the \texttt{compute()} call (see "\texttt{Computation}" on \ref{computation}).\\
The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (\texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc..). Afterwards data will point \textbf{after} the decoded data. You can skip "skip" bytes at the beginning of **data if the pointer
is not aligned with your data.
\begin{lstlisting}[language=C]
struct RAPTORQ_API RFC6330_Dec_Result {
uint64_t written;
uint8_t offset;
};
\end{lstlisting}
The result is the number of written \textbf{elements} (size depending on \texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc).\\
The \texttt{offset} parameter is the amount of bytes written in the last element if the alignment used in the encoder does not fit the
alignment used in the decoder. Usually you keep those synchronized, so it should be zero (element fully written)
\newpage
\item[decode\_block\_aligned]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: RFC6330\_Dec\_Result}\\
Same as before, but now you can decode a single block.
\item[decode\_symbol]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t sbn}\\
\textbf{return: uint64\_t}\\
As you can expect, decode a single symbol. The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (\texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc..). Afterwards data will point \textbf{after} the decoded data.\\
Returns the number of bytes written
\item[decode\_bytes]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
\textbf{return: uint64\_t}\\
Decode everything as asked by the \texttt{compute()} call (see "\texttt{Computation}" on \ref{computation}).\\
The \texttt{void **data} will be cast to the correct type as specified in the decoder construction (\texttt{RQ\_DEC\_8}, \texttt{RQ\_DEC\_16} etc..). Afterwards data will point \textbf{after} the decoded data. You can skip "skip" bytes at the beginning of **data if the pointer is not aligned with your data.\\
Returns the number of written bytes.
\item[decode\_block\_bytes]\textbf{Input: const struct RFC6330\_ptr *dec}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const void **data}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t size}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t skip}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const uint8\_t block}\\
\textbf{return: uint64\_t}\\
Same as before, but now decode a single block
\end{description}
\input{GNU_FDL.tex}
\printindex
\end{document}
doc/libRaptorQ.tex → doc/libRaptorQ-RaptorQ.tex
View file @ a394b22a
% Copyright (C) 2015, Luca Fulchir<luca@fulchir.it> % Copyright (C) 2018, Luca Fulchir<luca@fulchir.it>
% Permission is granted to copy, distribute and/or modify this document % Permission is granted to copy, distribute and/or modify this document
% under the terms of the GNU Free Documentation License, Version 1.3 % under the terms of the GNU Free Documentation License, Version 1.3
% or any later version published by the Free Software Foundation; % or any later version published by the Free Software Foundation;
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
\documentclass[11pt,a4paper]{refart} \documentclass[11pt,a4paper]{refart}
\usepackage[utf8]{inputenc} \usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{hyperref} \usepackage{hyperref}
\usepackage{makeidx} \usepackage{makeidx}
\usepackage{verbatimbox} \usepackage{verbatimbox}
...@@ -48,7 +49,7 @@ ...@@ -48,7 +49,7 @@
title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title
} }
\title{Using libRaptorQ library} \title{Using libRaptorQ library: RAW interface}
\date{\today} \date{\today}
\makeindex \makeindex
...@@ -57,20 +58,21 @@ ...@@ -57,20 +58,21 @@
\begin{abstract} \begin{abstract}
\textbf{libRaptorQ} is a C++11 implementation of the RaptorQ Forward Error Correction, as described in the \href{https://tools.ietf.org/html/rfc6330}{RFC6330} . \textbf{libRaptorQ} is a C++11
implementation of the RaptorQ Forward Error Correction. It includes two interfaces, one derived from the \href{https://tools.ietf.org/html/rfc6330}{RFC6330} standard, and one RAW API for a more flexible and easy usage.
This document is only about the RAW interface.
The implementation was started as a university laboratory project, and will be later used and included in \href{https://www.fenrirproject.org}{Fenrir}, the The implementation was started as a university laboratory project, and will be later used and included in \href{https://www.fenrirproject.org}{Fenrir}, the
maintainer's master thesis. maintainer's master thesis project.
This implementation is quite short (the core is $\sim3k$ lines), thanks to the chosen language and the use of external libraries for matrix handling (eigen3).
libRaptorQ is the only RaptorQ implementation in C++, include C hooks, and it is the only free (\textbf{LGPL3}) implementation of the rfc, except for the (apache2)
java implementation, OpenRQ , which is much bigger ($\sim 46k$) and slower.
\end{abstract} \end{abstract}
\vfill\hfill \vfill\hfill
\begin{verbbox}[\tiny] \begin{verbbox}[\tiny]
Copyright (C) 2015, Luca Fulchir<luker@fenrirproject.org> Copyright (C) 2015-2018, Luca Fulchir<luker@fenrirproject.org>
Permission is granted to copy, distribute and/or modify this document Permission is granted to copy, distribute and/or modify this document
under the terms of the GNU Free Documentation License, Version 1.3 under the terms of the GNU Free Documentation License, Version 1.3
or any later version published by the Free Software Foundation; or any later version published by the Free Software Foundation;
...@@ -90,9 +92,9 @@ Free Documentation License". ...@@ -90,9 +92,9 @@ Free Documentation License".
\section{Contacts}\index{Contacts} \section{Contacts}\index{Contacts}
The main development and dicussions on the project, along with bug reporting, happens on The main development and dicussions on the project, along with bug reporting, happens on
\href{https://www.fenrirproject.org/Luker/libRaptorQ}{the main website}. \href{https://fenrirproject.org/Luker/libRaptorQ}{the main website}.
\marginlabel{Mailing Lists} Mailing lists are available at \href{https://www.fenrirproject.org/lists}{https://www.fenrirproject.org/lists}\\ \marginlabel{Mailing Lists} Mailing lists are available at \href{https://fenrirproject.org/lists}{https://fenrirproject.org/lists}\\
The two mailing lists are for development and announcements, but due to the low traffic of the development mailing list, it The two mailing lists are for development and announcements, but due to the low traffic of the development mailing list, it
can also be used by users for questions on the project. can also be used by users for questions on the project.
...@@ -103,41 +105,34 @@ can also be used by users for questions on the project. ...@@ -103,41 +105,34 @@ can also be used by users for questions on the project.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Build \& install} \section{Build \& install}
\subsection{Get the source code} \subsection{Get the source code}
Although things seems to work, no stable release has been released yet, as of \today. This document follows the 1.0 stable release.
This means you can only check this out with git. You can get the tarballs from the main website here:\\
\href{https://fenrirproject.org/Luker/libRaptorQ/tags}{https://fenrirproject.org/Luker/libRaptorQ/tags}
To check out the repository: Or you can check out the repository (warning: development in progress):
\begin{verbatim} \begin{lstlisting}[language=bash,breaklines=true]
$ git clone https://github.com/LucaFulchir/libRaptorQ.git $ git clone --recurse-submodules \
\end{verbatim} https://fenrirproject.org/Luker/libRaptorQ.git
\end{lstlisting}
You can also get it from our main server: You can also get it from github:
\begin{verbatim} \begin{lstlisting}[language=bash,breaklines=true]
$ git clone https://www.fenrirproject.org/Luker/libRaptorQ.git $ git clone --recurse-submodules \
\end{verbatim} https://github.com/LucaFulchir/libRaptorQ.git
\end{lstlisting}
\index{GPG}\marginlabel{GPG verification:} \index{GPG}\marginlabel{GPG verification:}
Once you have cloned it, it's always a good thing to check the repository gpg Once you have cloned it, it's always a good thing to check the repository gpg
signatures, so you can import my key with: signatures, so you can import my key with:
\begin{verbatim} \begin{lstlisting}[language=bash,breaklines=true]
$ gpg --keyserver pgp.mit.edu --recv-key F61F6137 $ gpg --keyserver pgp.mit.edu --recv-key \
\end{verbatim} 7393DAD255BEB5751DBDA04AB11CD823BA278C85
\end{lstlisting}
please check the full fingerprint, it should be this:
\begin{verbbox}[\footnotesize]
$ gpg2 --fingerprint F61F6137
pub rsa2048/F61F6137 2016-01-11 [SC] [expires: 2017-01-25]
Key fingerprint = 95EB 6FA4 03D7 29A7 13C2 FC9F F85A DA1C F61F 6137
uid [ultimate] Luca Fulchir (2016 key) <luker@fenrirproject.org>
sub rsa2048/0B37F4EE 2016-01-11 [E] [expires: 2017-01-25]
\end{verbbox}
\theverbbox
Now you have the source, and the key, it's enough to check the signature of the Now you have the source, and the key, it's enough to check the signature of the
last commit: last commit:
...@@ -149,37 +144,42 @@ $ git log -n 1 --show-signature ...@@ -149,37 +144,42 @@ $ git log -n 1 --show-signature
The important part is that you get something like this: The important part is that you get something like this:
\begin{verbbox}[\footnotesize] \begin{verbbox}[\footnotesize]
gpg: Signature made Mon 30 May 2016 18:38:43 CEST using RSA key ID F61F6137 gpg: Signature made Mon 11 Dec 2017 21:55:28 CET
gpg: Good signature from "Luca Fulchir (2016 key) <luker@fenrirproject.org>" gpg: using RSA key 8F7474044095B405D0F042F0A2CCA134BC7C8572
[unknown] gpg: Good signature from "Luca Fulchir <luker@fenrirproject.org>" [unknown]
gpg: aka "Luca Fulchir <luca@fulchir.it>" [unknown]
gpg: WARNING: This key is not certified with a trusted signature! gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner. gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: 95EB 6FA4 03D7 29A7 13C2 FC9F F85A DA1C F61F 6137 Primary key fingerprint: 7393 DAD2 55BE B575 1DBD A04A B11C D823 BA27 8C85
Author: Luca Fulchir (2016 key) <luker@fenrirproject.org>
\end{verbbox} \end{verbbox}
\theverbbox \theverbbox
And as long as you got the right key, and you find the \textbf{"gpg: Good signature"} string, And as long as you got the right key, and you find the \textbf{"gpg: Good signature"} string,
you can be sure you have the right code. you can be sure you have the right code.
The main development happens in the $master$ branch, while we keep one branch for each stable version.
\subsection{Dependencies}\index{dependencies} \subsection{Dependencies}\index{dependencies}
libRaptorQ has 3 dependencies: libRaptorQ has 4 dependencies:
\begin{description} \begin{description}
\item[\textbf{Eigen3}] This is used for matrix manipulation, which is a big part of RaptorQ. \item[\textbf{Eigen3}] This is used for matrix manipulation, which is a big part of RaptorQ.
\item[\textbf{lz4}] Used to compress cached precomputations. \item[\textbf{lz4}] Used to compress cached precomputations.
\item[\textbf{git}] This is used not only to get the source, but also by the build system. We get the last git commit id and feed it to clang or gcc as seed for their \item[\textbf{git}] This is used not only to get the source, but also by the build system. We get the last git commit id and feed it to clang or gcc as seed for their internal random number generator. This makes it possible to have reproducible builds.
internal random number generator. This makes it possible to have reproducible builds. \item[\textbf{optionparser}] Library used to parse the command line easily in C++, without exceptions.
\end{description} \end{description}
All dependencies are included in the sources, so you do not need to download and compile them.\\ All dependencies are included in the sources, so you do not need to download and compile them.\\
Eigen3 is only needed at build time, so there is no need to download it again. we use the 3.2.8 version.\\ Eigen3 is only needed at build time, so there is no need to download it again. we use the 3.3.4 version.\\
LZ4 is included as a git submodule. if you do not have it, run: LZ4 is included as a git submodule. if you do not have it, run:
\begin{verbatim} \begin{verbatim}
$ git submodule init $ git submodule init
$ git submodule update $ git submodule update
\end{verbatim} \end{verbatim}
and LZ4 will be build statically in the library (it will not be installed on the system) and LZ4 w
ill be built
statically in the library (it will not be installed on the system)
\subsection{Build \& Install} \subsection{Build \& Install}
...@@ -188,6 +188,7 @@ libRaptorQ uses the cMake build system, so things are fairly standard: ...@@ -188,6 +188,7 @@ libRaptorQ uses the cMake build system, so things are fairly standard:
\begin{verbatim} \begin{verbatim}
$ cd libRaptorQ.git $ cd libRaptorQ.git
$ mkdir build $ mkdir build
$ cd build
$ cmake ../ $ cmake ../
$ make -j 4 $ make -j 4
\end{verbatim} \end{verbatim}
...@@ -210,9 +211,12 @@ Profiling compiles everything once, then runs a test to see which code paths are ...@@ -210,9 +211,12 @@ Profiling compiles everything once, then runs a test to see which code paths are
recompiles everything again, but making sure that the binary is optimized for those paths. Works with clang and gcc. Provides a slight speedup. recompiles everything again, but making sure that the binary is optimized for those paths. Works with clang and gcc. Provides a slight speedup.
\item[CMAKE\_C\_COMPILER] gcc and clang are directly supported. other should work, too. This is only used if you want to build the C example. \item[CMAKE\_C\_COMPILER] gcc and clang are directly supported. other should work, too. This is only used if you want to build the C example.
\item[CMAKE\_CXX\_COMPILER] g++, clang++ are directly supported. other should work, too. \item[CMAKE\_CXX\_COMPILER] g++, clang++ are directly supported. other should work, too.
\item[CLANG\_STDLIB] use clang's libc++ standard library. Only available with clang. \item[CLANG\_STDLIB] ON/OFF. Default:\textbf{OFF}.use clang's libc++ standard library. Only available with clang.
\item[CMAKE\_BUILD\_TYPE] Type of build. you can choose between ``Debug'', ``Release'', ``MinSizeRel'' or ``RelWithDebInfo'' \item[CMAKE\_BUILD\_TYPE] Type of build. you can choose between ``Debug'', ``Release'', ``MinSizeRel'' or ``RelWithDebInfo''
\item[CMAKE\_INSTALL\_PREFIX] Default to \textit{/usr/local}. Change it to fit your distribution guidelines. \item[CMAKE\_INSTALL\_PREFIX] Default to \textit{/usr/local}. Change it to fit your distribution guidelines.
\item[RQ\_ENDIANNESS] \texttt{Auto,BigEndian,LittleEndian} Force the build to use one endianness ot the other
\item[RQ\_LINKER] \texttt{Auto,gold,ld,bsd} Force the build to use one linker or the other
\item[USE\_LZ4] ON/OFF. Default:\textbf{ON}. compile with support for lz4
\end{description} \end{description}
Then you can build everything by running: Then you can build everything by running:
...@@ -267,45 +271,50 @@ It is still very quick. On a core i7, 2.4Ghz, you need to wait \textit{0.4ms} fo ...@@ -267,45 +271,50 @@ It is still very quick. On a core i7, 2.4Ghz, you need to wait \textit{0.4ms} fo
RaptorQ handles up to $56.403$ symbols. RaptorQ handles up to $56.403$ symbols.
\marginlabel{\textbf{Caching}} libRaptorQ can work with big matrices that take a lot of time to compute. For this reason the matrices can be saved once they have \marginlabel{\textbf{Caching}} libRaptorQ can work with big matrices that take a lot of time to compute. For this reason the matrices can be saved once they have
been computed the first time. libRaptorQ uses a \textbf{shared} and a \textbf{local} cache. In both cases the matrix is compressed with LZ4.\\ been computed the first time. libRaptorQ uses a \textbf{local} cache. The matrix can be compressed with LZ4.
Shared cache uses shared memory and is thus shared with all processes that use the save version of libRaptorQ. Local cache is only local to the same process.
\subsection{Blocks \& Symbols} \subsection{RaptorQ: Blocks \& Symbols}
To understand how to properly use and tune libRaptorQ, you first need to understand how RaptorQ handles its inputs, outputs, and what the time and memory To understand how to properly use and tune libRaptorQ, you first need to understand how RaptorQ handles its inputs, outputs, and what the time and memory
constraints are. constraints are.
\index{Sequencing}\marginlabel{Input sequencing:} \index{Sequencing}\marginlabel{Input sequencing:}
RaptorQ needs to have the whole input you want to send before it can start working.\\ The RC6330 API needs to have the whole input you want to send before it can start working.\\
This means that it might be a little more difficult to use in live-streaming contexts, or where you need real-time data, but libRaptorQ will have options to This means that You should buffer the data you are sending, and then when you have a block big enough you pass it to libRaptorQ, which will start generating symbols.
facilitate usage even in those contexts.
Once you have the whole input, RaptorQ divides it into \textbf{blocks}. Each block \textit{is encoded and decoded independently} and will be divided into \textbf{symbols}. Each symbol \textit{should} Contrary to the RFC6330, the RAW api is about a single block of data. You will have to divide your input in multiple blocks by hand, and decide by yourself how to send the encoded symbols. This will increase your flexibility, at the expense of not being conformant to RFC6330.
be transmitted separately in its own packet (if you are on the network). Note that a RaptorQ block does not need to be completely filled since the block is automatically padded with zeros, but you need to somehow tell the receiver how much data you have in that block so that it can know where the padding starts. RFC6330 does this with the \texttt{OTI} parameters. \textbf{with the RAW API you will have to do it by yourself}.
-
\index{Sequencing!Sizes}\marginlabel{Sizes:}Each input can have up to \textit{256 blocks}, each block can have up to \textit{$56.403$ symbols}, and each \index{Sequencing!Sizes}\marginlabel{Sizes:} Each block can have up to \textit{$56.403$ symbols}, and each
symbol can have up to \textit{$2^{16}-1$ bytes}long. This gives a maximum files size of almost $881$ GB (946270874880 bytes to be exact) symbol can have up to \textit{size\_t bytes} long ($2^{32}$ on 32 bit machines, $2^{
64}$ on 64 bits
). Please remember the Complexity discussion before using high values.
\index{Interleaving}\marginlabel{Interleaving:}
An other feature of RaptorQ is to automatically provide some interleaving of the input data before transmitting it. This means that one symbol will not
represent one sequential chunk of your input data, but more likely it's the concatenation of different \textbf{sub-symbols}. The size of the subsymbol must thus
be a fraction of the symbol size. This feature is not used if you set the size of the subsymbol to the size of symbol.
\index{Interleaving}\marginlabel{Interleaving:}
Contrary to the RFC6330 the RAW api does not include any kind of interleaving
\marginlabel{Memory and Time:} \marginlabel{Memory and Time:}
Memory and time requirements are to be considered, though, as RaptorQ needs to run a cubic algorithm on matrix of size $K*K$, where $K$ is the number of Memory and time requirements are to be considered since RaptorQ needs to run a cubic algorithm on matrix of size $K*K$, where $K$ is the number of symbols in each block.\\
symbols in each block.\\
The algorithm needs to keep in memory two of these matrices, although most of the work is done on only one.\\ The algorithm needs to keep in memory two of these matrices, although most of the work is done on only one.\\
This is actually a lot. More benchmarks and optimizations will come later, for now remember that with 10 symbols it takes something like 0.4ms on a core i7 2.4GHZ, 280ms with 1000 symbols, and up to an hour with 27.000 symbols. This is actually a lot. More benchmarks and optimizations will come later, for now remember that with 10 symbols it takes something like 0.4ms on a core i7 2.4GHZ, 280ms with 1000 symbols, and up to an hour with 27.000 symbols. YMMV.
\subsection{C++ interface} \subsection{C++ interface}
\index{Interface!C++} \index{Interface!C++}
To use the C++ interface you only need to include the \textbf{RaptorQ.hpp} header, and link \textbf{libRaptorQ} and your threading library (usually To use the C++ interface you only need to include the \textbf{RaptorQ/RaptorQ\_v1.hpp} header, and link \textbf{libRaptorQ} and your threading library (usually \textit{libpthread}).
\textit{libpthread}).
You can also use the library as a header-only include, by including \textbf{RaptorQ/RaptorQ\_v1\_hdr.hpp}. All your code should keep working even if you switch between linked and header-only at a later date. If you use the header-only version, remember to have Eigen3 in your include path, and to include the needed definitions, \texttt{RQ\_LITTLE\_ENDIAN} or \texttt{RQ\_BIG\_ENDIAN}
\marginlabel{Namespace}
Since the libRaptorQ library now has two very different usages, everything was split into two namespaces: \textbf{RFC6330} and \textbf{RaptorQ}. The names should be explanatory: the first namespace is for the RFC interface, while the second namespace is for the low-level interface.
To provide grater flexibility, the whole library uses iterators to read your data, and to write the data onto your data structures.\\ Since we will cover only the \textbf{RaptorQ} namespace here, remember that everything we talk about is in this namespace.
This means that a big part of the library is a template, which adapts to the alignment of the data in the data structures you use.
Both the RFC6330 and the RaptorQ namespace are versioned so you can keep using old APIs while we change stuff. just add \textbf{\texttt{\_\_v1}} to select the first version of the API. E.g: \textbf{\texttt{RaptorQ\_\_v1}}
\newpage
\index{Iterators}\marginlabel{Templates} \index{Iterators}\marginlabel{Templates}
There are two main classes you will use: There are two main classes you will use:
...@@ -318,57 +327,66 @@ class Decoder ...@@ -318,57 +327,66 @@ class Decoder
\end{verbatim} \end{verbatim}
As you might guess, the classes for the encoder and decoder take two template parameters.\\ As you might guess, the classes for the encoder and decoder take two template parameters.\\
For the \textbf{Encoder}, the first parameter \textit{MUST} be a \textit{random access iterator} to the input data, and the second parameter is an For the \textbf{Encoder}, the first parameter \textit{MUST} be a \textit{random access iterator} to the input data, and the second parameter is a
\textit{forward iterator}. The random access iterator will be used to scan your input data, and perform an interleaving (if you did not set the same size the \textit{forward iterator}. The random access iterator will be used to scan your input data. The forward iterator will be used to write the data to your output.\\
symbol and to the subsymbol). The forward iterator will be used to write the data to your structure.\\ The same is done for the \textbf{Decoder}, but the first iterator can be just an input iterator,
The same is done for the \textbf{Decoder}, but we do not need to do any interleaving on the input, so the first iterator can be just an input iterator,
and nothing more. and nothing more.
\subsection{Caching precomputations} \subsubsection{Caching precomputations}
\index{Cache!C++'} \index{Cache!C++'}
libRaptorQ can now cache the most used matrices for a quicker reference.\\ libRaptorQ can now cache the most used matrices for a quicker reference. The cache can be scaled up or down dynamically.
There are two level of caching: \textit{local} and \textit{global}.\\
\textbf{Global} caching will cache precomputed matrices for all programs who use your version of libRaptorQ in the system.
This uses shared memory to store the matrices. The shared memory is automatically increased to the maximum size any program requires,
and automatically scaled down once the program is not being executed anymore.
\textbf{Local} caching instead works only in the program, and is not shared with any other application. Once the shared cache is checked,
local caches is checked. The cache can be scaled up or down as needed dynamically.
\begin{description} \begin{description}
\item[shared\_cache\_size] \textbf{Input: const uint64\_t shared\_cache}\\
\textbf{return: uint64\_t}\\
Set the maximum shared cache size for this process. Default: 0 (do not use shared cache)
\item[local\_cache\_size] \textbf{const uint64\_t local\_cache}\\ \item[local\_cache\_size] \textbf{const uint64\_t local\_cache}\\
\textbf{return: bool}\\ \textbf{return: bool}\\
Set the local cache size. returns false on error. Set the local cache size. returns false on error.
\item[get\_shared\_cache\_size()] \textbf{return: uint64\_t}\\
get the size of our shared memory requirement. \textit{NOTE: the actual size might be larger,
since other processes might have requested more}
\item[get\_local\_cache\_size()] \textbf{return: uint64\_t}\\ \item[get\_local\_cache\_size()] \textbf{return: uint64\_t}\\
get the size of our local cache get the size of our local cache
\item[supported\_compressions()] \textbf{return: Compress} \\
Get the bitmask of all supported compression algorithms. currently only \textbf{Compress::NONE} and \textbf{Compress::LZ4}.
\item[get\_compression()] \textbf{return: Compress} \\
Get the current compression used.
\item [set\_compression] \textbf{const Compress compression} \\
\textbf{return: bool}\\
Try to set a compression algorithm. LZ4 is not a mandatory dependency, so we might not have it
\end{description} \end{description}
\newpage
\subsubsection{The Encoder} \subsubsection{The Encoder}
\index{Encoder!C++} \index{Encoder!C++}
The constructor is the following:
\begin{lstlisting}[language=C++]
Encoder (const Block_Size symbols, const size_t symbol_size);
\end{lstlisting}
Which is really simple. The first is an Enum that will help you get the correct block size, and the second is your arbitrary symbol size.
Since the RaptorQ algorithm only works with specific block sizes, You can get all the possible block sizes from \texttt{RaptorQ\_\_v1::*blocks}, which is a pointer to an array of 477 possible sizes, from $10$ to $56403$. You can cast each element in this array to \texttt{uint16\_t} to verify the actual value, or directly name them, as in \texttt{RaptorQ\_\_v1::Block\_Size::Block\_42}.
If you pass unsafe values the encoder (or decoder) will refuse to work.
You can instantiate an encoder for example by doing: You can instantiate an encoder for example by doing:
\begin{lstlisting}[language=C++] \begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output; std::vector<uint32_t> input, output;
... ...
using T_it = typename std::vector<uint32_t>::iterator; using T_it = typename std::vector<uint32_t>::iterator;
RaptorQ::Encoder<T_it, T_it> enc (input.begin(), using RaptorQ = RaptorQ__v1;
input.end(), RaptorQ::Encoder<T_it, T_it> enc (
4, 1444, 10000) RaptorQ::Block_Size::Block_42,
1500);
\end{lstlisting} \end{lstlisting}
This will create an Encoder that works on vectors of unsigned 32 bit integers for both input and output, that will create symbols of size 1444 bytes, interleaving This will create an Encoder that works on vectors of unsigned 32 bit integers for both input and output, that will be fed to a block of size 42
your input every 4 bytes, and try to work with big number of symbols per blocks (\textbf{TODO: explain memory requirements}) symbols of size 1500 bytes.
The available methods for the encoder are the following: The available methods for the encoder are the following:
...@@ -376,337 +394,202 @@ The available methods for the encoder are the following: ...@@ -376,337 +394,202 @@ The available methods for the encoder are the following:
\item[operator bool()] \textbf{return:bool}\\ \item[operator bool()] \textbf{return:bool}\\
False if constructor parameters did not make sense. Else true. False if constructor parameters did not make sense. Else true.
\item[OTI\_Common()] \textbf{return: OTI\_Common\_Data}, aka \textit{uint64\_t}.\\
Keeps total \textbf{file size and symbol size}. You need to send this to the receiver, so that it will be able to properly decode the data.
\item[OTI\_Scheme\_Specific\_Data()] \textbf{return: OTI\_Scheme\_Specific\_Data}, aka \textit{uint32\_t}.\\ \item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol.
Keeps number of \textbf{source blocks, sub blocks, and alignment}. As for the OTI\_Common\_Data, you need to send this to the receiver to be able to
properly decode the data.
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t esi, const uint8\_t sbn}.\\ \item[symbols] \textbf{Input:uint8\_t sbn}\\
\textbf{return:uint64\_t}.\\ \textbf{return: uint16\_t}\\
Take as input the iterators to the data structure into where we have to save the encoded data, the \textbf{Encoding Symbol Id} and the The number of symbols in a specific block. different blocks can have different symbols.
\textbf{Source Block Number}. As you are writing in C++, you probably want to use the iterators begin/end, though. Returns the number of written
iterators (\textbf{NOT} the bytes)
\item[encode] \textbf{Input: Fwd\_It \&output, const Fwd\_It end, const uint32\_t id}.\\ \item[max\_repair] \textbf{Input: const uint8\_t sbn)}\\
\textbf{return:uint64\_t}.\\ \textbf{return: uint32\_t}\\
Exactly as before, but the \textbf{id} contains both the \textit{source block number} and the \textit{encoding symbol id} The maximum amount of repair symbols that you can generate. Something less than $2^{24}$, but the exact number depends on the number of symbols in a block
\item[begin()] \textbf{return: Block\_Iterator<Rnd\_It, Fwd\_It>}\\ \item[begin\_source()] \textbf{return: Symbol\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the blocks in which RaptorQ divided the input data. See later to understand how to use it. This returns an iterator to the symbols in this block. Source symbols only
\item[end()] \textbf{return: const Block\_Iterator<Rnd\_It, Fwd\_It>}\\ \item[end\_source()] \textbf{return: const Symbol.\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the end of the blocks in which RaptorQ divided the input data. See later to understand how to use it. This returns an iterator to the end of the source symbols.
\item[begin\_repair()] \textbf{return: Symbol\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the symbols in this block. Source symbols only
\item[end\_repair] \textbf{Input: const uint32\_t max\_repair}\\
\textbf{return: const Symbol.\_Iterator<Rnd\_It, Fwd\_It>}\\
This returns an iterator to the end of the source symbols.
\item[precompute] \textbf{Input:const uint8\_t threads, const bool background}\\ \item[has\_data()] \textbf{return: bool}\\
\textbf{return: void}\\ Did you feed this encoder the data? That is: did you use \texttt{set\_data()} without calling \texttt{clear\_data()} later?
Do the work of computing all different blocks in multithread. If \textit{background} is true, then return immediately, else return only when the job is done.\\
If \textit{threads} is $0$, try to guess the maximum threads from the number of available cpus.
\item [precompute\_max\_memory] \textbf{return: size\_t}\\ \item[set\_data] \textbf{Input: const Rnd\_It \&from}\\
Each precomputation can take a lot of memory, depending on the configuration, so you might want to limit the number of precomputations run in parallel .\ \ \ \ \ \ \ \ \ \ \textbf{const Rnd\_It \&to}\\
depending on the memory used. This method returns the amount of memory taken by \textbf{ONE} precomputation. \textbf{return: size\_t}\\
Set the iterators from which we load all the data.
\item [free] \textbf{Input: const uint8\_t sbn}\\ \item[clear\_data]\textbf{return: void}\\
\textbf{return: void}\\ clear all data, without deallocating memory so that things might be slightly faster next time.
Each block takes some memory, (a bit more than $symbols * symbol_size$), so once you are done sending source and repair symbols for one block,
you might want to free the memory of that block.
\item[blocks()] \textbf{return: uint8\_t} The number of blocks. \item[ready()]\textbf{return: bool}\\
Is everything decoded?
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\ \item[stop()]\textbf{return void}\\
\textbf{return: uint32\_t}\\ Stop any current decoding and return error.
The block size, in bytes. Each block can have different symbols and thus different size.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol. \item[precompute\_sync()]\textbf{return: bool}\\
Start a precomputation so that next computations might be slightly faster. Only return when the precomputation is done or stop() was called.
\item[symbols] \textbf{Input:uint8\_t sbn}\\ \item[compute\_sync()]\textbf{return: bool}\\
\textbf{return: uint16\_t}\\ Compute internal symbols from the input. Only return when the computation is done or stop() was called.
The number of symbols in a specific block. different blocks can have different symbols.
\item[max\_repair] \textbf{Input: const uint8\_t sbn)}\\ \item[precompute()] \textbf{return: std::shared\_future<Error>}\\
\textbf{return: uint32\_t}\\ Start a precomputation so that next computations might be slightly faster. Returns immediately, but the user can wait or poll on the returned future.
The maximum amount of repair symbols that you can generate. Something less than $2^{24}$, but the exact number depends on the number of symbols
in a block
\end{description}
~\\ \item[compute()] \textbf{return: std::shared\_future<Error>}\\
Start a computation on the data. Returns immediately, but the user can wait or poll on the returned future.
\subsubsection{Blocks} \item[encode]\textbf{Input:Fwd\_It \&output}\\
\index{Blocks!C++} .\ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
With the \textit{begin()/end()} calls you get \textit{Input iterators} to the blocks. a Block is has the following type: .\ \ \ \ \ \ \ \ \textbf{const uint32\_t id}\\
\begin{lstlisting}[language=C++] \textbf{return: size\_t}\\
template <typename Rnd_It, typename Fwd_It> Once the computation is finished, you can get the encoded symbols. The first \texttt{symbols()} are source symbols, and the next are repair symbols. Returns the number of iterators written into the \texttt{output} iterator, which will point to the first non-written element after the symbol.
class Block
\end{lstlisting}
and exposes the 4 following methods:
\begin{description}
\item[begin\_source]\textbf{return: Symbol\_Iterator}
\item[end\_source]\textbf{return: Symbol\_Iterator}
\item[begin\_repair]\textbf{return: Symbol\_Iterator}
\item[end\_repair]\textbf{Input: const uint32\_t max\_repair}\\
\textbf{return: Symbol\_Iterator}
\item[max\_repair]\textbf{return:uint32\_t}
\item[symbols]\textbf{return: uint16\_t}
\item[block\_size]\textbf{return: uint32\_t}
\end{description} \end{description}
As the names explain, you will get an iterator to the symbols in the block. As the number of repair symbols can vary, for now you get two separate begin/ends,
so that you can check when you sent the source symbols, and how many repair symbols you send.\\
The other functions are helpers for the details of the block.
\subsubsection{Symbols} \subsubsection{Symbols}
\index{Symbols!C++} \index{Symbols!C++}
Finally, through the \textit{Symbol\_Iterator} \textit{Input Iterator} we get the \textbf{Symbol} class: With the \textit{begin\_source()/end\_source()/begin\_repair()/end\_repair(..)} calls you get \textit{input iterators} to the symbols. A symbol has the following type:
\begin{lstlisting}[language=C++] \begin{lstlisting}[language=C++]
template <typename Rnd_It, typename Fwd_It> template <typename Rnd_It, typename Fwd_It>
class Symbol class Symbol
\end{lstlisting} \end{lstlisting}
and exposes the following methods:
which exposes the 2 methods we need to get the symbol data:
\begin{description} \begin{description}
\item[operator*]\textbf{Input:Fwd\_It \&start, const Fwd\_It end}\\ \item[id()]\textbf{return: uint32\_t}
\textbf{return: uint64\_t}\\ \item[operator()]\textbf{Input: Fwd\_It \&start}\\
takes a forward iterator, and fill it with the symbol data. returns the number of written iterators. .\ \ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
\item[id()]\textbf{return: uint32\_t}\\ \textbf{return: uint32\_t}\\
return the id (\textit{$sbn + esi$}) of this symbol, that you need to include in every packet you send, before the symbols. Write the symbol into the start-end iterators provided by the user. Afterwards the start iterator will point to the first non-written element after the symbol
\end{description} \end{description}
\newpage
\subsubsection{The Decoder} \subsubsection{The Decoder}
\index{Decoder!C++} \index{Decoder!C++}
The decoder is a bit simpler than the encoder. There are two constructors for the Decoder:
Theere are two constructors for the Decoder:
\begin{lstlisting}[language=C++] \begin{lstlisting}[language=C++]
std::vector<uint32_t> input, output; std::vector<uint32_t> input, output;
using T_it = typename std::vector<uint32_t>::iterator; using T_it = typename std::vector<uint32_t>::iterator;
RaptorQ::Decoder<T_it, T_it> dec (
const OTI_Common_Data common,
const OTI_Scheme_Specific_Data scheme)
RaptorQ::Decoder<T_it, T_it> dec (uint64_t size,
uint16_t symbol_size,
uint16_t sub_blocks,
uint8_t blocks)
\end{lstlisting}
Which should be pretty self-explanatory, once you understand how the encoder works.
The remaining methods are:
\begin{description}
\item[decode]\textbf{Input: Fwd\_It \&start, const Fwd\_It end}\\
\textbf{return:uint64\_t}\\
Write \textbf{all} the blocks into the iterator. refuses to write if the input has not been completely received. Return the number of iterators written.
\item[decode]\textbf{Input: Fwd\_It \&start, const Fwd\_It end, const uint8\_t sbn}\\
\textbf{return:uint64\_t}\\
Write a specific block into the iterator. Refuses to write if the input for that block has not been completely received.
\item[add\_symbol]\textbf{In\_It \&start, const In\_It end, const uint32\_t esi, const uint8\_t sbn}\\
\textbf{return: bool}\\
Add one symbol, while explicitly specifying the symbol id and the block id.
\item[add\_symbol]\textbf{In\_It \&start, const In\_It end, const uint32\_t id}\\
\textbf{return: bool}\\
Same as before, but extract the block id and the symbol id from the \textit{id} parameter
\item[free]\textbf{Input: const uint8\_t sbn}\\
\textbf{return: void}\\
You might have stopped using a block, but the memory is still there. free it.
\item[blocks()] \textbf{return: uint8\_t} The number of blocks.
\item[block\_size()] \textbf{Input: const uint8\_t sbn}\\
\textbf{return: uint32\_t}\\
The block size, in bytes. Each block can have different symbols and thus different size.
\item[symbol\_size()] \textbf{return: uint16\_t} The size of a symbol. RaptorQ__v1::Decoder<T_it, T_it> dec (
const Block_Size symbols,
\item[symbols] \textbf{Input:uint8\_t sbn}\\ const size_t symbol_size,
\textbf{return: uint16\_t}\\ const Report type);
The number of symbols in a specific block. different blocks can have different symbols.
\end{description}
\newpage
\subsection{C interface}\index{Interface!C}
The C interface looks a lot like the C++ one.\\
You need to include the \textbf{cRaptorQ.h} header, and link the \textit{libRaptorQ} library.\\
\marginlabel{Static linking}\index{Static Linking}
If you are working with the static version of libRaptorQ remember to link the C++ standard library used when
compiling the library (\textit{libstdc++} for gcc or maybe \textit{libc++} for clang), your threding library (usually \textit{libpthread}), and the C math library (\textit{libm}).\\
As for the C++11 interface, you can get and set the shared and local memory cache.\\
Shared cache uses shared memory, while local cache can be used only be the current process.
\index{Cache!C}
\marginlabel{Cache settings}
\begin{lstlisting}[language=C]
uint64_t RaptorQ_shared_cache_size(
const uint64_t shared_cache);
bool RaptorQ_local_cache_size (
const uint64_t local_cache);
uint64_t RaptorQ_get_shared_cache_size ();
uint64_t RaptorQ_get_local_cache_size ();
\end{lstlisting}
To use libRaptorQ you need to build the encoder or the decoder.
The C interface is just a wrapper around the C++ code, so you still have to specify the same things as before.
A quick glance at the constructors should give you all the information you need:
\marginlabel{C Constructors}
\begin{lstlisting}[language=C]
typedef enum { RQ_NONE = 0, RQ_ENC_8 = 1, RQ_ENC_16 = 2,
RQ_ENC_32 = 3, RQ_ENC_64 = 4, RQ_DEC_8 = 5,
RQ_DEC_16 = 6, RQ_DEC_32 = 7, RQ_DEC_64 = 8}
RaptorQ_type;
struct RaptorQ_ptr
{
void *ptr = nullptr;
const RaptorQ_type type;
};
RaptorQ_ptr* RaptorQ_Enc (const RaptorQ_type type,
void *data,
const uint64_t size,
const uint16_t min_subsymbol_size,
const uint16_t symbol_size,
const size_t max_memory);
RaptorQ_ptr* RaptorQ_Dec (const RaptorQ_type type,
const RaptorQ_OTI_Common_Data common,
const RaptorQ_OTI_Scheme_Specific_Data
scheme);
\end{lstlisting} \end{lstlisting}
The encoder and decoder must have a specific alignment. in C++ you can also have different The only parameter that is not self-explanatory is the Report type: it regulates when the decoder will tell you that there is available data and will let you extract it.\\
alignments for the input and output, while in C things are a bit more strict as we have to enumerate The possible values are:
all the possible cases. So you only get the same data alignment for both input and output.\\
Still, you don't lose anything in performance.\\
\begin{itemize}
\item \textbf{Report::PARTIAL\_FROM\_BEGINNING} report the data as it become available, but the data will be sequential
\item \textbf{Report::PARTIAL\_ANY} report any symbol as it becomes available. Could be out-of-order, and you can get symbols before the decoding starts.
\item \textbf{Report::COMPLETE} wait until the whole block has been decoded.
\end{itemize}
\subsubsection{Common functions for (de/en)coding}
These functions are used by both the decoder and the decoder, and will be helpful in tracking how much memory The decoding methods are:
you will need to allocate, or in general in managing the encoder and decoder.
The \textbf{ptr} must be a valid encoder or decoder. \begin{description}
The names for now are self-explanatory. Blocks can have different symbols, so the size of a block and the number of symbols in a block \item[symbols()]\textbf{return: uint16\_t}\\
depend on which block we are talking about. The number of symbols in this block
\item[symbol\_size()]\textbf{return: size\_t}\\
\marginlabel{symbols, blocks, memory} The bytes in each symbol
\begin{lstlisting}[language=C] \item[can\_decode()]\textbf{return: bool}\\
uint16_t RaptorQ_symbol_size (struct RaptorQ_ptr *ptr); Whether we have enough data to start decoding or not
uint8_t RaptorQ_blocks (struct RaptorQ_ptr *ptr); \item[ready()]\textbf{return: bool}\\
uint32_t RaptorQ_block_size (struct RaptorQ_ptr *ptr, Whether everything has been decoded or not.
const uint8_t sbn); \item[needed\_symbols()]\textbf{uint16\_t}\\
uint16_t RaptorQ_symbols (struct RaptorQ_ptr *ptr, How many more symbols we need before we try to decode again. If this is 0, you can decode.
const uint8_t sbn); \item[clear\_data()]\textbf{return: void}\\
clear all data, without deallocating memory so that things might be slightly faster next time.
void RaptorQ_free (struct RaptorQ_ptr **ptr); \item[stop()]\textbf{return: void}\\
void RaptorQ_free_block (struct RaptorQ_ptr *ptr, Stop any decoding done in the background.
const uint8_t sbn);
\end{lstlisting} \item[begin()] \textbf{return: Symbol\_Iterator<In\_It, Fwd\_It>}\\
Finally, when you are done working with a block, you can free the memory associated with the single block This returns an iterator to the symbols in this block.
and just free the whole (en/de)coder when you are done. Freeing the whole (en/de)coder will obviously free \item[end()] \textbf{return: const Symbol\_Iterator<In\_It, Fwd\_It>}\\
also all the blocks. This returns an iterator to the end of the symbols in this block
\subsubsection{Encoding} \item[add\_symbols] \textbf{Input: In\_it \&from}\\
.\ \ \ \ \ \ \ \ \ \ \textbf{const In\_it \&to}\\
\marginlabel{OTI Data} .\ \ \ \ \ \ \ \ \ \ \textbf{const uint32\_t esi}\\
First, we need to tell the receiver all the parameters that the encoder is using, and for that two functions are provided: \textbf{return: RaptorQ\_\_v1::Error}\\
Add one symbol to this block. If the symbol is less than \texttt{symbol\_size()} the symbol will be padded with zeros
\item[end\_of\_input] \textbf{Input: const Fill\_With\_Zeros fill}\\
\textbf{return: std::vector<bool>}\\
Tell the decoder that we know that there will be no more data for this block.
The first parameter is an \texttt{enum}, either \textbf{Fill\_With\_Zeros::YES} or \textbf{Fill\_With\_Zeros::NO}
If you choose \texttt{YES} then the repair symbols will be discarded, the missing symbols will be filled with zeros
and for output you will get the bitmask for which byte was received (true) or was filled with zeros (false).\\
If you choose \textbf{Fill\_With\_Zeros::NO} you will get \textit{an empty vector}, and the decoding will end with error if it was waiting for more input.
\item[set\_max\_concurrency] \textbf{Input: const uint16\_t max\_threads}\\
\textbf{return: void}\\
The decoder can try to decode the same block multiple times, while more repair symbols arrive.
Unless you really need it, leave the default, which is $1$, so no decoding concurrency or the same block.
\begin{lstlisting}[language=C] \item[decode\_once()] \textbf{return: RaptorQ\_\_v1::Decoder\_Result}\\
typedef uint64_t RaptorQ_OTI_Common_Data; Try to decode the block, only return once the decoding is finished, do not try again even if more repair symbols arrived.
typedef uint32_t RaptorQ_OTI_Scheme_Specific_Data;
RaptorQ_OTI_Common_Data RaptorQ_OTI_Common ( \item[poll()] \textbf{return: RaptorQ\_\_v1::Decoder\_wait\_res}\\
struct RaptorQ_ptr *enc); \begin{lstlisting}[language=c++]
RaptorQ_OTI_Scheme_Specific_Data RaptorQ_OTI_Scheme ( struct Decoder_wait_res {
struct RaptorQ_ptr *enc); Error error;
uint16_t symbol;
};
\end{lstlisting} \end{lstlisting}
In general, the \texttt{error} can be \textbf{Error::WORKING}, \texttt{Error::NEED\_DATA}, \texttt{Error::NONE}. The last one signals that the block (or symbol, see later) has been decoded.
\index{Encoder!C}\marginlabel{Encoding} The \texttt{symbol} parameter changes depending on the \textbf{RaptorQ\_\_v1::Report} type that you set when you constructed the decoder:
\begin{itemize}
\begin{lstlisting}[language=C] \item \textbf{Report::PARTIAL\_FROM\_BEGINNING}: \texttt{symbols} will be the number of symbols available, but counting only the sequential ones starting from the beginning
// maximum number of repair symbol in a block \item \textbf{Report::PARTIAL\_ANY}: \texttt{symbols} will be the first symbol available (if \texttt{error} == \texttt{Error::NONE}). If the block has been decoded \texttt{symbols} == \texttt{decoder.symbols()}
uint32_t RaptorQ_max_repair (RaptorQ_ptr *enc, \item \textbf{Report::COMPLETE} \texttt{symbols} is not used.
const uint8_t sbn); \end{itemize}
//estimate bytes of ram used in the precomputation
// of one block
size_t RaptorQ_precompute_max_memory ( \item[wait\_sync()] \textbf{return: RaptorQ\_\_v1::Decoder\_wait\_res}\\
struct RaptorQ_ptr *enc); Start the decoding, and keep trying or waiting for new data to arrive until either we successfully decode the block or \texttt{stop()} is called. Also stops waiting for data after \texttt{end\_of\_input} is called (but will keep retrying if there already is enough).\\
// do the precomputation. The result is the same as above
void RaptorQ_precompute (struct RaptorQ_ptr *enc,
const uint8_t threads, \item[wait()] \textbf{return: std::future<RaptorQ\_\_v1::Decoder\_wait\_res>}\\
const bool background); Same as before, but returns a future immediately. This call is enabled only if you are compiling with C++11 or later
// encode one symbol. source block number and \item[decode\_symbol()] \textbf{Input: Fwd\_It \&start}\\
// symbol id are in the ``id'' field. .\ \ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
// returns number of alignments written. .\ \ \ \ \ \ \ \ \ \ \ \textbf{const uint16\_t esi} \\
uint64_t RaptorQ_encode_id (struct RaptorQ_ptr *enc, \textbf{return: RaptorQ\_\_v1::Error}\\
void **data, Decode a single symbol. If $end-start$ is less then the symbol size, then you will get a partial symbol.
const uint64_t size, \textbf{return: RaptorQ\_\_v1::Error}\\
const uint32_t id);
// encode one symbol. same as before \item[decode\_bytes] \textbf{Input: Fwd\_It \&start}\\
uint64_t RaptorQ_encode (struct RaptorQ_ptr *enc, .\ \ \ \ \ \ \ \ \ \ \textbf{const Fwd\_It end}\\
void **data, .\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t from\_byte}\\
const uint64_t size, .\ \ \ \ \ \ \ \ \ \ \textbf{const size\_t skip}\\
const uint32_t esi, \textbf{return: Decoder\_written} \\
const uint8_t sbn); Decode a specific range of the block. The \texttt{skip} parameter is the offset (in bytes) to skip from the \texttt{size} iterator. This is done to allow to work with iterators whose data is bigger than 1 byte. Although we recommend to work with uint8\_t data.\\
// build an ``id'' field out of an esi and sbn field. Returns the number of bytes written, offset of the data in the last iterator (0 if it is aligned to the iterator)
uint32_t RaptorQ_id (const uint32_t esi, \begin{lstlisting}[language=c++]
const uint8_t sbn); struct RAPTORQ_API Decoder_written {
size_t written;
size_t offset;
};
\end{lstlisting} \end{lstlisting}
As for the C++ version, everything is thread-safe.
You can start the precomputation in background and not worry about it.\\
If you request repair symbols before the computation is finished, the call will block until the data is available.
The \textbf{encode} functions are the same, and will encode \textbf{one} symbol. \end{description}
They work for both source symbols and repair symbols, just keep increasing the \textit{esi} field
\newpage
\subsubsection{Decoding}
\index{Decoder!C}\marginlabel{Decoding}
\begin{lstlisting}[language=C]
// return the total size of the data that will be
// decoded
uint64_t RaptorQ_bytes (struct RaptorQ_ptr *dec);
// decode all blocks.
// returns the number of written alignments.
// returns 0 if decoding of *everything* is not possible
uint64_t RaptorQ_decode (struct RaptorQ_ptr *dec,
void **data,
const size_t size);
// decode only one block (if possible)
// returns 0 if decoding of the block is not possible.
uint64_t RaptorQ_decode_block (struct RaptorQ_ptr *dec,
void **data,
const size_t size,
const uint8_t sbn);
// add a received symbol to the structure.
// either by using the `id' field
bool RaptorQ_add_symbol_id (struct RaptorQ_ptr *dec,
void **data,
const uint32_t size,
const uint32_t id);
// or by explicitely declaring esi and sbn
bool RaptorQ_add_symbol (struct RaptorQ_ptr *dec,
void **data,
const uint32_t size,
const uint32_t esi,
const uint8_t sbn);
\end{lstlisting}
......
examples/example_cpp_raw.cpp 0 → 100644
View file @ a394b22a
/*
* Copyright (c) 2015-2018, Luca Fulchir<luker@fenrirproject.org>,
* All rights reserved.
*
* This file is part of "libRaptorQ".
*
* libRaptorQ is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3
* of the License, or (at your option) any later version.
*
* libRaptorQ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* and a copy of the GNU Lesser General Public License
* along with libRaptorQ. If not, see <http://www.gnu.org/licenses/>.
*/
#include "../src/RaptorQ/RaptorQ_v1_hdr.hpp"
#include <algorithm>
#include <fstream>
#include <iostream>
#include <limits>
#include <random>
#include <stdlib.h>
#include <vector>
// Demonstration of how to use the C++ RAW interface
// it's pretty simple, we generate some input,
// then encode, drop some packets (source and repair)
// and finally decode everything.
// rename the main namespace for ease of use
namespace RaptorQ = RaptorQ__v1;
// mysize is bytes.
bool test_rq (const uint32_t mysize, std::mt19937_64 &rnd,
float drop_probability,
const uint8_t overhead);
// the "overhead" variable tells us how many symbols more than the
// minimum we will generate. RaptorQ can not always decode a block,
// but there is a small probability that it will fail.
// More overhead => less probability of failure
// overhead 0 => 1% failures
// overhead 1 => 0.01% failures
// overhead 2 => 0.0001% failures
// etc... as you can see, it make little sense to work with more than 3-4
// overhead symbols, but at least one should be considered
bool test_rq (const uint32_t mysize, std::mt19937_64 &rnd,
float drop_probability,
const uint8_t overhead)
{
// the actual input.
std::vector<uint8_t> input;
input.reserve (mysize);
// initialize vector with random data
// distr should be "uint8_t". But visual studio does not like it, so
// we use uint16_t
// generate a random distribution between all values of uint8_t
std::uniform_int_distribution<int16_t> distr (0,
std::numeric_limits<uint8_t>::max());
// fill our input with random data
for (size_t idx = 0; idx < mysize; ++idx) {
input.push_back (static_cast<uint8_t> (distr(rnd)));
}
// the input will be divided in blocks of 4 bytes.
// it's a bit low, but this is just an example.
// NOTE: the symbol size must be a multiple of the container size.
// since sizeof(uint8_t) == 1 and 4 is a multiple of 1, we are safe.
const uint16_t symbol_size = 4; // bytes
// how many symbols do we need to encode all our input in a single block?
auto min_symbols = (input.size() * sizeof(uint8_t)) / symbol_size;
if ((input.size() * sizeof(uint8_t)) % symbol_size != 0)
++min_symbols;
// convert "symbols" to a typesafe equivalent, RaptorQ::Block_Size
// This is needed becouse not all numbers are valid block sizes, and this
// helps you choose the right block size
RaptorQ::Block_Size block = RaptorQ::Block_Size::Block_10;
for (auto blk : *RaptorQ::blocks) {
// RaptorQ::blocks is a pointer to an array, just scan it to find your
// block.
if (static_cast<uint16_t> (blk) >= min_symbols) {
block = blk;
break;
}
}
// now initialize the encoder.
// the input for the encoder is std::vector<uint8_t>
// the output for the encoder is std::vector<uint8_t>
// yes, you can have different types, but most of the time you will
// want to work with uint8_t
RaptorQ::Encoder<typename std::vector<uint8_t>::iterator,
typename std::vector<uint8_t>::iterator> enc (
block, symbol_size);
// give the input to the encoder. the encoder answers with the size of what
// it can use
if (enc.set_data (input.begin(), input.end()) != mysize) {
std::cout << "Could not give data to the encoder :(\n";
return false;
}
// actual symbols. you could just use static_cast<uint16_t> (blok)
// but this way you can actually query the encoder.
uint16_t _symbols = enc.symbols();
// print some stuff in output
std::cout << "Size: " << mysize << " symbols: " <<
static_cast<uint32_t> (_symbols) <<
" symbol size: " <<
static_cast<int32_t>(enc.symbol_size()) << "\n";
// RQ need to do its magic on the input before you can ask the symbols.
// multiple ways to do this are available.
// The simplest is to run the computation and block everything until
// the work has been done. Not a problem for small sizes (<200),
// but big sizes will take **a lot** of time, consider running this with the
// asynchronous calls
if (!enc.compute_sync()) {
// if this happens it's a bug in the library.
// the **Decoder** can fail, but the **Encoder** can never fail.
std::cout << "Enc-RaptorQ failure! really bad!\n";
return false;
}
// the probability that a symbol will be dropped.
if (drop_probability > static_cast<float> (90.0))
drop_probability = 90.0; // this is still too high probably.
// we will store here all encoded and transmitted symbols
// std::pair<symbol id (esi), symbol data>
using symbol_id = uint32_t; // just a better name
std::vector<std::pair<symbol_id, std::vector<uint8_t>>> received;
{
// in this block we will generate the symbols that will be sent to
// the decoder.
// a block of size X will need at least X symbols to be decoded.
// we will randomly drop some symbols, but we will keep generating
// repari symbols until we have the required number of symbols.
std::uniform_real_distribution<float> drop_rnd (0.0, 100.0);
uint32_t received_tot = 0;
// Now get the source symbols.
// source symbols are specials because they contain the input data
// as-is, so if you get all of these, you don't need repair symbols
// to make sure that we are using the decoder, drop the first
// source symbol.
auto source_sym_it = enc.begin_source();
++source_sym_it; // ignore the first soure symbol (=> drop it)
source_sym_it++;
for (; source_sym_it != enc.end_source(); ++source_sym_it) {
// we save the symbol here:
// make sure the vector has enough space for the symbol:
// fill it with zeros for the size of the symbol
std::vector<uint8_t> source_sym_data (symbol_size, 0);
// save the data of the symbol into our vector
auto it = source_sym_data.begin();
auto written = (*source_sym_it) (it, source_sym_data.end());
if (written != symbol_size) {
// this can only happen if "source_sym_data" did not have
// enough space for a symbol (here: never)
std::cout << written << "-vs-" << symbol_size <<
" Could not get the whole source symbol!\n";
return false;
}
// can we keep this symbol or do we randomly drop it?
float dropped = drop_rnd (rnd);
if (dropped <= drop_probability) {
continue; // start the cycle again
}
// good, the symbol was received.
++received_tot;
// add it to the vector of received symbols
symbol_id tmp_id = (*source_sym_it).id();
received.emplace_back (tmp_id, std::move(source_sym_data));
}
std::cout << "Source Packet lost: " << enc.symbols() - received.size()
<< "\n";
//--------------------------------------------
// we finished working with the source symbols.
// now we need to transmit the repair symbols.
auto repair_sym_it = enc.begin_repair();
auto max_repair = enc.max_repair(); // RaptorQ can theoretically handle
// infinite repair symbols
// but computers are not so infinite
// we need to have at least enc.symbols() + overhead symbols.
for (; received.size() < (enc.symbols() + overhead) &&
repair_sym_it != enc.end_repair (max_repair);
++repair_sym_it) {
// we save the symbol here:
// make sure the vector has enough space for the symbol:
// fill it with zeros for the size of the symbol
std::vector<uint8_t> repair_sym_data (symbol_size, 0);
// save the data of the symbol into our vector
auto it = repair_sym_data.begin();
auto written = (*repair_sym_it) (it, repair_sym_data.end());
if (written != symbol_size) {
// this can only happen if "repair_sym_data" did not have
// enough space for a symbol (here: never)
std::cout << written << "-vs-" << symbol_size <<
" Could not get the whole repair symbol!\n";
return false;
}
// can we keep this symbol or do we randomly drop it?
float dropped = drop_rnd (rnd);
if (dropped <= drop_probability) {
continue; // start the cycle again
}
// good, the symbol was received.
++received_tot;
// add it to the vector of received symbols
symbol_id tmp_id = (*repair_sym_it).id();
received.emplace_back (tmp_id, std::move(repair_sym_data));
}
if (repair_sym_it == enc.end_repair (enc.max_repair())) {
// we dropped waaaay too many symbols!
// should never happen in real life. it means that we do not
// have enough repair symbols.
// at this point you can actually start to retransmit the
// repair symbols from enc.begin_repair(), but we don't care in
// this example
std::cout << "Maybe losing " << drop_probability << "% is too much?\n";
return false;
}
}
// Now we all the source and repair symbols are in "received".
// we will use those to start decoding:
// define "Decoder_type" to write less afterwards
using Decoder_type = RaptorQ::Decoder<
typename std::vector<uint8_t>::iterator,
typename std::vector<uint8_t>::iterator>;
Decoder_type dec (block, symbol_size, Decoder_type::Report::COMPLETE);
// "Decoder_type::Report::COMPLETE" means that the decoder will not
// give us any output until we have decoded all the data.
// there are modes to extract the data symbol by symbol in an ordered
// an unordered fashion, but let's keep this simple.
// we will store the output of the decoder here:
// note: the output need to have at least "mysize" bytes, and
// we fill it with zeros
std::vector<uint8_t> output (mysize, 0);
// now push every received symbol into the decoder
for (auto &rec_sym : received) {
// as a reminder:
// rec_sym.first = symbol_id (uint32_t)
// rec_sym.second = std::vector<uint8_t> symbol_data
symbol_id tmp_id = rec_sym.first;
auto it = rec_sym.second.begin();
auto err = dec.add_symbol (it, rec_sym.second.end(), tmp_id);
if (err != RaptorQ::Error::NONE && err != RaptorQ::Error::NOT_NEEDED) {
// When you add a symbol, you can get:
// NONE: no error
// NOT_NEEDED: libRaptorQ ignored it because everything is
// already decoded
// INITIALIZATION: wrong parameters to the decoder contructor
// WRONG_INPUT: not enough data on the symbol?
// some_other_error: errors in the library
std::cout << "error adding?\n";
return false;
}
}
// by now we now there will be no more input, so we tell this to the
// decoder. You can skip this call, but if the decoder does not have
// enough data it sill wait forever (or until you call .stop())
dec.end_of_input (RaptorQ::Fill_With_Zeros::NO);
// optional if you want partial decoding without using the repair
// symbols
// std::vector<bool> symbols_bitmask = dec.end_of_input (
// RaptorQ::Fill_With_Zeros::YES);
// decode, and do not return until the computation is finished.
auto res = dec.wait_sync();
if (res.error != RaptorQ::Error::NONE) {
std::cout << "Couldn't decode.\n";
return false;
}
// now save the decoded data in our output
size_t decode_from_byte = 0;
size_t skip_bytes_at_begining_of_output = 0;
auto out_it = output.begin();
auto decoded = dec.decode_bytes (out_it, output.end(), decode_from_byte,
skip_bytes_at_begining_of_output);
// "decode_from_byte" can be used to have only a part of the output.
// it can be used in advanced setups where you ask only a part
// of the block at a time.
// "skip_bytes_at_begining_of_output" is used when dealing with containers
// which size does not align with the output. For really advanced usage only
// Both should be zero for most setups.
if (decoded.written != mysize) {
if (decoded.written == 0) {
// we were really unlucky and the RQ algorithm needed
// more symbols!
std::cout << "Couldn't decode, RaptorQ Algorithm failure. "
"Can't Retry.\n";
} else {
// probably a library error
std::cout << "Partial Decoding? This should not have happened: " <<
decoded.written << " vs " << mysize << "\n";
}
return false;
} else {
std::cout << "Decoded: " << mysize << "\n";
}
// byte-wise check: did we actually decode everything the right way?
for (uint64_t i = 0; i < mysize; ++i) {
if (input[i] != output[i]) {
// this is a bug in the library, please report
std::cout << "The output does not correspond to the input!\n";
return false;
}
}
return true;
}
int main (void)
{
// get a random number generator
std::mt19937_64 rnd;
std::ifstream rand("/dev/urandom");
uint64_t seed = 0;
rand.read (reinterpret_cast<char *> (&seed), sizeof(seed));
rand.close ();
rnd.seed (seed);
// keep some computation in memory. If you use only one block size it
// will make things faster on bigger block size.
// allocate 5Mb
RaptorQ__v1::local_cache_size (5000000);
// for our test, we use an input of random size, between 100 and 10.000
// bytes.
std::uniform_int_distribution<uint32_t> distr(100, 10000);
uint32_t input_size = distr (rnd);
if (!test_rq (input_size, rnd, 20.0, 4))
return -1;
std::cout << "The example completed successfully\n";
return 0;
}
external/build_deps.cmake
View file @ a394b22a
...@@ -27,6 +27,8 @@ if(RQ_BUILD_LZ4) ...@@ -27,6 +27,8 @@ if(RQ_BUILD_LZ4)
set(RQ_LZ4_VERSION_PATCH r128) set(RQ_LZ4_VERSION_PATCH r128)
set(RQ_LZ4_VERSION_STRING " \"${RQ_LZ4_VERSION_MAJOR}.${RQ_LZ4_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ") set(RQ_LZ4_VERSION_STRING " \"${RQ_LZ4_VERSION_MAJOR}.${RQ_LZ4_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ")
set(RQ_LZ4_LIBNAME rq_lz4_static)
set(RQ_LZ4_DIR ${PROJECT_SOURCE_DIR}/external/lz4/lib/) set(RQ_LZ4_DIR ${PROJECT_SOURCE_DIR}/external/lz4/lib/)
set(RQ_LZ4_SRCS_LIB ${RQ_LZ4_DIR}lz4.c set(RQ_LZ4_SRCS_LIB ${RQ_LZ4_DIR}lz4.c
${RQ_LZ4_DIR}lz4hc.c ${RQ_LZ4_DIR}lz4hc.c
...@@ -36,38 +38,45 @@ if(RQ_BUILD_LZ4) ...@@ -36,38 +38,45 @@ if(RQ_BUILD_LZ4)
${RQ_LZ4_DIR}lz4frame.h ${RQ_LZ4_DIR}lz4frame.h
${RQ_LZ4_DIR}xxhash.c) ${RQ_LZ4_DIR}xxhash.c)
add_library(rq_lz4_static STATIC ${RQ_LZ4_SRCS_LIB}) add_library(${RQ_LZ4_LIBNAME} STATIC ${RQ_LZ4_SRCS_LIB})
include(CheckCCompilerFlag) include(CheckCCompilerFlag)
check_c_compiler_flag("-fPIC" RQ_LZ4_PIC)
if(RQ_LZ4_PIC)
target_compile_options(
${RQ_LZ4_LIBNAME} PRIVATE
"-fPIC"
)
endif()
check_c_compiler_flag("-std=c99" RQ_FLAG_C_99) check_c_compiler_flag("-std=c99" RQ_FLAG_C_99)
if(RQ_FLAG_C_99) if(RQ_FLAG_C_99)
target_compile_options( target_compile_options(
rq_lz4_static PRIVATE ${RQ_LZ4_LIBNAME} PRIVATE
"-std=c99" "-std=c99"
) )
endif() endif()
set_target_properties(rq_lz4_static PROPERTIES set_target_properties(${RQ_LZ4_LIBNAME} PROPERTIES
SOVERSION "${RQ_LZ4_VERSION_MAJOR}.${RQ_LZ4_VERSION_MINOR}") SOVERSION "${RQ_LZ4_VERSION_MAJOR}.${RQ_LZ4_VERSION_MINOR}")
#set_property(TARGET lz4_static PROPERTY LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib") #set_property(TARGET lz4_static PROPERTY LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib")
ADD_DEFINITIONS("-DLZ4_VERSION=\"${RQ_LZ4_VERSION_PATCH}\"") ADD_DEFINITIONS("-DLZ4_VERSION=\"${RQ_LZ4_VERSION_PATCH}\"")
INCLUDE_DIRECTORIES (${RQ_LZ4_DIR}) INCLUDE_DIRECTORIES (SYSTEM PUBLIC ${RQ_LZ4_DIR})
# NOTE: will be used like "-Wl..flag,flag,flag" . no spaces, and keep the comma
set(RQ_LZ4_EXCLUDE_SYM ",--exclude-libs lib${RQ_LZ4_LIBNAME}.a")
# set the dependency and build it all # set the dependency and build it all
if(CMAKE_SYSTEM_NAME MATCHES "Windows") if(CMAKE_SYSTEM_NAME MATCHES "Windows")
add_custom_target(LZ4 DEPENDS rq_lz4_static) add_custom_target(LZ4 DEPENDS ${RQ_LZ4_LIBNAME})
set(RQ_LZ4_DEP rq_lz4_static) set(RQ_LZ4_DEP ${RQ_LZ4_LIBNAME})
else() else()
add_custom_command( add_custom_command(
OUTPUT lz4_deterministic.run OUTPUT lz4_deterministic.run
COMMAND make_deterministic ${CMAKE_CURRENT_BINARY_DIR}/liblz4.a COMMAND make_deterministic ${CMAKE_CURRENT_BINARY_DIR}/lib${RQ_LZ4_LIBNAME}.a
DEPENDS lz4_static make_deterministic DEPENDS ${RQ_LZ4_LIBNAME} make_deterministic
COMMENT "Removing creation date from lz4 library..." COMMENT "Removing creation date from lz4 library..."
VERBATIM VERBATIM
) )
add_library(rq_lz4 STATIC IMPORTED)
set_target_properties(rq_lz4 PROPERTIES IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/liblz4.a)
add_custom_target(LZ4 DEPENDS lz4_deterministic.run) add_custom_target(LZ4 DEPENDS lz4_deterministic.run)
set(RQ_LZ4_DEP rq_lz4_static) set(RQ_LZ4_DEP ${RQ_LZ4_LIBNAME})
endif() endif()
else() else()
add_custom_target(LZ4) add_custom_target(LZ4)
......
external/eigen3/.hg_archival.txt
View file @ a394b22a
repo: 8a21fd850624c931e448cbcfb38168cb2717c790 repo: 8a21fd850624c931e448cbcfb38168cb2717c790
node: b9cd8366d4e8f49471c7afafc4c2a1b00e54a54d node: 5a0156e40feb7c4136680b493c6e433d91a6f355
branch: 3.2 branch: 3.3
tag: 3.2.10 tag: 3.3.4
external/eigen3/.hgeol
View file @ a394b22a
[patterns] [patterns]
*.sh = LF
*.MINPACK = CRLF
scripts/*.in = LF scripts/*.in = LF
debug/msvc/*.dat = CRLF debug/msvc/*.dat = CRLF
debug/msvc/*.natvis = CRLF
unsupported/test/mpreal/*.* = CRLF unsupported/test/mpreal/*.* = CRLF
** = native ** = native
......
external/eigen3/.hgignore
View file @ a394b22a
...@@ -30,3 +30,5 @@ log ...@@ -30,3 +30,5 @@ log
patch patch
a a
a.* a.*
lapack/testing
lapack/reference
external/eigen3/.hgtags
View file @ a394b22a
...@@ -21,15 +21,13 @@ a810d5dbab47acfe65b3350236efdd98f67d4d8a 3.1.0-alpha1 ...@@ -21,15 +21,13 @@ a810d5dbab47acfe65b3350236efdd98f67d4d8a 3.1.0-alpha1
8383e883ebcc6f14695ff0b5e20bb631abab43fb 3.1.0-rc1 8383e883ebcc6f14695ff0b5e20bb631abab43fb 3.1.0-rc1
bf4cb8c934fa3a79f45f1e629610f0225e93e493 3.1.0-rc2 bf4cb8c934fa3a79f45f1e629610f0225e93e493 3.1.0-rc2
da195914abcc1d739027cbee7c52077aab30b336 3.2-beta1 da195914abcc1d739027cbee7c52077aab30b336 3.2-beta1
4b687cad1d23066f66863f4f87298447298443df 3.2-rc1 a8e0d153fc5e239ef8b06e3665f1f9e8cb8d49c8 before-evaluators
1eeda7b1258bcd306018c0738e2b6a8543661141 3.2-rc2 09a8e21866106b49c5dec1d6d543e5794e82efa0 3.3-alpha1
ffa86ffb557094721ca71dcea6aed2651b9fd610 3.2.0 ce5a455b34c0a0ac3545a1497cb4a16c38ed90e8 3.3-beta1
6b38706d90a9fe182e66ab88477b3dbde34b9f66 3.2.1 69d418c0699907bcd0bf9e0b3ba0a112ed091d85 3.3-beta2
1306d75b4a21891e59ff9bd96678882cf831e39f 3.2.2 bef509908b9da05d0d07ffc0da105e2c8c6d3996 3.3-rc1
36fd1ba04c120cfdd90f3e4cede47f43b21d19ad 3.2.3 04ab5fa4b241754afcf631117572276444c67239 3.3-rc2
10219c95fe653d4962aa9db4946f6fbea96dd740 3.2.4 26667be4f70baf4f0d39e96f330714c87b399090 3.3.0
bdd17ee3b1b3a166cd5ec36dcad4fc1f3faf774a 3.2.5 f562a193118d4f40514e2f4a0ace6e974926ef06 3.3.1
c58038c56923e0fd86de3ded18e03df442e66dfb 3.2.6 da9b4e14c2550e0d11078a3c39e6d56eba9905df 3.3.2
b30b87236a1b1552af32ac34075ee5696a9b5a33 3.2.7 67e894c6cd8f5f1f604b27d37ed47fdf012674ff 3.3.3
07105f7124f9aef00a68c85e0fc606e65d3d6c15 3.2.8
dc6cfdf9bcec5efc7b6593bddbbb3d675de53524 3.2.9
external/eigen3/CMakeLists.txt
View file @ a394b22a
project(Eigen) project(Eigen3)
cmake_minimum_required(VERSION 2.8.5) cmake_minimum_required(VERSION 2.8.5)
# guard against in-source builds # guard against in-source builds
...@@ -7,6 +8,11 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) ...@@ -7,6 +8,11 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR})
message(FATAL_ERROR "In-source builds not allowed. Please make a new directory (called a build directory) and run CMake from there. You may need to remove CMakeCache.txt. ") message(FATAL_ERROR "In-source builds not allowed. Please make a new directory (called a build directory) and run CMake from there. You may need to remove CMakeCache.txt. ")
endif() endif()
# Alias Eigen_*_DIR to Eigen3_*_DIR:
set(Eigen_SOURCE_DIR ${Eigen3_SOURCE_DIR})
set(Eigen_BINARY_DIR ${Eigen3_BINARY_DIR})
# guard against bad build-type strings # guard against bad build-type strings
if (NOT CMAKE_BUILD_TYPE) if (NOT CMAKE_BUILD_TYPE)
...@@ -92,9 +98,11 @@ else() ...@@ -92,9 +98,11 @@ else()
endif() endif()
option(EIGEN_BUILD_BTL "Build benchmark suite" OFF) option(EIGEN_BUILD_BTL "Build benchmark suite" OFF)
if(NOT WIN32)
# Disable pkgconfig only for native Windows builds
if(NOT WIN32 OR NOT CMAKE_HOST_SYSTEM_NAME MATCHES Windows)
option(EIGEN_BUILD_PKGCONFIG "Build pkg-config .pc file for Eigen" ON) option(EIGEN_BUILD_PKGCONFIG "Build pkg-config .pc file for Eigen" ON)
endif(NOT WIN32) endif()
set(CMAKE_INCLUDE_CURRENT_DIR ON) set(CMAKE_INCLUDE_CURRENT_DIR ON)
...@@ -108,7 +116,8 @@ endif() ...@@ -108,7 +116,8 @@ endif()
set(EIGEN_TEST_MAX_SIZE "320" CACHE STRING "Maximal matrix/vector size, default is 320") set(EIGEN_TEST_MAX_SIZE "320" CACHE STRING "Maximal matrix/vector size, default is 320")
macro(ei_add_cxx_compiler_flag FLAG) macro(ei_add_cxx_compiler_flag FLAG)
string(REGEX REPLACE "-" "" SFLAG ${FLAG}) string(REGEX REPLACE "-" "" SFLAG1 ${FLAG})
string(REGEX REPLACE "\\+" "p" SFLAG ${SFLAG1})
check_cxx_compiler_flag(${FLAG} COMPILER_SUPPORT_${SFLAG}) check_cxx_compiler_flag(${FLAG} COMPILER_SUPPORT_${SFLAG})
if(COMPILER_SUPPORT_${SFLAG}) if(COMPILER_SUPPORT_${SFLAG})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG}")
...@@ -117,18 +126,13 @@ endmacro(ei_add_cxx_compiler_flag) ...@@ -117,18 +126,13 @@ endmacro(ei_add_cxx_compiler_flag)
if(NOT MSVC) if(NOT MSVC)
# We assume that other compilers are partly compatible with GNUCC # We assume that other compilers are partly compatible with GNUCC
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions") # clang outputs some warnings for unknown flags that are not caught by check_cxx_compiler_flag
set(CMAKE_CXX_FLAGS_DEBUG "-g3")
set(CMAKE_CXX_FLAGS_RELEASE "-g0 -O2")
# clang outputs some warnings for unknwon flags that are not caught by check_cxx_compiler_flag
# adding -Werror turns such warnings into errors # adding -Werror turns such warnings into errors
check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR)
if(COMPILER_SUPPORT_WERROR) if(COMPILER_SUPPORT_WERROR)
set(CMAKE_REQUIRED_FLAGS "-Werror") set(CMAKE_REQUIRED_FLAGS "-Werror")
endif() endif()
ei_add_cxx_compiler_flag("-pedantic") ei_add_cxx_compiler_flag("-pedantic")
ei_add_cxx_compiler_flag("-Wall") ei_add_cxx_compiler_flag("-Wall")
ei_add_cxx_compiler_flag("-Wextra") ei_add_cxx_compiler_flag("-Wextra")
...@@ -142,6 +146,18 @@ if(NOT MSVC) ...@@ -142,6 +146,18 @@ if(NOT MSVC)
ei_add_cxx_compiler_flag("-Wpointer-arith") ei_add_cxx_compiler_flag("-Wpointer-arith")
ei_add_cxx_compiler_flag("-Wwrite-strings") ei_add_cxx_compiler_flag("-Wwrite-strings")
ei_add_cxx_compiler_flag("-Wformat-security") ei_add_cxx_compiler_flag("-Wformat-security")
ei_add_cxx_compiler_flag("-Wshorten-64-to-32")
ei_add_cxx_compiler_flag("-Wlogical-op")
ei_add_cxx_compiler_flag("-Wenum-conversion")
ei_add_cxx_compiler_flag("-Wc++11-extensions")
ei_add_cxx_compiler_flag("-Wdouble-promotion")
# ei_add_cxx_compiler_flag("-Wconversion")
# -Wshadow is insanely too strict with gcc, hopefully it will become usable with gcc 6
# if(NOT CMAKE_COMPILER_IS_GNUCXX OR (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "5.0.0"))
if(NOT CMAKE_COMPILER_IS_GNUCXX)
ei_add_cxx_compiler_flag("-Wshadow")
endif()
ei_add_cxx_compiler_flag("-Wno-psabi") ei_add_cxx_compiler_flag("-Wno-psabi")
ei_add_cxx_compiler_flag("-Wno-variadic-macros") ei_add_cxx_compiler_flag("-Wno-variadic-macros")
...@@ -151,7 +167,8 @@ if(NOT MSVC) ...@@ -151,7 +167,8 @@ if(NOT MSVC)
ei_add_cxx_compiler_flag("-fno-common") ei_add_cxx_compiler_flag("-fno-common")
ei_add_cxx_compiler_flag("-fstrict-aliasing") ei_add_cxx_compiler_flag("-fstrict-aliasing")
ei_add_cxx_compiler_flag("-wd981") # disable ICC's "operands are evaluated in unspecified order" remark ei_add_cxx_compiler_flag("-wd981") # disable ICC's "operands are evaluated in unspecified order" remark
ei_add_cxx_compiler_flag("-wd2304") # disbale ICC's "warning #2304: non-explicit constructor with single argument may cause implicit type conversion" produced by -Wnon-virtual-dtor ei_add_cxx_compiler_flag("-wd2304") # disable ICC's "warning #2304: non-explicit constructor with single argument may cause implicit type conversion" produced by -Wnon-virtual-dtor
# The -ansi flag must be added last, otherwise it is also used as a linker flag by check_cxx_compiler_flag making it fails # The -ansi flag must be added last, otherwise it is also used as a linker flag by check_cxx_compiler_flag making it fails
# Moreover we should not set both -strict-ansi and -ansi # Moreover we should not set both -strict-ansi and -ansi
...@@ -163,6 +180,11 @@ if(NOT MSVC) ...@@ -163,6 +180,11 @@ if(NOT MSVC)
else() else()
ei_add_cxx_compiler_flag("-ansi") ei_add_cxx_compiler_flag("-ansi")
endif() endif()
if(ANDROID_NDK)
ei_add_cxx_compiler_flag("-pie")
ei_add_cxx_compiler_flag("-fPIE")
endif()
set(CMAKE_REQUIRED_FLAGS "") set(CMAKE_REQUIRED_FLAGS "")
...@@ -196,18 +218,65 @@ if(NOT MSVC) ...@@ -196,18 +218,65 @@ if(NOT MSVC)
message(STATUS "Enabling SSE4.2 in tests/examples") message(STATUS "Enabling SSE4.2 in tests/examples")
endif() endif()
option(EIGEN_TEST_AVX "Enable/Disable AVX in tests/examples" OFF)
if(EIGEN_TEST_AVX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
message(STATUS "Enabling AVX in tests/examples")
endif()
option(EIGEN_TEST_FMA "Enable/Disable FMA in tests/examples" OFF)
if(EIGEN_TEST_FMA AND NOT EIGEN_TEST_NEON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma")
message(STATUS "Enabling FMA in tests/examples")
endif()
option(EIGEN_TEST_AVX512 "Enable/Disable AVX512 in tests/examples" OFF)
if(EIGEN_TEST_AVX512)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -fabi-version=6 -DEIGEN_ENABLE_AVX512")
message(STATUS "Enabling AVX512 in tests/examples")
endif()
option(EIGEN_TEST_F16C "Enable/Disable F16C in tests/examples" OFF)
if(EIGEN_TEST_F16C)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mf16c")
message(STATUS "Enabling F16C in tests/examples")
endif()
option(EIGEN_TEST_ALTIVEC "Enable/Disable AltiVec in tests/examples" OFF) option(EIGEN_TEST_ALTIVEC "Enable/Disable AltiVec in tests/examples" OFF)
if(EIGEN_TEST_ALTIVEC) if(EIGEN_TEST_ALTIVEC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec -mabi=altivec") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec -mabi=altivec")
message(STATUS "Enabling AltiVec in tests/examples") message(STATUS "Enabling AltiVec in tests/examples")
endif() endif()
option(EIGEN_TEST_VSX "Enable/Disable VSX in tests/examples" OFF)
if(EIGEN_TEST_VSX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m64 -mvsx")
message(STATUS "Enabling VSX in tests/examples")
endif()
option(EIGEN_TEST_NEON "Enable/Disable Neon in tests/examples" OFF) option(EIGEN_TEST_NEON "Enable/Disable Neon in tests/examples" OFF)
if(EIGEN_TEST_NEON) if(EIGEN_TEST_NEON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=neon -mcpu=cortex-a8") if(EIGEN_TEST_FMA)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=neon-vfpv4")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=neon")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfloat-abi=hard")
message(STATUS "Enabling NEON in tests/examples")
endif()
option(EIGEN_TEST_NEON64 "Enable/Disable Neon in tests/examples" OFF)
if(EIGEN_TEST_NEON64)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
message(STATUS "Enabling NEON in tests/examples") message(STATUS "Enabling NEON in tests/examples")
endif() endif()
option(EIGEN_TEST_ZVECTOR "Enable/Disable S390X(zEC13) ZVECTOR in tests/examples" OFF)
if(EIGEN_TEST_ZVECTOR)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z13 -mzvector")
message(STATUS "Enabling S390X(zEC13) ZVECTOR in tests/examples")
endif()
check_cxx_compiler_flag("-fopenmp" COMPILER_SUPPORT_OPENMP) check_cxx_compiler_flag("-fopenmp" COMPILER_SUPPORT_OPENMP)
if(COMPILER_SUPPORT_OPENMP) if(COMPILER_SUPPORT_OPENMP)
option(EIGEN_TEST_OPENMP "Enable/Disable OpenMP in tests/examples" OFF) option(EIGEN_TEST_OPENMP "Enable/Disable OpenMP in tests/examples" OFF)
...@@ -284,11 +353,23 @@ if(EIGEN_TEST_NO_EXPLICIT_ALIGNMENT) ...@@ -284,11 +353,23 @@ if(EIGEN_TEST_NO_EXPLICIT_ALIGNMENT)
message(STATUS "Disabling alignment in tests/examples") message(STATUS "Disabling alignment in tests/examples")
endif() endif()
option(EIGEN_TEST_C++0x "Enables all C++0x features." OFF) option(EIGEN_TEST_NO_EXCEPTIONS "Disables C++ exceptions" OFF)
if(EIGEN_TEST_NO_EXCEPTIONS)
ei_add_cxx_compiler_flag("-fno-exceptions")
message(STATUS "Disabling exceptions in tests/examples")
endif()
option(EIGEN_TEST_CXX11 "Enable testing with C++11 and C++11 features (e.g. Tensor module)." OFF)
set(EIGEN_CUDA_COMPUTE_ARCH 30 CACHE STRING "The CUDA compute architecture level to target when compiling CUDA code")
include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
# Backward compatibility support for EIGEN_INCLUDE_INSTALL_DIR # Backward compatibility support for EIGEN_INCLUDE_INSTALL_DIR
if(EIGEN_INCLUDE_INSTALL_DIR)
message(WARNING "EIGEN_INCLUDE_INSTALL_DIR is deprecated. Use INCLUDE_INSTALL_DIR instead.")
endif()
if(EIGEN_INCLUDE_INSTALL_DIR AND NOT INCLUDE_INSTALL_DIR) if(EIGEN_INCLUDE_INSTALL_DIR AND NOT INCLUDE_INSTALL_DIR)
set(INCLUDE_INSTALL_DIR ${EIGEN_INCLUDE_INSTALL_DIR} set(INCLUDE_INSTALL_DIR ${EIGEN_INCLUDE_INSTALL_DIR}
CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where Eigen header files are installed") CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where Eigen header files are installed")
...@@ -298,9 +379,8 @@ else() ...@@ -298,9 +379,8 @@ else()
CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where Eigen header files are installed" CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where Eigen header files are installed"
) )
endif() endif()
set(CMAKEPACKAGE_INSTALL_DIR set(CMAKEPACKAGE_INSTALL_DIR
"${CMAKE_INSTALL_LIBDIR}/cmake/eigen3" "${CMAKE_INSTALL_DATADIR}/eigen3/cmake"
CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where Eigen3Config.cmake is installed" CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where Eigen3Config.cmake is installed"
) )
set(PKGCONFIG_INSTALL_DIR set(PKGCONFIG_INSTALL_DIR
...@@ -308,6 +388,7 @@ set(PKGCONFIG_INSTALL_DIR ...@@ -308,6 +388,7 @@ set(PKGCONFIG_INSTALL_DIR
CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where eigen3.pc is installed" CACHE PATH "The directory relative to CMAKE_PREFIX_PATH where eigen3.pc is installed"
) )
# similar to set_target_properties but append the property instead of overwriting it # similar to set_target_properties but append the property instead of overwriting it
macro(ei_add_target_property target prop value) macro(ei_add_target_property target prop value)
...@@ -329,7 +410,7 @@ if(EIGEN_BUILD_PKGCONFIG) ...@@ -329,7 +410,7 @@ if(EIGEN_BUILD_PKGCONFIG)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/eigen3.pc install(FILES ${CMAKE_CURRENT_BINARY_DIR}/eigen3.pc
DESTINATION ${PKGCONFIG_INSTALL_DIR} DESTINATION ${PKGCONFIG_INSTALL_DIR}
) )
endif(EIGEN_BUILD_PKGCONFIG) endif()
add_subdirectory(Eigen) add_subdirectory(Eigen)
...@@ -355,6 +436,13 @@ else() ...@@ -355,6 +436,13 @@ else()
add_subdirectory(lapack EXCLUDE_FROM_ALL) add_subdirectory(lapack EXCLUDE_FROM_ALL)
endif() endif()
# add SYCL
option(EIGEN_TEST_SYCL "Add Sycl support." OFF)
if(EIGEN_TEST_SYCL)
set (CMAKE_MODULE_PATH "${CMAKE_ROOT}/Modules" "cmake/Modules/" "${CMAKE_MODULE_PATH}")
include(FindComputeCpp)
endif()
add_subdirectory(unsupported) add_subdirectory(unsupported)
add_subdirectory(demos EXCLUDE_FROM_ALL) add_subdirectory(demos EXCLUDE_FROM_ALL)
...@@ -403,6 +491,7 @@ if(cmake_generator_tolower MATCHES "makefile") ...@@ -403,6 +491,7 @@ if(cmake_generator_tolower MATCHES "makefile")
message(STATUS "make check | Build and run the unit-tests. Read this page:") message(STATUS "make check | Build and run the unit-tests. Read this page:")
message(STATUS " | http://eigen.tuxfamily.org/index.php?title=Tests") message(STATUS " | http://eigen.tuxfamily.org/index.php?title=Tests")
message(STATUS "make blas | Build BLAS library (not the same thing as Eigen)") message(STATUS "make blas | Build BLAS library (not the same thing as Eigen)")
message(STATUS "make uninstall| Removes files installed by make install")
message(STATUS "--------------+--------------------------------------------------------------") message(STATUS "--------------+--------------------------------------------------------------")
else() else()
message(STATUS "To build/run the unit tests, read this page:") message(STATUS "To build/run the unit tests, read this page:")
...@@ -410,3 +499,98 @@ else() ...@@ -410,3 +499,98 @@ else()
endif() endif()
message(STATUS "") message(STATUS "")
set ( EIGEN_VERSION_STRING ${EIGEN_VERSION_NUMBER} )
set ( EIGEN_VERSION_MAJOR ${EIGEN_WORLD_VERSION} )
set ( EIGEN_VERSION_MINOR ${EIGEN_MAJOR_VERSION} )
set ( EIGEN_VERSION_PATCH ${EIGEN_MINOR_VERSION} )
set ( EIGEN_DEFINITIONS "")
set ( EIGEN_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/${INCLUDE_INSTALL_DIR}" )
set ( EIGEN_ROOT_DIR ${CMAKE_INSTALL_PREFIX} )
# Interface libraries require at least CMake 3.0
if (NOT CMAKE_VERSION VERSION_LESS 3.0)
include (CMakePackageConfigHelpers)
# Imported target support
add_library (eigen INTERFACE)
target_compile_definitions (eigen INTERFACE ${EIGEN_DEFINITIONS})
target_include_directories (eigen INTERFACE
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<INSTALL_INTERFACE:${INCLUDE_INSTALL_DIR}>
)
# Export as title case Eigen
set_target_properties (eigen PROPERTIES EXPORT_NAME Eigen)
install (TARGETS eigen EXPORT Eigen3Targets)
configure_package_config_file (
${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3Config.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake
PATH_VARS EIGEN_INCLUDE_DIR EIGEN_ROOT_DIR
INSTALL_DESTINATION ${CMAKEPACKAGE_INSTALL_DIR}
NO_CHECK_REQUIRED_COMPONENTS_MACRO # Eigen does not provide components
)
# Remove CMAKE_SIZEOF_VOID_P from Eigen3ConfigVersion.cmake since Eigen does
# not depend on architecture specific settings or libraries. More
# specifically, an Eigen3Config.cmake generated from a 64 bit target can be
# used for 32 bit targets as well (and vice versa).
set (_Eigen3_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P})
unset (CMAKE_SIZEOF_VOID_P)
write_basic_package_version_file (Eigen3ConfigVersion.cmake
VERSION ${EIGEN_VERSION_NUMBER}
COMPATIBILITY SameMajorVersion)
set (CMAKE_SIZEOF_VOID_P ${_Eigen3_CMAKE_SIZEOF_VOID_P})
# The Eigen target will be located in the Eigen3 namespace. Other CMake
# targets can refer to it using Eigen3::Eigen.
export (TARGETS eigen NAMESPACE Eigen3:: FILE Eigen3Targets.cmake)
# Export Eigen3 package to CMake registry such that it can be easily found by
# CMake even if it has not been installed to a standard directory.
export (PACKAGE Eigen3)
install (EXPORT Eigen3Targets NAMESPACE Eigen3:: DESTINATION ${CMAKEPACKAGE_INSTALL_DIR})
else (NOT CMAKE_VERSION VERSION_LESS 3.0)
# Fallback to legacy Eigen3Config.cmake without the imported target
# If CMakePackageConfigHelpers module is available (CMake >= 2.8.8)
# create a relocatable Config file, otherwise leave the hardcoded paths
include(CMakePackageConfigHelpers OPTIONAL RESULT_VARIABLE CPCH_PATH)
if(CPCH_PATH)
configure_package_config_file (
${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3ConfigLegacy.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake
PATH_VARS EIGEN_INCLUDE_DIR EIGEN_ROOT_DIR
INSTALL_DESTINATION ${CMAKEPACKAGE_INSTALL_DIR}
NO_CHECK_REQUIRED_COMPONENTS_MACRO # Eigen does not provide components
)
else()
# The PACKAGE_* variables are defined by the configure_package_config_file
# but without it we define them manually to the hardcoded paths
set(PACKAGE_INIT "")
set(PACKAGE_EIGEN_INCLUDE_DIR ${EIGEN_INCLUDE_DIR})
set(PACKAGE_EIGEN_ROOT_DIR ${EIGEN_ROOT_DIR})
configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Eigen3ConfigLegacy.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake
@ONLY ESCAPE_QUOTES )
endif()
write_basic_package_version_file( Eigen3ConfigVersion.cmake
VERSION ${EIGEN_VERSION_NUMBER}
COMPATIBILITY SameMajorVersion )
endif (NOT CMAKE_VERSION VERSION_LESS 3.0)
install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/UseEigen3.cmake
${CMAKE_CURRENT_BINARY_DIR}/Eigen3Config.cmake
${CMAKE_CURRENT_BINARY_DIR}/Eigen3ConfigVersion.cmake
DESTINATION ${CMAKEPACKAGE_INSTALL_DIR} )
# Add uninstall target
add_custom_target ( uninstall
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/EigenUninstall.cmake)
external/eigen3/COPYING.MINPACK
View file @ a394b22a
Minpack Copyright Notice (1999) University of Chicago. All rights reserved Minpack Copyright Notice (1999) University of Chicago. All rights reserved
Redistribution and use in source and binary forms, with or Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the without modification, are permitted provided that the
following conditions are met: following conditions are met:
1. Redistributions of source code must retain the above 1. Redistributions of source code must retain the above
copyright notice, this list of conditions and the following copyright notice, this list of conditions and the following
disclaimer. disclaimer.
2. Redistributions in binary form must reproduce the above 2. Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials disclaimer in the documentation and/or other materials
provided with the distribution. provided with the distribution.
3. The end-user documentation included with the 3. The end-user documentation included with the
redistribution, if any, must include the following redistribution, if any, must include the following
acknowledgment: acknowledgment:
"This product includes software developed by the "This product includes software developed by the
University of Chicago, as Operator of Argonne National University of Chicago, as Operator of Argonne National
Laboratory. Laboratory.
Alternately, this acknowledgment may appear in the software Alternately, this acknowledgment may appear in the software
itself, if and wherever such third-party acknowledgments itself, if and wherever such third-party acknowledgments
normally appear. normally appear.
4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS" 4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS"
WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE
UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND
THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE
OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY
OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR
USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF
THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4)
DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION
UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL
BE CORRECTED. BE CORRECTED.
5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT 5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT
HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF
ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT, ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT,
INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF
ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF
PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER
SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT
(INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE, (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE,
EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE
POSSIBILITY OF SUCH LOSS OR DAMAGES. POSSIBILITY OF SUCH LOSS OR DAMAGES.