diff --git a/.gitmodules b/.gitmodules index be3096f1..3316355e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "ext/SuperLU_MT_3.1"] path = ext/SuperLU_MT_3.1 - url = https://github.com/shavvn/SuperLU_MT_3.1.git + url = https://github.com/umd-memsys/SuperLU_MT_3.1.git ignore = dirty diff --git a/.vscode/.cmaketools.json b/.vscode/.cmaketools.json deleted file mode 100644 index 0a3d8ea8..00000000 --- a/.vscode/.cmaketools.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "variant": { - "label": "Debug", - "keywordSettings": { - "buildType": "debug" - }, - "description": "Emit debug information without performing optimizations" - }, - "activeEnvironments": [] -} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 333904ff..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "version": "0.2.0", - "configurations": [ - - { - "name": "C++ Launch", - "type": "cppdbg", - "request": "launch", - "program": "${workspaceRoot}/build/dramcoremain", - "args": ["-c", "./../configs/ddr4_8G_x8_config.ini", "-n", "100000", "--trace-cpu", "--trace-file", "./../sample_trace.txt"], - "stopAtEntry": true, - "cwd": "${workspaceRoot}/build", - "environment": [], - "externalConsole": true, - "linux": { - "MIMode": "gdb", - "setupCommands": [ - { - "description": "Enable pretty-printing for gdb", - "text": "-enable-pretty-printing", - "ignoreFailures": true - } - ] - }, - "osx": { - "MIMode": "lldb" - }, - "windows": { - "MIMode": "gdb", - "setupCommands": [ - { - "description": "Enable pretty-printing for gdb", - "text": "-enable-pretty-printing", - "ignoreFailures": true - } - ] - } - }, - { - "name": "C++ Attach", - "type": "cppdbg", - "request": "attach", - "program": "enter program name, for example ${workspaceRoot}/a.out", - "processId": "${command.pickProcess}", - "linux": { - "MIMode": "gdb", - "setupCommands": [ - { - "description": "Enable pretty-printing for gdb", - "text": "-enable-pretty-printing", - "ignoreFailures": true - } - ] - }, - "osx": { - "MIMode": "lldb" - }, - "windows": { - "MIMode": "gdb", - "setupCommands": [ - { - "description": "Enable pretty-printing for gdb", - "text": "-enable-pretty-printing", - "ignoreFailures": true - } - ] - } - } - ] -} \ No newline at end of file diff --git a/.vscode/settings-backup.json b/.vscode/settings-backup.json deleted file mode 100644 index 31fd83bc..00000000 --- a/.vscode/settings-backup.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "files.associations": { - "array": "cpp", - "initializer_list": "cpp" - } -} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index d7501528..00000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format - "version": "0.1.0", - "command": "make", - "isShellCommand": true, - "args": ["-j5"], - "showOutput": "always", - "options": { - "cwd": "${workspaceRoot}/build" - } -} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 3866e9fe..80bfc59c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,9 @@ add_library(dramsim3 SHARED src/simple_stats.cc src/timing.cc src/memory_system.cc + src/bob.cc + src/custom_cpu.cc + src/ndp_address_table.cc ) if (THERMAL) @@ -67,16 +70,19 @@ if (THERMAL) target_compile_options(thermalreplay PRIVATE -DTHERMAL -D_LONGINT -DAdd_ ${OpenMP_C_FLAGS}) endif (THERMAL) -find_package(OpenMP) -if (OPENMP_FOUND AND USE_OPENMP) # only use OpenMP explicitly - target_compile_options(dramsim3 PRIVATE ${OpenMP_CXX_FLAGS}) - target_link_libraries(dramsim3 INTERFACE ${OpenMP_CXX_FLAGS}) -endif (OPENMP_FOUND AND USE_OPENMP) - if (CMD_TRACE) target_compile_options(dramsim3 PRIVATE -DCMD_TRACE) endif (CMD_TRACE) +if (MY_DEBUG) + target_compile_options(dramsim3 PRIVATE -DMY_DEBUG) +endif (MY_DEBUG) + +if (ADDR_TRACE) + target_compile_options(dramsim3 PRIVATE -DADDR_TRACE) +endif (ADDR_TRACE) + + target_include_directories(dramsim3 INTERFACE src) target_compile_options(dramsim3 PRIVATE -Wall) target_link_libraries(dramsim3 PRIVATE inih format) diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..fd15300f --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2019, University of Maryland Memory-Systems Research + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile index 130f7b54..fee6c0aa 100644 --- a/Makefile +++ b/Makefile @@ -2,23 +2,32 @@ CC=gcc CXX=g++ - +MY_DEBUG=0 +EN_CMD_TRACE=0 FMT_LIB_DIR=ext/fmt/include INI_LIB_DIR=ext/headers JSON_LIB_DIR=ext/headers ARGS_LIB_DIR=ext/headers INC=-Isrc/ -I$(FMT_LIB_DIR) -I$(INI_LIB_DIR) -I$(ARGS_LIB_DIR) -I$(JSON_LIB_DIR) -CXXFLAGS=-Wall -O3 -fPIC -std=c++11 $(INC) -DFMT_HEADER_ONLY=1 - +#CXXFLAGS=-Wall -O3 -fPIC -std=c++11 $(INC) -DFMT_HEADER_ONLY=1 +CXXFLAGS=-Wall -g -fPIC -std=c++11 $(INC) -DFMT_HEADER_ONLY=1 +ifeq (${MY_DEBUG},1) +CXXFLAGS+=-DMY_DEBUG +endif +ifeq (${EN_CMD_TRACE},1) +CXXFLAGS+=-DCMD_TRACE +endif LIB_NAME=libdramsim3.so EXE_NAME=dramsim3main.out SRCS = src/bankstate.cc src/channel_state.cc src/command_queue.cc src/common.cc \ src/configuration.cc src/controller.cc src/dram_system.cc src/hmc.cc \ - src/memory_system.cc src/refresh.cc src/simple_stats.cc src/timing.cc + src/memory_system.cc src/refresh.cc src/simple_stats.cc src/timing.cc \ + src/bob.cc + -EXE_SRCS = src/cpu.cc src/main.cc +EXE_SRCS = src/ndp_address_table.cc src/custom_cpu.cc src/cpu.cc src/main.cc OBJECTS = $(addsuffix .o, $(basename $(SRCS))) EXE_OBJS = $(addsuffix .o, $(basename $(EXE_SRCS))) diff --git a/README.md b/README.md index c2784c32..d47707fe 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,44 @@ -[![Build Status](https://travis-ci.com/shavvn/DRAMSim3.svg?token=pCfCJ4yBqyhn3rfWbJVF&branch=master)](https://travis-ci.com/shavvn/DRAMSim3) +[![Build Status](https://travis-ci.com/umd-memsys/DRAMsim3.svg?branch=master)](https://travis-ci.com/umd-memsys/DRAMsim3) -# DRAMSim3 -Model the timing paramaters and memory controller behaviour for several DRAM protocols such as DDR3, DDR4, LPDDR3, LPDDR4, GDDR5, HBM, HMC, STT-MRAM. +# About DRAMsim3 + +DRAMsim3 models the timing paramaters and memory controller behavior for several DRAM protocols such as DDR3, DDR4, LPDDR3, LPDDR4, GDDR5, GDDR6, HBM, HMC, STT-MRAM. It is implemented in C++ as an objected oriented model that includes a parameterized DRAM bank model, DRAM controllers, command queues and system-level interfaces to interact with a CPU simulator (GEM5, ZSim) or trace workloads. It is designed to be accurate, portable and parallel. + +If you use this simulator in your work, please consider cite: + +[1] S. Li, Z. Yang, D. Reddy, A. Srivastava and B. Jacob, "DRAMsim3: a Cycle-accurate, Thermal-Capable DRAM Simulator," in IEEE Computer Architecture Letters. [Link](https://ieeexplore.ieee.org/document/8999595) + +See [Related Work](#related-work) for more work done with this simulator. ## Building and running the simulator + This simulator by default uses a CMake based build system. The advantage in using a CMake based build system is portability and dependency management. We require CMake 3.0+ to build this simulator. If `cmake-3.0` is not available, we also supply a Makefile to build the most basic version of the simulator. -## From the command line - ### Building + Doing out of source builds with CMake is recommended to avoid the build files cluttering the main directory. ```bash -# Create and jump to the build directory -mkdir build +# cmake out of source build +mkdir build cd build - -# Create Makefile using the CMakeLists.txt file in the parent directory cmake .. -# Alternatively, build with thermal module enabled (still testing) -cmake .. -DTHERMAL=1 - -# Build dramsim3 and executables +# Build dramsim3 library and executables make -j4 -``` +# Alternatively, build with thermal module enabled +cmake .. -DTHERMAL=1 -The build process creates `dramsim3main` and executables in the build -directory. -By default, it also creates `libdramsim3.so` shared library in the -project root directory. +``` +The build process creates `dramsim3main` and executables in the `build` directory. +By default, it also creates `libdramsim3.so` shared library in the project root directory. ### Running @@ -47,26 +49,110 @@ project root directory. # Running random stream with a config file ./build/dramsim3main configs/DDR4_8Gb_x8_3200.ini --stream random -c 100000 -# Running trace cpu with a trace file and a config file +# Running a trace file ./build/dramsim3main configs/DDR4_8Gb_x8_3200.ini -c 100000 -t sample_trace.txt +# Running with gem5 +--mem-type=dramsim3 --dramsim3-ini=configs/DDR4_4Gb_x4_2133.ini + ``` The output can be directed to another directory by `-o` option -or can be configured in the config file. +or can be configured in the config file. You can control the verbosity in the config file as well. -### Plotting (in dev) +### Output Visualization `scripts/plot_stats.py` can visualize some of the output (requires `matplotlib`): ```bash # generate histograms from overall output -python3 scripts/plot_stats dramsim.json +python3 scripts/plot_stats dramsim3.json # or # generate time series for a variety stats from epoch outputs -python3 scripts/plot_stats dramsim_epoch.json +python3 scripts/plot_stats dramsim3epoch.json ``` Currently stats from all channels are squashed together for cleaner plotting. + +### Integration with other simulators + +**Gem5** integration: works with a forked Gem5 version, see https://github.com/umd-memsys/gem5 at `dramsim3` branch for reference. + +**SST** integration: see http://git.ece.umd.edu/shangli/sst-elements/tree/dramsim3 for reference. We will try to merge to official SST repo. + +**ZSim** integration: see http://git.ece.umd.edu/shangli/zsim/tree/master for reference. + +## Simulator Design + +### Code Structure + +``` +├── configs # Configs of various protocols that describe timing constraints and power consumption. +├── ext # +├── scripts # Tools and utilities +├── src # DRAMsim3 source files +├── tests # Tests of each model, includes a short example trace +├── CMakeLists.txt +├── Makefile +├── LICENSE +└── README.md + +├── src + bankstate.cc: Records and manages DRAM bank timings and states which is modeled as a state machine. + channelstate.cc: Records and manages channel timings and states. + command_queue.cc: Maintains per-bank or per-rank FIFO queueing structures, determine which commands in the queues can be issued in this cycle. + configuration.cc: Initiates, manages system and DRAM parameters, including protocol, DRAM timings, address mapping policy and power parameters. + controller.cc: Maintains the per-channel controller, which manages a queue of pending memory transactions and issues corresponding DRAM commands, + follows FR-FCFS policy. + cpu.cc: Implements 3 types of simple CPU: + 1. Random, can handle random CPU requests at full speed, the entire parallelism of DRAM protocol can be exploited without limits from address mapping and scheduling pocilies. + 2. Stream, provides a streaming prototype that is able to provide enough buffer hits. + 3. Trace-based, consumes traces of workloads, feed the fetched transactions into the memory system. + dram_system.cc: Initiates JEDEC or ideal DRAM system, registers the supplied callback function to let the front end driver know that the request is finished. + hmc.cc: Implements HMC system and interface, HMC requests are translates to DRAM requests here and a crossbar interconnect between the high-speed links and the memory controllers is modeled. + main.cc: Handles the main program loop that reads in simulation arguments, DRAM configurations and tick cycle forward. + memory_system.cc: A wrapper of dram_system and hmc. + refresh.cc: Raises refresh request based on per-rank refresh or per-bank refresh. + timing.cc: Initiate timing constraints. +``` + +## Experiments + +### Verilog Validation + +First we generate a DRAM command trace. +There is a `CMD_TRACE` macro and by default it's disabled. +Use `cmake .. -DCMD_TRACE=1` to enable the command trace output build and then +whenever a simulation is performed the command trace file will be generated. + +Next, `scripts/validation.py` helps generate a Verilog workbench for Micron's Verilog model +from the command trace file. +Currently DDR3, DDR4, and LPDDR configs are supported by this script. + +Run + +```bash +./script/validataion.py DDR4.ini cmd.trace +``` + +To generage Verilog workbench. +Our workbench format is compatible with ModelSim Verilog simulator, +other Verilog simulators may require a slightly different format. + + +## Related Work + +[1] Li, S., Yang, Z., Reddy D., Srivastava, A. and Jacob, B., (2020) DRAMsim3: a Cycle-accurate, Thermal-Capable DRAM Simulator, IEEE Computer Architecture Letters. + +[2] Jagasivamani, M., Walden, C., Singh, D., Kang, L., Li, S., Asnaashari, M., ... & Yeung, D. (2019). Analyzing the Monolithic Integration of a ReRAM-Based Main Memory Into a CPU's Die. IEEE Micro, 39(6), 64-72. + +[3] Li, S., Reddy, D., & Jacob, B. (2018, October). A performance & power comparison of modern high-speed DRAM architectures. In Proceedings of the International Symposium on Memory Systems (pp. 341-353). + +[4] Li, S., Verdejo, R. S., Radojković, P., & Jacob, B. (2019, September). Rethinking cycle accurate DRAM simulation. In Proceedings of the International Symposium on Memory Systems (pp. 184-191). + +[5] Li, S., & Jacob, B. (2019, September). Statistical DRAM modeling. In Proceedings of the International Symposium on Memory Systems (pp. 521-530). + +[6] Li, S. (2019). Scalable and Accurate Memory System Simulation (Doctoral dissertation). + diff --git a/configs/DDR4_8Gb_x4_3200_LRDIMM.ini b/configs/DDR4_8Gb_x4_3200_LRDIMM.ini new file mode 100644 index 00000000..f43da86b --- /dev/null +++ b/configs/DDR4_8Gb_x4_3200_LRDIMM.ini @@ -0,0 +1,69 @@ +[dram_structure] +protocol = DDR4 +bankgroups = 4 +banks_per_group = 4 +rows = 131072 +columns = 1024 +device_width = 4 +BL = 8 + +[timing] +tCK = 0.63 +AL = 0 +CL = 22 +CWL = 16 +tRCD = 22 +tRP = 22 +tRAS = 52 +tRFC = 560 +tRFC2 = 416 +tRFC4 = 256 +tREFI = 12480 +tRPRE = 1 +tWPRE = 1 +tRRD_S = 4 +tRRD_L = 8 +tWTR_S = 4 +tWTR_L = 12 +tFAW = 16 +tWR = 24 +tWR2 = 25 +tRTP = 12 +tCCD_S = 4 +tCCD_L = 8 +tCKE = 8 +tCKESR = 9 +tXS = 576 +tXP = 10 +tRTRS = 1 + +[power] +VDD = 1.2 +IDD0 = 52 +IPP0 = 3.0 +IDD2P = 25 +IDD2N = 37 +IDD3P = 38 +IDD3N = 47 +IDD4W = 130 +IDD4R = 143 +IDD5AB = 250 +IDD6x = 30 + +[system] +channel_size = 98304 +channels = 1 +bus_width = 64 +address_mapping = rochrababgco +queue_structure = PER_BANK +refresh_policy = RANK_LEVEL_STAGGERED +row_buf_policy = OPEN_PAGE +cmd_queue_size = 8 +trans_queue_size = 32 +is_LRDIMM = true +ranks_per_dimm = 4 + +[other] +epoch_period = 1587301 +output_level = 1 + diff --git a/configs/GDDR5X_8Gb_x32.ini b/configs/GDDR5X_8Gb_x32.ini index c27cfe3b..d8050da1 100644 --- a/configs/GDDR5X_8Gb_x32.ini +++ b/configs/GDDR5X_8Gb_x32.ini @@ -9,7 +9,7 @@ BL = 16 bankgroup_enable = false [timing] -tCK = 0.666 +tCK = 0.666 (1/1.5) CL = 24 CWL = 7 tRCDRD = 18 diff --git a/configs/GDDR6_8Gb_x16.ini b/configs/GDDR6_8Gb_x16.ini new file mode 100644 index 00000000..f54fde94 --- /dev/null +++ b/configs/GDDR6_8Gb_x16.ini @@ -0,0 +1,69 @@ +[dram_structure] +protocol = GDDR6 +bankgroups = 4 +banks_per_group = 4 +rows = 16384 +columns = 128 +device_width = 16 +BL = 16 +bankgroup_enable = false + +[timing] +tCK = 0.66 +CL = 24 +CWL = 16 +tRCDRD = 24 +tRCDWR = 20 +tRP = 24 +tRAS = 54 +tRFC = 126 +tREFI = 11862 +tRPRE = 1; read preamble +tWPRE = 1; TODO figure this out, should be 1 or 2 +tRRD_S = 9 +tRRD_L = 9 +tWTR_S = 7 +tWTR_L = 7 +tFAW = 32 +tWR = 16 +tCCD_S = 3 +tCCD_L = 4 +tXS = 132 +tCKESR = 8 +tXP = 12 +tRTRS = 1 +tRTP_L = 3 +tRTP_S = 3 +tPPD = 2 +t32AW = 420 +tRFCb = 30 +tREFIb = 238 + +[power] +VDD = 1.35 +IDD0 = 71 +IDD2P = 45 +IDD2N = 60 +IDD3P = 50 +IDD3N = 61 +IDD4W = 231 +IDD4R = 248 +IDD5AB = 286 +IDD5PB = 45 +IDD6x = 35 + +[system] +channel_size = 4096 +channels = 1 +bus_width = 128 +address_mapping = rochrababgco +queue_structure = PER_BANK +row_buf_policy = OPEN_PAGE +cmd_queue_size = 8 +trans_queue_size = 32 + +[other] +epoch_period = 1499250 +output_level = 1 + +;16n prefetch \ No newline at end of file diff --git a/configs/LPDDR4_8Gb_x16_2400.ini b/configs/LPDDR4_8Gb_x16_2400.ini new file mode 100644 index 00000000..ae110c38 --- /dev/null +++ b/configs/LPDDR4_8Gb_x16_2400.ini @@ -0,0 +1,68 @@ +[dram_structure] +protocol = LPDDR4 +bankgroups = 2 +banks_per_group = 4 +rows = 65536 +columns = 1024 +device_width = 16 +BL = 16 + +[timing] +tCK = 0.83 +AL = 0 +CL = 17 +CWL = 14 +tRCD = 15 +tRP = 15 +tRAS = 32 +tRFC = 392 +tRFC2 = 268 +tRFC4 = 172 +tREFI = 8660 +tRPRE = 1 +tWPRE = 1 +tRRD_S = 8 +tRRD_L = 8 +tWTR_S = 8 +tWTR_L = 16 +tFAW = 32 +tWR = 30 +tWR2 = 32 +tRTP = 12 +tCCD_S = 4 +tCCD_L = 6 +tCKE = 6 +tCKESR = 7 +tXS = 432 +tXP = 6 +tRTRS = 1 +tPPD = 2 + +[power] +VDD = 1.2 +IDD0 = 80 +IPP0 = 4.0 +IDD2P = 25 +IDD2N = 34 +IDD3P = 41 +IDD3N = 47 +IDD4W = 228 +IDD4R = 243 +IDD5AB = 280 +IDD6x = 30 + +[system] +channel_size = 8192 +channels = 1 +bus_width = 64 +address_mapping = rochrababgco +queue_structure = PER_BANK +refresh_policy = RANK_LEVEL_STAGGERED +row_buf_policy = OPEN_PAGE +cmd_queue_size = 8 +trans_queue_size = 32 + +[other] +epoch_period = 1204819 +output_level = 1 + diff --git a/src/FP16.cc b/src/FP16.cc new file mode 100644 index 00000000..f10286dc --- /dev/null +++ b/src/FP16.cc @@ -0,0 +1,33 @@ +#include "FP16.h" + +FP16::FP16(fp16 val) : value(val) {} + +FP16 FP16::operator+(const FP16& other) const { + fp16 sum = value + other.value; + return FP16(sum); +} + +FP16 FP16::operator-(const FP16& other) const { + fp16 diff = value - other.value; + return FP16(diff); +} + +FP16 FP16::operator*(const FP16& other) const { + fp16 product = value * other.value; + return FP16(product); +} + +FP16 FP16::operator/(const FP16& other) const { + fp16 div = value / other.value; + return FP16(div); +} + +float FP16::toFloat() const { + float fp32 = value; + return fp32; +} + +FP16 FP16::fromFloat(float fp32) { + fp16 val0(fp32); + return val0; +} \ No newline at end of file diff --git a/src/FP16.h b/src/FP16.h new file mode 100644 index 00000000..b04e2663 --- /dev/null +++ b/src/FP16.h @@ -0,0 +1,23 @@ +#ifndef FP16_H +#define FP16_H + +#include "half.hpp" + +typedef half_float::half fp16; + +struct FP16 { + fp16 value; + + FP16(fp16 val); + + FP16 operator+(const FP16& other) const; + FP16 operator-(const FP16& other) const; + FP16 operator*(const FP16& other) const; + FP16 operator/(const FP16& other) const; + + float toFloat() const; + + static FP16 fromFloat(float fp32); +}; + +#endif // FP16_H \ No newline at end of file diff --git a/src/FixedPoint.cc b/src/FixedPoint.cc new file mode 100644 index 00000000..4422d0af --- /dev/null +++ b/src/FixedPoint.cc @@ -0,0 +1,34 @@ + + +#include "FixedPoint.h" + +FixedPoint::FixedPoint(int32_t val, int frac_bits) : value(val), fractional_bits(frac_bits) {} + +FixedPoint FixedPoint::operator+(const FixedPoint& other) const { + int32_t sum = value + other.value; + return FixedPoint(sum, fractional_bits); +} + +FixedPoint FixedPoint::operator-(const FixedPoint& other) const { + int32_t diff = value - other.value; + return FixedPoint(diff, fractional_bits); +} + +FixedPoint FixedPoint::operator*(const FixedPoint& other) const { + int32_t product = (static_cast(value) * other.value) >> fractional_bits; + return FixedPoint(product, fractional_bits); +} + +FixedPoint FixedPoint::operator/(const FixedPoint& other) const { + int32_t div = (static_cast(value) << fractional_bits) / other.value; + return FixedPoint(div, fractional_bits); +} + +float FixedPoint::toFloat() const { + return (float(value) / (1 << fractional_bits)); +} + +FixedPoint FixedPoint::fromFloat(float fp32, int frac_bits) { + int32_t fixed_value = static_cast(fp32 * (1 << frac_bits)); + return FixedPoint(fixed_value, frac_bits); +} diff --git a/src/FixedPoint.h b/src/FixedPoint.h new file mode 100644 index 00000000..a05d6f0a --- /dev/null +++ b/src/FixedPoint.h @@ -0,0 +1,23 @@ +#ifndef FIXEDPOINT_H +#define FIXEDPOINT_H + +#include + +struct FixedPoint { + int32_t value; + int fractional_bits; + + FixedPoint(int32_t val, int frac_bits); + + FixedPoint operator+(const FixedPoint& other) const; + FixedPoint operator-(const FixedPoint& other) const; + FixedPoint operator*(const FixedPoint& other) const; + FixedPoint operator/(const FixedPoint& other) const; + + float toFloat() const; + + static FixedPoint fromFloat(float fp32, int frac_bits); + +}; + +#endif \ No newline at end of file diff --git a/src/bankstate.cc b/src/bankstate.cc index 46aa5116..fa1540f4 100644 --- a/src/bankstate.cc +++ b/src/bankstate.cc @@ -7,6 +7,10 @@ BankState::BankState() cmd_timing_(static_cast(CommandType::SIZE)), open_row_(-1), row_hit_count_(0) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor"<(CommandType::READ)] = 0; cmd_timing_[static_cast(CommandType::READ_PRECHARGE)] = 0; cmd_timing_[static_cast(CommandType::WRITE)] = 0; @@ -16,6 +20,7 @@ BankState::BankState() cmd_timing_[static_cast(CommandType::REFRESH)] = 0; cmd_timing_[static_cast(CommandType::SREF_ENTER)] = 0; cmd_timing_[static_cast(CommandType::SREF_EXIT)] = 0; + cmd_timing_[static_cast(CommandType::MRS)] = 0; } @@ -33,6 +38,7 @@ Command BankState::GetReadyCommand(const Command& cmd, uint64_t clk) const { case CommandType::REFRESH: case CommandType::REFRESH_BANK: case CommandType::SREF_ENTER: + case CommandType::MRS: required_type = cmd.cmd_type; break; default: @@ -56,6 +62,7 @@ Command BankState::GetReadyCommand(const Command& cmd, uint64_t clk) const { case CommandType::REFRESH: case CommandType::REFRESH_BANK: case CommandType::SREF_ENTER: + case CommandType::MRS: required_type = CommandType::PRECHARGE; break; default: @@ -70,6 +77,7 @@ Command BankState::GetReadyCommand(const Command& cmd, uint64_t clk) const { case CommandType::READ_PRECHARGE: case CommandType::WRITE: case CommandType::WRITE_PRECHARGE: + case CommandType::MRS: required_type = CommandType::SREF_EXIT; break; default: @@ -113,6 +121,7 @@ void BankState::UpdateState(const Command& cmd) { case CommandType::REFRESH_BANK: case CommandType::SREF_ENTER: case CommandType::SREF_EXIT: + case CommandType::MRS: default: AbruptExit(__FILE__, __LINE__); } @@ -121,6 +130,7 @@ void BankState::UpdateState(const Command& cmd) { switch (cmd.cmd_type) { case CommandType::REFRESH: case CommandType::REFRESH_BANK: + case CommandType::MRS: break; case CommandType::ACTIVATE: state_ = State::OPEN; @@ -154,6 +164,7 @@ void BankState::UpdateState(const Command& cmd) { case CommandType::REFRESH: case CommandType::REFRESH_BANK: case CommandType::SREF_ENTER: + case CommandType::MRS: default: AbruptExit(__FILE__, __LINE__); } diff --git a/src/bob.cc b/src/bob.cc new file mode 100644 index 00000000..de3813d2 --- /dev/null +++ b/src/bob.cc @@ -0,0 +1,405 @@ +#include "bob.h" +#include + +namespace dramsim3 { + +// constexpr uint64_t LRDIMM::wr_dq_map_per_db[2][8][8]; +// constexpr uint64_t LRDIMM::rd_dq_map_per_db[2][8][8]; + +RCD::RCD(uint32_t dimm_idx, const Config& config, + SimpleStats& simple_stats) + : dimm_idx_(dimm_idx), + config_(config), + simple_stats_(simple_stats), + clk_(0) { + + /* + (A[12:8] == 5'b00100) F0RC4x: CW Source Selection Control (DA[7:0]) + DA[7:5] : Function Space 0 ~ 7 + DA[4:0] : A12 ~ A8 + (A[12:8] == 5'b00110) F0RC6x: CW Data Control Word (DA[7:0]) + DA[7:0] : A7 ~ A0 + (A[12:4] == 10'b100000110) F0RC06: Command Space Control Word + DA[3:0] == 0101 : CMD5 (CW Write operation) + DA[3:0] == 0100 : CMD4 (CW Read operation) + */ + f0rc4x = 0; // CW Source Selection Control + f0rc6x = 0; // CW Data Control Word + f0rc06 = 0; // Command Space Control Word + +} + +void RCD::recDDRcmd(const Command& cmd) { + if(cmd.IsRead()) { + bcom_cmds_.push_back(BCOMCmd(BCOMCmdType::BCOM_RD)); + bcom_cmds_timing_.push_back(4); // 4 tCK + simple_stats_.Increment("num_bcom_rd"); + simple_stats_.IncrementVec("bcom_rd_cmd",dimm_idx_); + + #ifdef MY_DEBUG + std::cout<<"["<> 8) == 0x4) { // Setting F0RC4x + f0rc4x = da & 0xFF; + } else if((da >> 8) == 0x6) { // Settting CW Data Control Word + f0rc6x = da & 0xFF; + } else if((da >> 4) == 0x106) { // Settting CW Data Control Word + int cmd_type = da & 0xF; + assert(cmd_type == 0x4); // Current Only Support WR Write Operation + if(cmd_type==0x4) {}; // remove warning Message + int fs = (f0rc4x >> 5) & 0x7; + int bcw_da = (f0rc4x & 0x1F) << 7 | f0rc6x; + bcom_cmds_.push_back(BCOMCmd(BCOMCmdType::BCOM_BCW_WR,bcw_da,fs)); + bcom_cmds_timing_.push_back(8); // 8 tCK + simple_stats_.Increment("num_bcom_bcw_wr"); + simple_stats_.IncrementVec("bcom_bcw_wr_cmd",dimm_idx_); + } + } +} + +BCOMCmd RCD::updateRCD() { + ClockTick(); + // BCOM Command Timing Check and Send BCOM to DB + bool is_issued = false; + for(uint32_t i=0;i=0); + if(bcom_cmds_timing_[i]==0) { + assert(i==0); + if(is_issued) assert(false); + is_issued = true; + } + } + if(is_issued) { + auto bcom_cmd = bcom_cmds_[0]; + bcom_cmds_.erase(bcom_cmds_.begin()); + bcom_cmds_timing_.erase(bcom_cmds_timing_.begin()); + return bcom_cmd; + } + else { + return BCOMCmd(); + } + +} + +DB::DB(uint32_t dimm_idx, uint32_t idx, const Config& config, + SimpleStats& simple_stats) + : dimm_idx_(dimm_idx), + db_idx_(idx), + config_(config), + simple_stats_(simple_stats), + clk_(0) {} + +void DB::updateDB() { + ClockTick(); +} + +void DB::recBCOMcmd(const BCOMCmd& cmd) { + //TBD +} + +void DB::recWRData(std::vector &payload) { + //TBD +} + +void DB::recRDData(std::vector &payload) { + //TBD +} + +LRDIMM::LRDIMM(uint32_t dimm_idx, const Config& config, + SimpleStats& simple_stats) + : dimm_idx_(dimm_idx), + config_(config), + simple_stats_(simple_stats), + rcd_(dimm_idx,config_,simple_stats_), + clk_(0) { + + // Access Granularity = bus_width * BL + // Payload Size = Access Granularity / 64 bit + payload_size = config_.bus_width * config_.BL / sizeof(uint64_t) / 8; + for(int i=0;i &payload) { + payload_q.push_back(make_pair(hex_addr,payload)); +} + +void LRDIMM::wrDataMem(uint64_t hex_addr, std::vector &payload) { + // Currently, data payload size is fixed to 8 (hardcode) + auto addr = config_.AddressMapping(hex_addr); + if(data_memory.find(hex_addr) != data_memory.end()) { + assert(data_memory[hex_addr].size() == payload.size()); + for(uint32_t i=0;i new_payload; + for(uint32_t i=0;i LRDIMM::rdDataMem(u_int64_t hex_addr) { + if(data_memory.find(hex_addr) != data_memory.end()) { + auto addr = config_.AddressMapping(hex_addr); + std::vector new_payload; + auto written_payload = data_memory[hex_addr]; + for(uint32_t i=0;i unwritten_memory; + for(uint32_t i=0;i> (8*burst_idx)) & 0xFF; + for(uint32_t bit_idx=0;bit_idx<8;bit_idx++) + reshaped_data|= (((one_byte >> wr_dq_map_per_db[rank_idx%2][db_idx][7-bit_idx]) & + 0x1)) << (8*burst_idx+bit_idx); + } + return reshaped_data; +} + +uint64_t LRDIMM::data_reshape_rd(int rank_idx, uint64_t db_idx, uint64_t wr_data) { + // Currently, data payload size is fixed to 8 (hardcode) + assert(db_idx<8); + uint64_t reshaped_data = 0; + uint64_t one_byte = 0; + for(uint32_t burst_idx=0;burst_idx<(uint32_t)config_.BL;burst_idx++) { + one_byte = (wr_data >> (8*burst_idx)) & 0xFF; + for(uint32_t bit_idx=0;bit_idx<8;bit_idx++) + reshaped_data|= (((one_byte >> rd_dq_map_per_db[rank_idx%2][db_idx][7-bit_idx]) & + 0x1)) << (8*burst_idx+bit_idx); + } + return reshaped_data; +} + +bool LRDIMM::isRDResp() { + if(rd_resp_timing_.size()!=0) + if(rd_resp_timing_[0]==0) return true; + + return false; +} + +rd_resp_t LRDIMM::getRDResp() { + if(isRDResp()) { + assert(rd_cmd_pipe.size() > 0); + auto it = rd_cmd_pipe.front(); + rd_cmd_pipe.erase(rd_cmd_pipe.begin()); + rd_resp_timing_.erase(rd_resp_timing_.begin()); + return it; + } + + return make_pair(Command(),null_payload); +} + +void LRDIMM::updateLRDIMM() { + ClockTick(); + + //update RD Resp Timing + for(u_int32_t i=0;i payload_4b; + for(u_int32_t i=0;i(0xFFFF & (it.second[i]>>j*16))); + } + uint32_t each_db_payload_size = sizeof(uint64_t) * payload_size/config_.dbs_per_dimm/sizeof(uint16_t); + uint32_t offset = 0; + for(int i=0;i a; + for(u_int32_t j=0;j payload_4b; + for(u_int32_t i=0;i(0xFFFF & (rd_data[i]>>j*16))); + } + uint32_t each_db_payload_size = payload_size/config_.dbs_per_dimm/sizeof(uint16_t); + uint32_t offset = 0; + for(int i=0;i a; + for(u_int32_t j=0;j(ceil((double)config_.tRPRE/2)) + + config_.burst_cycle - 5; + rd_data_timing_.push_back(rd_latency); + } + } + for(int i=0;i=0 && dimm_idx (cmd.cmd_type)]; + std::cout<<" Rec DDR Command (R:"< &payload) { + dimms_[rank / config_.ranks_per_dimm].enqueWRData(hex_addr, payload); + #ifdef MY_DEBUG + std::cout<<"== "<<__FILE__<<":"<<__func__<<" == " << + "["<> rd_resp_t; +enum class BCOMCmdType { + BCOM_RD, + BCOM_WR, + BCOM_BCW_WR, + BCOM_BCW_RD, // Maybe.. not use + SIZE +}; + +struct BCOMCmd { + BCOMCmd() : cmd_type(BCOMCmdType::SIZE), da(0), function_space(0) {} + BCOMCmd(BCOMCmdType cmd_type) : cmd_type(cmd_type), da(0), function_space(0) {} + BCOMCmd(BCOMCmdType cmd_type, int da, int function_space) : cmd_type(cmd_type), da(da), function_space(function_space) {} + + bool IsValid() const { return cmd_type != BCOMCmdType::SIZE; } + bool IsRead() const { return cmd_type == BCOMCmdType::BCOM_RD; } + bool IsWrite() const { return cmd_type == BCOMCmdType::BCOM_WR; } + bool IsBCWWrite() const { return cmd_type == BCOMCmdType::BCOM_WR; } + BCOMCmdType cmd_type; + int da; + int function_space; +}; +class RCD { + public: + RCD(uint32_t dimm_idx, const Config& config, SimpleStats& simple_stats); + BCOMCmd updateRCD(); // Update RCD State and Send BCOM command to DB + void recDDRcmd(const Command& cmd); // Received DDR Command from MC + void ClockTick() { clk_ += 1; }; // Update Clock Tick + + private: + uint32_t dimm_idx_; + const Config& config_; + SimpleStats& simple_stats_; + uint64_t clk_; + + // RCD Register for BCW Command for future NDP function + uint16_t f0rc4x; + uint16_t f0rc6x; + uint16_t f0rc06; + + // BCOM Command Timing Tacking + std::vector bcom_cmds_; + std::vector bcom_cmds_timing_; +}; + +class DB { + public: + DB(uint32_t dimm_idx, uint32_t idx, const Config& config, SimpleStats& simple_stats); + void recBCOMcmd(const BCOMCmd& cmd); // recieved BCOM Command From RCD + void updateDB(); // Update LRDIMM State + void ClockTick() { clk_ += 1; }; // Update Clock Tick + void recWRData(std::vector &payload); // Write Data into DB + void recRDData(std::vector &payload); // Write Data into DB + + private: + uint32_t dimm_idx_; + uint32_t db_idx_; + const Config& config_; + SimpleStats& simple_stats_; + uint64_t clk_; + +}; + +class LRDIMM { + public: + LRDIMM(uint32_t dimm_idx, const Config& config, SimpleStats& simple_stats); + void recDDRcmd(const Command& cmd); // Send DDR Command to RCD + void updateLRDIMM(); // Update LRDIMM State + void ClockTick() { clk_ += 1; }; // Update Clock Tick + void enqueWRData(u_int64_t hex_addr, std::vector &payload); // Write Data into payload Queue + void wrDataMem(u_int64_t hex_addr, std::vector &payload); // Write Data into data memory + rd_resp_t getRDResp(); // Host Get Read Resp with Data + bool isRDResp(); // is Resp Data + std::vector rdDataMem(u_int64_t hex_addr); // Read Data into data memory + uint64_t data_reshape_wr(int rank_idx, uint64_t db_idx, uint64_t wr_data); + uint64_t data_reshape_rd(int rank_idx, uint64_t db_idx, uint64_t rd_data); + private: + uint32_t dimm_idx_; + + const Config& config_; + SimpleStats& simple_stats_; + RCD rcd_; // LDRIMM Submodule + + uint64_t clk_; + uint32_t payload_size; + + std::vector dbs_; // LDRIMM Submodule + std::vector>> payload_q; + std::unordered_map> data_memory; + // @TODO: If memory accesses to data_memory cause high latency, you should replace the data structure of data_memory with another data structure. + + // BCOM Command Timing Tacking + std::vector wr_data_timing_; + std::vector rd_data_timing_; + std::vector rd_resp_timing_; + std::vector rd_addr_pipe; + std::vector rd_cmd_pipe; + std::vector null_payload; + + /* + static constexpr uint64_t wr_dq_map_per_db[2][8][8] = { + {{4,6,5,7,3,1,2,0}, // DB0 for Rank0, 2 ( 4, 6, 5, 7, 3, 1, 2, 0) + {5,7,4,6,1,3,0,2}, // DB1 for Rank0, 2 (13,15,12,14, 9,11, 8,10) + {4,7,5,6,2,0,3,1}, // DB2 for Rank0, 2 (20,23,21,22,18,16,19,17) + {4,6,5,7,2,0,3,1}, // DB3 for Rank0, 2 (28,30,29,31,26,24,27,25) + {5,7,4,6,1,3,0,2}, // DB4 for Rank0, 2 (37,39,36,38,33,35,32,34) + {5,7,4,6,3,1,2,0}, // DB5 for Rank0, 2 (45,47,44,46,43,41,42,40) + {7,5,6,4,1,3,0,2}, // DB6 for Rank0, 2 (55,53,54,52,49,51,48,50) + {7,5,6,4,0,3,1,2}}, // DB7 for Rank0, 2 (63,61,62,60,56,59,57,58) + {{6,4,7,5,1,3,0,2}, // DB0 for Rank1, 3 ( 6, 4, 7, 5, 1, 3, 0, 2) + {7,5,6,4,3,1,2,0}, // DB1 for Rank1, 3 (15,13,14,12,11, 9,10, 8) + {7,4,6,5,0,2,1,3}, // DB2 for Rank1, 3 (23,20,22,21,16,18,17,19) + {6,4,7,5,0,2,1,3}, // DB3 for Rank1, 3 (30,28,31,29,24,26,25,27) + {7,5,6,4,3,1,2,0}, // DB4 for Rank1, 3 (39,37,38,36,35,33,34,32) + {7,5,6,4,1,3,0,2}, // DB5 for Rank1, 3 (47,45,46,44,41,43,40,42) + {5,7,4,6,3,1,2,0}, // DB6 for Rank1, 3 (53,55,52,54,51,49,50,48) + {5,7,4,6,3,0,2,1}}};// DB7 for Rank1, 3 (61,63,60,62,59,56,58,57) + + static constexpr uint64_t rd_dq_map_per_db[2][8][8] = { + {{4,6,5,7,3,1,2,0}, // DB0 for Rank0, 2 + {6,4,7,5,2,0,3,1}, // DB1 for Rank0, 2 + {6,4,5,7,1,3,0,2}, // DB2 for Rank0, 2 + {4,6,5,7,1,3,0,2}, // DB3 for Rank0, 2 + {6,4,7,5,2,0,3,1}, // DB4 for Rank0, 2 + {6,4,7,5,3,1,2,0}, // DB5 for Rank0, 2 + {7,5,6,4,2,0,3,1}, // DB6 for Rank0, 2 + {7,5,6,4,2,0,1,3}}, // DB7 for Rank0, 2 + {{5,7,4,6,2,0,3,1}, // DB0 for Rank1, 3 + {7,5,6,4,3,1,2,0}, // DB1 for Rank1, 3 + {7,5,4,6,0,2,1,3}, // DB2 for Rank1, 3 + {5,7,4,6,0,2,1,3}, // DB3 for Rank1, 3 + {7,5,6,4,3,1,2,0}, // DB4 for Rank1, 3 + {7,5,6,4,2,0,3,1}, // DB5 for Rank1, 3 + {6,4,7,5,3,1,2,0}, // DB6 for Rank1, 3 + {6,4,7,5,3,1,0,2}}};// DB7 for Rank1, 3 + */ +}; + + +class BufferOnBoard { + public: + BufferOnBoard(const Config& config, SimpleStats& simple_stats); + void recDDRcmd(const Command& cmd); // Send DDR Command to target DIMM + void updateBoB(); // Update BoB State + void ClockTick() { clk_ += 1; }; // Update Clock Tick + void enqueWRData(int rank, uint64_t hex_addr, std::vector &payload); // Write Data into payload Queue + rd_resp_t getRDResp(); // Host Get Read Resp with Data + + private: + const Config& config_; + SimpleStats& simple_stats_; + uint64_t clk_; + + // LRDIMM Vector + std::vector dimms_; + +}; +} // namespace dramsim3 +#endif diff --git a/src/channel_state.cc b/src/channel_state.cc index 765a18a9..02f81c68 100644 --- a/src/channel_state.cc +++ b/src/channel_state.cc @@ -8,8 +8,13 @@ ChannelState::ChannelState(const Config& config, const Timing& timing) rank_is_sref_(config.ranks, false), four_aw_(config_.ranks, std::vector()), thirty_two_aw_(config_.ranks, std::vector()) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor"<>(); rank_states.reserve(config_.bankgroups); for (auto j = 0; j < config_.bankgroups; j++) { @@ -175,6 +180,7 @@ void ChannelState::UpdateTiming(const Command& cmd, uint64_t clk) { case CommandType::REFRESH: case CommandType::SREF_ENTER: case CommandType::SREF_EXIT: + case CommandType::MRS: UpdateSameRankTiming( cmd.addr, timing_.same_rank[static_cast(cmd.cmd_type)], clk); diff --git a/src/command_queue.cc b/src/command_queue.cc index f121d28f..ad1d9c3f 100644 --- a/src/command_queue.cc +++ b/src/command_queue.cc @@ -13,6 +13,10 @@ CommandQueue::CommandQueue(int channel_id, const Config& config, queue_size_(static_cast(config_.cmd_queue_size)), queue_idx_(0), clk_(0) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor"<(); + mrs_queue.reserve(config_.cmd_queue_size); + mrs_queue_ = mrs_queue; } Command CommandQueue::GetCommandToIssue() { - for (int i = 0; i < num_queues_; i++) { - auto& queue = GetNextQueue(); - // if we're refresing, skip the command queues that are involved - if (is_in_ref_) { - if (ref_q_indices_.find(queue_idx_) != ref_q_indices_.end()) { - continue; + if(mrs_queue_.size() > 0) { + // MRP Command also can be issued when not refreshing + if(!is_in_ref_) { + auto cmd = GetFirstReadyInQueue(mrs_queue_); + if(cmd.IsValid()) { + #ifdef MY_DEBUG + std::cout<<"== "<<__FILE__<<":"<<__func__<<" == " << + "["<IsMRSCMD()) break; + else continue; } - if (cmd.cmd_type == CommandType::PRECHARGE) { + if (cmd.cmd_type == CommandType::PRECHARGE && !cmd_it->IsMRSCMD()) { if (!ArbitratePrecharge(cmd_it, queue)) { continue; } @@ -207,6 +262,17 @@ void CommandQueue::EraseRWCommand(const Command& cmd) { exit(1); } +void CommandQueue::EraseMRSCommand(const Command& cmd) { + for (auto cmd_it = mrs_queue_.begin(); cmd_it != mrs_queue_.end(); cmd_it++) { + if (cmd.hex_addr == cmd_it->hex_addr && cmd.cmd_type == cmd_it->cmd_type) { + mrs_queue_.erase(cmd_it); + return; + } + } + std::cerr << "cannot find cmd!" << std::endl; + exit(1); +} + int CommandQueue::QueueUsage() const { int usage = 0; for (auto i = queues_.begin(); i != queues_.end(); i++) { diff --git a/src/command_queue.h b/src/command_queue.h index 5bc8d3fc..7de4da07 100644 --- a/src/command_queue.h +++ b/src/command_queue.h @@ -22,6 +22,7 @@ class CommandQueue { Command FinishRefresh(); void ClockTick() { clk_ += 1; }; bool WillAcceptCommand(int rank, int bankgroup, int bank) const; + bool WillAcceptMRSCommand() const; bool AddCommand(Command cmd); bool QueueEmpty() const; int QueueUsage() const; @@ -38,6 +39,7 @@ class CommandQueue { CMDQueue& GetNextQueue(); void GetRefQIndices(const Command& ref); void EraseRWCommand(const Command& cmd); + void EraseMRSCommand(const Command& cmd); Command PrepRefCmd(const CMDIterator& it, const Command& ref) const; QueueStructure queue_structure_; @@ -46,6 +48,8 @@ class CommandQueue { SimpleStats& simple_stats_; std::vector queues_; + CMDQueue mrs_queue_; // MRS Command has high priority than other command + // So, Add Dedicated Queue for MRS Command // Refresh related data structures std::unordered_set ref_q_indices_; diff --git a/src/common.cc b/src/common.cc index 5be3303c..c9173b8e 100644 --- a/src/common.cc +++ b/src/common.cc @@ -18,6 +18,7 @@ std::ostream& operator<<(std::ostream& os, const Command& cmd) { "refresh", "self_refresh_enter", "self_refresh_exit", + "mrs", "WRONG"}; os << fmt::format("{:<20} {:>3} {:>3} {:>3} {:>3} {:>#8x} {:>#8x}", command_string[static_cast(cmd.cmd_type)], @@ -27,17 +28,18 @@ std::ostream& operator<<(std::ostream& os, const Command& cmd) { } std::ostream& operator<<(std::ostream& os, const Transaction& trans) { - const std::string trans_type = trans.is_write ? "WRITE" : "READ"; + const std::string trans_type = trans.is_MRS ? "MRS" : trans.is_write ? "WRITE" : "READ"; os << fmt::format("{:<30} {:>8}", trans.addr, trans_type); return os; } std::istream& operator>>(std::istream& is, Transaction& trans) { std::unordered_set write_types = {"WRITE", "write", "P_MEM_WR", - "BOFF"}; + "BOFF", "MRS"}; std::string mem_op; is >> std::hex >> trans.addr >> mem_op >> std::dec >> trans.added_cycle; trans.is_write = write_types.count(mem_op) == 1; + trans.is_MRS = (mem_op == "MRS"); return is; } diff --git a/src/common.h b/src/common.h index 2f3f0443..b23f6383 100644 --- a/src/common.h +++ b/src/common.h @@ -3,7 +3,10 @@ #include #include +#include +#include #include +#include namespace dramsim3 { @@ -30,6 +33,77 @@ struct Address { int bank; int row; int column; + + void display() { + std::cout<<"Address: CH["< &payload_) + : addr(addr), + added_cycle(0), + complete_cycle(0), + is_write(is_write), + is_MRS(is_mrs) { + assert(!(!is_write && is_mrs)); // MRS Command is Write-Type + // payload Value Copy + assert(payload.size() == 0); // If size of payload must be zero when generating object, assert + for(uint32_t i=0;i payload; bool is_write; - + bool is_MRS; // Temporaily add an MRS Flag @TODO it must be removed later + bool is_valid; + void updatePayload(std::vector &payload_) { + for(uint32_t i=0;i D:"; + for(auto value : payload) std::cout<<"["<>(std::istream& is, Transaction& trans); }; + +static constexpr uint64_t wr_dq_map_per_db[2][8][8] = { + {{4,6,5,7,3,1,2,0}, // DB0 for Rank0, 2 ( 4, 6, 5, 7, 3, 1, 2, 0) + {5,7,4,6,1,3,0,2}, // DB1 for Rank0, 2 (13,15,12,14, 9,11, 8,10) + {4,7,5,6,2,0,3,1}, // DB2 for Rank0, 2 (20,23,21,22,18,16,19,17) + {4,6,5,7,2,0,3,1}, // DB3 for Rank0, 2 (28,30,29,31,26,24,27,25) + {5,7,4,6,1,3,0,2}, // DB4 for Rank0, 2 (37,39,36,38,33,35,32,34) + {5,7,4,6,3,1,2,0}, // DB5 for Rank0, 2 (45,47,44,46,43,41,42,40) + {7,5,6,4,1,3,0,2}, // DB6 for Rank0, 2 (55,53,54,52,49,51,48,50) + {7,5,6,4,0,3,1,2}}, // DB7 for Rank0, 2 (63,61,62,60,56,59,57,58) + {{6,4,7,5,1,3,0,2}, // DB0 for Rank1, 3 ( 6, 4, 7, 5, 1, 3, 0, 2) + {7,5,6,4,3,1,2,0}, // DB1 for Rank1, 3 (15,13,14,12,11, 9,10, 8) + {7,4,6,5,0,2,1,3}, // DB2 for Rank1, 3 (23,20,22,21,16,18,17,19) + {6,4,7,5,0,2,1,3}, // DB3 for Rank1, 3 (30,28,31,29,24,26,25,27) + {7,5,6,4,3,1,2,0}, // DB4 for Rank1, 3 (39,37,38,36,35,33,34,32) + {7,5,6,4,1,3,0,2}, // DB5 for Rank1, 3 (47,45,46,44,41,43,40,42) + {5,7,4,6,3,1,2,0}, // DB6 for Rank1, 3 (53,55,52,54,51,49,50,48) + {5,7,4,6,3,0,2,1}}};// DB7 for Rank1, 3 (61,63,60,62,59,56,58,57) + +static constexpr uint64_t rd_dq_map_per_db[2][8][8] = { +{{4,6,5,7,3,1,2,0}, // DB0 for Rank0, 2 + {6,4,7,5,2,0,3,1}, // DB1 for Rank0, 2 + {6,4,5,7,1,3,0,2}, // DB2 for Rank0, 2 + {4,6,5,7,1,3,0,2}, // DB3 for Rank0, 2 + {6,4,7,5,2,0,3,1}, // DB4 for Rank0, 2 + {6,4,7,5,3,1,2,0}, // DB5 for Rank0, 2 + {7,5,6,4,2,0,3,1}, // DB6 for Rank0, 2 + {7,5,6,4,2,0,1,3}}, // DB7 for Rank0, 2 + {{5,7,4,6,2,0,3,1}, // DB0 for Rank1, 3 + {7,5,6,4,3,1,2,0}, // DB1 for Rank1, 3 + {7,5,4,6,0,2,1,3}, // DB2 for Rank1, 3 + {5,7,4,6,0,2,1,3}, // DB3 for Rank1, 3 + {7,5,6,4,3,1,2,0}, // DB4 for Rank1, 3 + {7,5,6,4,2,0,3,1}, // DB5 for Rank1, 3 + {6,4,7,5,3,1,2,0}, // DB6 for Rank1, 3 + {6,4,7,5,3,1,0,2}}};// DB7 for Rank1, 3 + } // namespace dramsim3 #endif diff --git a/src/configuration.cc b/src/configuration.cc index 6608aec4..74c58e1b 100644 --- a/src/configuration.cc +++ b/src/configuration.cc @@ -1,7 +1,8 @@ #include "configuration.h" +#include #include - +#include #ifdef THERMAL #include #endif // THERMAL @@ -41,6 +42,26 @@ Address Config::AddressMapping(uint64_t hex_addr) const { return Address(channel, rank, bg, ba, ro, co); } + +uint64_t Config::MergedAddress(uint64_t channel, uint64_t rank, uint64_t bg, + uint64_t ba, uint64_t ro, uint64_t co) const { + uint64_t merged_addr = (ro << ro_pos) | (channel << ch_pos) | (rank << ra_pos) | + (ba << ba_pos) | (bg << bg_pos) | co; + + return merged_addr; +} + +uint64_t Config::MergedAddress(Address addr) const { + uint64_t merged_addr = (addr.rank << ro_pos) | + (addr.channel << ch_pos) | + (addr.rank << ra_pos) | + (addr.bank << ba_pos) | + (addr.bankgroup << bg_pos) | + addr.column; + + return merged_addr; +} + void Config::CalculateSize() { // calculate rank and re-calculate channel_size devices_per_rank = bus_width / device_width; @@ -59,13 +80,56 @@ void Config::CalculateSize() { ranks = channel_size / megs_per_rank; channel_size = ranks * megs_per_rank; } + + if(is_LRDIMM) { + if(ranks_per_dimm > ranks) { + std::cout << "WARNING: Ranks is lower than Ranks per DIMM. Ranks (" + << ranks + << ") Ranks Per DIMM " + << ranks_per_dimm << ") --> Changes Ranks (=Ranks per DIMM)"< Changes Ranks (multiple of ranks per DIMM)"< LRDIMM ranks : "< LRDIMM ranks_per_dimm : "< LRDIMM dimms : "< DBs per DIMM : "< protocol_pairs = { {"DDR3", DRAMProtocol::DDR3}, {"DDR4", DRAMProtocol::DDR4}, - {"GDDR5", DRAMProtocol::GDDR5}, {"GDDR5X", DRAMProtocol::GDDR5X}, + {"GDDR5", DRAMProtocol::GDDR5}, {"GDDR5X", DRAMProtocol::GDDR5X}, {"GDDR6", DRAMProtocol::GDDR6}, {"LPDDR", DRAMProtocol::LPDDR}, {"LPDDR3", DRAMProtocol::LPDDR3}, {"LPDDR4", DRAMProtocol::LPDDR4}, {"HBM", DRAMProtocol::HBM}, {"HBM2", DRAMProtocol::HBM2}, {"HMC", DRAMProtocol::HMC}}; @@ -92,7 +156,7 @@ void Config::InitDRAMParams() { banks_per_group = GetInteger("dram_structure", "banks_per_group", 2); bool bankgroup_enable = reader.GetBoolean("dram_structure", "bankgroup_enable", true); - // GDDR5 can chose to enable/disable bankgroups + // GDDR5/6 can chose to enable/disable bankgroups if (!bankgroup_enable) { // aggregating all banks to one group banks_per_group *= bankgroups; bankgroups = 1; @@ -120,7 +184,7 @@ void Config::InitDRAMParams() { BL = block_size * 8 / device_width; } // set burst cycle according to protocol - // We use burst_cycle for timing and use BL for capaticyt calculation + // We use burst_cycle for timing and use BL for capacity calculation // BL = 0 simulate perfect BW if (protocol == DRAMProtocol::GDDR5) { burst_cycle = (BL == 0) ? 0 : BL / 4; @@ -128,11 +192,14 @@ void Config::InitDRAMParams() { } else if (protocol == DRAMProtocol::GDDR5X) { burst_cycle = (BL == 0) ? 0 : BL / 8; BL = (BL == 0) ? 8 : BL; + } else if (protocol == DRAMProtocol::GDDR6){ + burst_cycle = (BL == 0) ? 0 : BL / 16; + BL = (BL == 0 ) ? 8 : BL; } else { burst_cycle = (BL == 0) ? 0 : BL / 2; BL = (BL == 0) ? (IsHBM() ? 4 : 8) : BL; } - // every protocol has a different defination of "column", + // every protocol has a different definition of "column", // in DDR3/4, each column is exactly device_width bits, // but in GDDR5, a column is device_width * BL bits // and for HBM each column is device_width * 2 (prefetch) @@ -241,10 +308,15 @@ void Config::InitSystemParams() { aggressive_precharging_enabled = reader.GetBoolean("system", "aggressive_precharging_enabled", false); - mega_tick = GetInteger("system", "mega_tick", 1); - if (mega_tick > 1) { - std::cout << "Mega Tick! " << std::endl; - } + // LRDIMM + is_LRDIMM = + reader.GetBoolean("system", "is_LRDIMM", false); + ranks_per_dimm = GetInteger("system", "ranks_per_dimm", 4); + dqs_per_db = GetInteger("system", "dqs_per_db", 8); + dimms = GetInteger("system", "dimms", 1); + dbs_per_dimm = GetInteger("system", "dbs_per_dimm", 8); + + return; } @@ -327,14 +399,21 @@ void Config::InitTimingParams() { tRPRE = GetInteger("timing", "tRPRE", 1); tWPRE = GetInteger("timing", "tWPRE", 1); - // LPDDR4 and GDDR5 + // LPDDR4 and GDDR5/6 tPPD = GetInteger("timing", "tPPD", 0); - // GDDR5 + // GDDR5/6 t32AW = GetInteger("timing", "t32AW", 330); tRCDRD = GetInteger("timing", "tRCDRD", 24); tRCDWR = GetInteger("timing", "tRCDWR", 20); + // MRS AC Parameter + tMRD = GetInteger("timing", "tMRD", 10); // DDR4-3200 + tMOD = GetInteger("timing", "tMOD", 24); // DDR4-3200 + + tPDM_RD = GetInteger("timing", "tPDM_RD", static_cast(ceil((1.37 + tCK/4)/tCK))); // 1.37 + tCK/4 + tPDM_WR = GetInteger("timing", "tPDM_WR", static_cast(ceil((1.37 + tCK/4)/tCK))); // 1.37 + tCK/4 + ideal_memory_latency = GetInteger("timing", "ideal_memory_latency", 10); // calculated timing diff --git a/src/configuration.h b/src/configuration.h index 7a058c86..2b224e40 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -14,6 +14,7 @@ enum class DRAMProtocol { DDR4, GDDR5, GDDR5X, + GDDR6, LPDDR, LPDDR3, LPDDR4, @@ -33,11 +34,15 @@ enum class RefreshPolicy { class Config { public: Config(std::string config_file, std::string out_dir); - Address AddressMapping(uint64_t hex_addr) const; + Address AddressMapping(uint64_t hex_addr) const; + uint64_t MergedAddress(uint64_t channel, uint64_t rank, uint64_t bg, uint64_t ba, uint64_t ro, uint64_t co) const; + uint64_t MergedAddress(Address addr) const; + // DRAM physical structure DRAMProtocol protocol; int channel_size; int channels; + int dimms; int ranks; int banks; int bankgroups; @@ -56,7 +61,7 @@ class Config { // Generic DRAM timing parameters double tCK; - int burst_cycle; // seperate BL with timing since fot GDDRx it's not BL/2 + int burst_cycle; // seperate BL with timing since for GDDRx it's not BL/2 int AL; int CL; int CWL; @@ -90,6 +95,9 @@ class Config { int tWPRE; int read_delay; int write_delay; + // MRS Timing Parameter + int tMRD; // Delay of MRS-to-MRS + int tMOD; // Delay of MRS-to-NonMRS // LPDDR4 and GDDR5 int tPPD; @@ -147,7 +155,8 @@ class Config { bool IsGDDR() const { return (protocol == DRAMProtocol::GDDR5 || - protocol == DRAMProtocol::GDDR5X); + protocol == DRAMProtocol::GDDR5X || + protocol == DRAMProtocol::GDDR6); } bool IsHBM() const { return (protocol == DRAMProtocol::HBM || @@ -157,9 +166,15 @@ class Config { // yzy: add another function bool IsDDR4() const { return (protocol == DRAMProtocol::DDR4); } - int ideal_memory_latency; + // LRDIMM + bool is_LRDIMM; + int ranks_per_dimm; + int dqs_per_db; + int dbs_per_dimm; + int tPDM_RD; + int tPDM_WR; - int mega_tick; + int ideal_memory_latency; #ifdef THERMAL std::string loc_mapping; diff --git a/src/controller.cc b/src/controller.cc index a582dcc2..8cf9786d 100644 --- a/src/controller.cc +++ b/src/controller.cc @@ -18,6 +18,7 @@ Controller::Controller(int channel, const Config &config, const Timing &timing) channel_state_(config, timing), cmd_queue_(channel_id_, config, channel_state_, simple_stats_), refresh_(config, channel_state_), + BufferOnBoard_(config,simple_stats_), #ifdef THERMAL thermal_calc_(thermal_calc), #endif // THERMAL @@ -27,6 +28,11 @@ Controller::Controller(int channel, const Config &config, const Timing &timing) : RowBufPolicy::OPEN_PAGE), last_trans_clk_(0), write_draining_(0) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor ("< Controller::ReturnDoneTrans(uint64_t clk) { auto it = return_queue_.begin(); while (it != return_queue_.end()) { if (clk >= it->complete_cycle) { - if (it->is_write) { + if (it->is_MRS) { + #ifdef MY_DEBUG + std::cout<<"== "<<__FILE__<<":"<<__func__<<" == " << + "["<is_write) { simple_stats_.Increment("num_writes_done"); } else { simple_stats_.Increment("num_reads_done"); simple_stats_.AddValue("read_latency", clk_ - it->added_cycle); } + if(config_.is_LRDIMM) { + assert(it->payload.size()!=0); + if(!it->is_write) resp_data_.push_back(it->payload); + } auto pair = std::make_pair(it->addr, it->is_write); it = return_queue_.erase(it); return pair; @@ -62,9 +84,34 @@ std::pair Controller::ReturnDoneTrans(uint64_t clk) { return std::make_pair(-1, -1); } +std::vector Controller::GetRespData() { + if(resp_data_.size() == 0) assert(false); + auto it = resp_data_.front(); + resp_data_.erase(resp_data_.begin()); + return it; +} + void Controller::ClockTick() { // update refresh counter refresh_.ClockTick(); + if(config_.is_LRDIMM) { + BufferOnBoard_.updateBoB(); + auto resp = BufferOnBoard_.getRDResp(); + if(resp.first.IsValid()) { + assert(resp.first.IsRead()); + bool isResp = false; + for(uint32_t i=0;isecond.updatePayload(trans.payload); + } trans.complete_cycle = clk_ + 1; return_queue_.push_back(trans); return true; } else { // read + #ifdef MY_DEBUG + std::cout<<"== "<<__FILE__<<":"<<__func__<<" == " << + "["< 0) { trans.complete_cycle = clk_ + 1; + auto pending_wr = pending_wr_q_.find(trans.addr); + assert(pending_wr_q_.count(trans.addr) == 1); + trans.updatePayload(pending_wr->second.payload); // Update Payload return_queue_.push_back(trans); return true; } @@ -204,25 +299,51 @@ void Controller::ScheduleTransaction() { } } - std::vector &queue = - is_unified_queue_ ? unified_queue_ - : write_draining_ > 0 ? write_buffer_ : read_queue_; + bool pop_MRS_Trsaction = mrs_buffer_.size() > 0; + std::vector &queue = + pop_MRS_Trsaction ? mrs_buffer_ : + is_unified_queue_ ? unified_queue_ : + write_draining_ > 0 ? write_buffer_ : read_queue_; + for (auto it = queue.begin(); it != queue.end(); it++) { - auto cmd = TransToCommand(*it); - if (cmd_queue_.WillAcceptCommand(cmd.Rank(), cmd.Bankgroup(), - cmd.Bank())) { - if (!is_unified_queue_ && cmd.IsWrite()) { - // Enforce R->W dependency - if (pending_rd_q_.count(it->addr) > 0) { - write_draining_ = 0; - break; - } - write_draining_ -= 1; + auto cmd = TransToCommand(*it); + if(pop_MRS_Trsaction) { + if(cmd_queue_.WillAcceptMRSCommand()) { + #ifdef MY_DEBUG + std::cout<<"["<W dependency + if (pending_rd_q_.count(it->addr) > 0) { + write_draining_ = 0; + break; + } + write_draining_ -= 1; + } + cmd_queue_.AddCommand(cmd); + #ifdef MY_DEBUG + std::cout<<"["< 0) { auto it = pending_rd_q_.find(cmd.hex_addr); it->second.complete_cycle = clk_ + config_.read_delay; + if(config_.is_LRDIMM) it->second.complete_cycle+=(config_.tPDM_RD+config_.tRPRE); return_queue_.push_back(it->second); pending_rd_q_.erase(it); num_reads -= 1; @@ -256,9 +378,15 @@ void Controller::IssueCommand(const Command &cmd) { std::cerr << cmd.hex_addr << " not in write queue!" << std::endl; exit(1); } + if(config_.is_LRDIMM) BufferOnBoard_.enqueWRData(cmd.Rank(), cmd.hex_addr, it->second.payload); auto wr_lat = clk_ - it->second.added_cycle + config_.write_delay; simple_stats_.AddValue("write_latency", wr_lat); pending_wr_q_.erase(it); + } else if (cmd.IsMRSCMD()) { + // Each MRS Command must be issued so that the MRS command does not merge + // with the previous MRS Command. There is no a pending queue for the MRS command. + + // @TODO Add State..Value? } // must update stats before states (for row hits) UpdateCommandStats(cmd); @@ -274,6 +402,8 @@ Command Controller::TransToCommand(const Transaction &trans) { cmd_type = trans.is_write ? CommandType::WRITE_PRECHARGE : CommandType::READ_PRECHARGE; } + + cmd_type = trans.is_MRS ? CommandType::MRS : cmd_type; return Command(cmd_type, addr, trans.addr); } @@ -339,9 +469,13 @@ void Controller::UpdateCommandStats(const Command &cmd) { case CommandType::SREF_EXIT: simple_stats_.Increment("num_srefx_cmds"); break; + case CommandType::MRS: + simple_stats_.Increment("num_mrs_cmds"); + break; default: AbruptExit(__FILE__, __LINE__); } } + } // namespace dramsim3 diff --git a/src/controller.h b/src/controller.h index 77bd6afd..84792f4e 100644 --- a/src/controller.h +++ b/src/controller.h @@ -10,6 +10,7 @@ #include "common.h" #include "refresh.h" #include "simple_stats.h" +#include "bob.h" #ifdef THERMAL #include "thermal.h" @@ -29,6 +30,7 @@ class Controller { #endif // THERMAL void ClockTick(); bool WillAcceptTransaction(uint64_t hex_addr, bool is_write) const; + bool WillAcceptTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) const; bool AddTransaction(Transaction trans); int QueueUsage() const; // Stats output @@ -36,6 +38,7 @@ class Controller { void PrintFinalStats(); void ResetStats() { simple_stats_.Reset(); } std::pair ReturnDoneTrans(uint64_t clock); + std::vector GetRespData(); int channel_id_; @@ -46,6 +49,8 @@ class Controller { ChannelState channel_state_; CommandQueue cmd_queue_; Refresh refresh_; + BufferOnBoard BufferOnBoard_; + #ifdef THERMAL ThermalCalculator &thermal_calc_; @@ -56,6 +61,7 @@ class Controller { std::vector unified_queue_; std::vector read_queue_; std::vector write_buffer_; + std::vector mrs_buffer_; // transactions that are not completed, use map for convenience std::multimap pending_rd_q_; @@ -67,6 +73,9 @@ class Controller { // row buffer policy RowBufPolicy row_buf_policy_; + //RD Resp Data Buffer + std::vector> resp_data_; + #ifdef CMD_TRACE std::ofstream cmd_trace_; #endif // CMD_TRACE diff --git a/src/cpu.cc b/src/cpu.cc index 08efb8eb..e0563309 100644 --- a/src/cpu.cc +++ b/src/cpu.cc @@ -1,4 +1,7 @@ #include "cpu.h" +#include +#include +#include namespace dramsim3 { @@ -7,17 +10,15 @@ void RandomCPU::ClockTick() { // this is useful to exploit the parallelism of a DRAM protocol // and is also immune to address mapping and scheduling policies memory_system_.ClockTick(); - for (int i = 0; i < mega_tick_; i++) { - if (get_next_) { - last_addr_ = gen(); - last_write_ = (gen() % 3 == 0); - } - get_next_ = memory_system_.WillAcceptTransaction(last_addr_, last_write_); - if (get_next_) { - memory_system_.AddTransaction(last_addr_, last_write_); - } else { - break; - } + if (get_next_) { + last_addr_ = gen(); + last_write_ = (gen() % 3 == 0); + last_mrs_ = last_write_ && (gen() % 20 == 0); + } + // get_next_ = memory_system_.WillAcceptTransaction(last_addr_, last_write_); + get_next_ = memory_system_.WillAcceptTransaction(last_addr_, last_write_,last_mrs_); + if (get_next_) { + memory_system_.AddTransaction(last_addr_, last_write_,last_mrs_); } clk_++; return; @@ -85,7 +86,8 @@ void TraceBasedCPU::ClockTick() { get_next_ = memory_system_.WillAcceptTransaction(trans_.addr, trans_.is_write); if (get_next_) { - memory_system_.AddTransaction(trans_.addr, trans_.is_write); + // memory_system_.AddTransaction(trans_.addr, trans_.is_write); + memory_system_.AddTransaction(trans_.addr, trans_.is_write, trans_.is_MRS); } } } diff --git a/src/cpu.h b/src/cpu.h index 781ca7f6..a06e6376 100644 --- a/src/cpu.h +++ b/src/cpu.h @@ -6,6 +6,7 @@ #include #include #include "memory_system.h" +#include namespace dramsim3 { @@ -16,17 +17,17 @@ class CPU { config_file, output_dir, std::bind(&CPU::ReadCallBack, this, std::placeholders::_1), std::bind(&CPU::WriteCallBack, this, std::placeholders::_1)), - clk_(0), - mega_tick_(memory_system_.GetMegaTick()) {} + clk_(0) {} virtual void ClockTick() = 0; void ReadCallBack(uint64_t addr) { return; } void WriteCallBack(uint64_t addr) { return; } void PrintStats() { memory_system_.PrintStats(); } + virtual ~CPU() = default; + std::vector access_history; protected: MemorySystem memory_system_; uint64_t clk_; - int mega_tick_; }; class RandomCPU : public CPU { @@ -39,6 +40,8 @@ class RandomCPU : public CPU { bool last_write_ = false; std::mt19937_64 gen; bool get_next_ = true; + bool last_mrs_ = false; + // std::unordered_map access_history; }; class StreamCPU : public CPU { diff --git a/src/custom_cpu.cc b/src/custom_cpu.cc new file mode 100644 index 00000000..3ae91e41 --- /dev/null +++ b/src/custom_cpu.cc @@ -0,0 +1,590 @@ +#include "custom_cpu.h" +#include +#include +#include + + +// #define _PRINT_TRANS +#define NUM_TRANSACTION 1024 +namespace dramsim3 { + +Address Address_IO::mask_addr; + +void CUSTOM_CPU::initialize() { + + //Initializte Counters + trans_cnt = 0; + hex_addr = 0; + rd_pass_cnt = 0; + rd_fail_cnt = 0; + wr_cnt = 0; + rd_cnt = 0; + wr_resp_cnt = 0; + rd_resp_cnt = 0; + ndp_pass_cnt = 0; + ndp_fail_cnt = 0; + + // Mask Addrses of Address IO must be set to some mask value + Address_IO tmp; + tmp.mask_addr = Address(config_->ch_mask, config_->ra_mask, config_->bg_mask, + config_->ba_mask, config_->ro_mask, config_->co_mask); + + // address_table[0].print_table(); +} + +void CUSTOM_CPU::ClockTick() { + memory_system_.ClockTick(); + + if(get_next_) trans = genTransaction(); + if(trans.is_valid) { + get_next_ = memory_system_.WillAcceptTransaction(trans.addr, trans.is_write, + trans.is_MRS); + if(get_next_) { + if(trans.is_write) wr_cnt++; + else rd_cnt++; + + #ifdef _PRINT_TRANS + if(trans.is_write) { + std::cout<<"["< MC [WR] ["< MC [RD] ["< rd_data = memory_system_.GetRespData(addr); + assert(rd_data.size()!=0); + #ifdef _PRINT_TRANS + std::cout<<"["< Host [RD] ["< Host [RD] ["< Host [WR] ["< Preloading Data"< Preloading Inst."< Kernel Exec"< Read Result"< payload; + payload.resize(8); + for(auto &a : payload) a = ((u_int64_t)(std::rand()) << 32) | (u_int64_t)(std::rand()); + // StoreWRTrans(hex_addr,payload); + return Transaction(hex_addr,true,false,payload); + } + else { + return Transaction(hex_addr,true,false); + } + } + else { + if(access_history.size()!=0) { + hex_addr = access_history[gen()%access_history.size()]; + return Transaction(hex_addr,false,false); + } + else return Transaction(); + } + + } + } + else { + if(trans_cnt < NUM_TRANSACTION) { + uint64_t addr = hex_addr; + trans_cnt++; + hex_addr++; + if(hex_addr==NUM_TRANSACTION) hex_addr = 0; + if(use_data) { + std::vector payload; + payload.resize(8); + for(auto &a : payload) a = addr; + // StoreWRTrans(addr,payload); + return Transaction(addr,true,false,payload); + } + else { + return Transaction(addr,true,false); + } + } + else if(trans_cnt < (NUM_TRANSACTION*2)) { + uint64_t addr = hex_addr; + hex_addr++; + trans_cnt++; + return Transaction(addr,false,false); + } + } + + return Transaction(); +} + +bool CUSTOM_CPU::NoTransInSim() { + return (rd_resp_cnt == rd_cnt) && (wr_resp_cnt == wr_cnt); +} + +void CUSTOM_CPU::StoreWRTrans(uint64_t hex_addr, std::vector &payload) { + //To check the consistency of the read response data, Store WR Data with Hex Address + if(wr_req.find(hex_addr) != wr_req.end()) { + assert(wr_req[hex_addr].size() == payload.size()); + for(uint32_t i=0;i new_payload; + for(uint32_t i=0;i &payload) { + if(wr_req.find(hex_addr) != wr_req.end()) { + assert(wr_req[hex_addr].size() == payload.size()); + bool error = false; + for(uint32_t i=0;i wr_DQMapping(std::vector &payload, uint64_t rank_address){ + + uint8_t DataRemapping; + uint64_t ResultData; + uint8_t Burst_num = 0; + uint8_t OneByte = 0; //64bit -> 8bit + std::vector MergedData; + MergedData.resize(8); + for(auto &a : payload){ + ResultData = 0; + for(int DB_num = 0; DB_num < 8; DB_num++){ + DataRemapping = 0; + OneByte = (a >> (8*DB_num)) & 0xFF; + + for(int i = 0; i <8; i++){ + DataRemapping |= (((OneByte >> wr_dq_map_per_db[rank_address%2][DB_num][7-i]) & 0x01) << i); //DB_num + } + ResultData |= static_cast(DataRemapping) << (8*DB_num); + } + MergedData[Burst_num] = ResultData; + Burst_num++; + } + return MergedData; +} + +std::vector rd_DQMapping(std::vector &payload, uint64_t rank_address){ + uint8_t DataRemapping; + uint8_t Burst_num = 0; + uint8_t OneByte = 0; //64bit -> 8bit + std::vector MergedData; + MergedData.resize(8); + for(auto &a : payload){ + for(int DB_num = 0; DB_num < 8; DB_num++){ + DataRemapping = 0; + OneByte = (a >> (8*DB_num)) & 0xFF; //OneByte = 8bit + for(int i = 0; i <8; i++){ + DataRemapping |= (((OneByte >> rd_dq_map_per_db[rank_address%2][DB_num][7-i]) & 0x01) << i); + } + MergedData[DB_num] |= (static_cast(DataRemapping) << (8*Burst_num)); + } + Burst_num++; + } + return MergedData; +} + +void CUSTOM_CPU::genRefData(const std::string& kernal_type) { + std::cout<<"Generating Reference Data "< real_dist(0.0f, 100.0f); + if(kernal_type == "EWA") { + std::cout<<" Kernel Type: Element-Wise Add"< ndp_inst_list; + if(kernal_type == "EWA") { + for(uint32_t i=0;i<32;i++) + ndp_inst_list.push_back(gen()); + /* + TBD + */ + } + else if(kernal_type == "EWM") { + for(uint32_t i=0;i<16;i++) + ndp_inst_list.push_back(gen()); + /* + TBD + */ + } + else { + std::cerr<<"NOT SUPPORTED KERNEL TYPE"<dbs_per_dimm == 8); + + Address_IO addr_io = Address_IO(ROCO,address_table[0]["ADDR_NDP_INST_MEM"]); + for(uint32_t i=0;i new_payload; + data_per_db = (static_cast(ndp_inst_list[i+1]) << 32) | ndp_inst_list[i]; + for(int j=0;jdbs_per_dimm;j++) new_payload.push_back(data_per_db); + ndp_insts.push_back(Transaction(config_->MergedAddress(addr_io.addr),true,false,new_payload)); + addr_io++; + } + + // Dummpy Write + Address_IO dummpy_address = Address_IO(BKCO,Address(0,0,0,0,0x200,0)); + for(uint32_t i=0;i<1024;i++) { + std::vector new_payload; + for(int j=0;jdbs_per_dimm;j++) new_payload.push_back(0); + ndp_insts.push_back(Transaction(config_->MergedAddress(dummpy_address.addr),true,false,new_payload)); + dummpy_address++; + } + +} + +void CUSTOM_CPU::genNDPData(const std::string& kernal_type) { + std::cout<<"Generating NDP Data and Memory Transactions "< new_payload; + for(int j=0;jdbs_per_dimm;j++) new_payload.push_back(0); + ndp_config.push_back(Transaction(config_->MergedAddress(address_table[0]["ADDR_REGION_START_NDP_DRAM"]),true,false,new_payload)); + ndp_config.push_back(Transaction(config_->MergedAddress(address_table[0]["ADDR_REGION_START_NDP"]),true,false,new_payload)); + ndp_config.push_back(Transaction(config_->MergedAddress(address_table[0]["ADDR_NDP_CONFIG"]),true,false,new_payload)); + ndp_config.push_back(Transaction(config_->MergedAddress(address_table[0]["ADDR_NDP_DB_CONFIG"]),true,false,new_payload)); + ndp_config.push_back(Transaction(config_->MergedAddress(address_table[0]["ADDR_NDP_RCD_CONFIG"]),true,false,new_payload)); + ndp_config.push_back(Transaction(config_->MergedAddress(address_table[0]["ADDR_NDP_INST_MEM"]),true,false,new_payload)); + ndp_config.push_back(Transaction(config_->MergedAddress(address_table[0]["ADDR_NDP_DATA_MEM"]),true,false,new_payload)); + + if(kernal_type == "EWA" || kernal_type == "EWM") { + // Some Configuration depending on kernel type + } + else { + std::cerr<<"NOT SUPPORTED KERNEL TYPE"<MergedAddress(vec_x_io.addr),false,false)); + vec_x_io++; + } + for(uint32_t i=0;iMergedAddress(vec_y_io.addr),false,false)); + vec_y_io++; + } + + } + else { + std::cerr<<"NOT SUPPORTED KERNEL TYPE"<MergedAddress(vec_z_io.addr),false,false)); + vec_z_io++; + } + + } + else { + std::cerr<<"NOT SUPPORTED KERNEL TYPE"< f_resp_data = convertUint64ToFloat(resp_data); + for(uint64_t i=0;i(float_value); +} + +float CUSTOM_CPU::Uint32ToFloat(uint32_t uint_value) { + float converted_value; + memcpy(&converted_value, &uint_value, sizeof(converted_value)); + return converted_value; + // return reinterpret_cast(uint_value); +} + +std::vector CUSTOM_CPU::convertFloatToUint64(std::vector &f_payload) { + std::vector payload; + for(uint32_t i=0;i(FloattoUint32(f_payload[i])); + if(i%2==0) payload.push_back(value_64b); + else payload[i/2] = payload[i/2] | (value_64b << 32); + } + return payload; +} + +std::vector CUSTOM_CPU::convertUint64ToFloat(std::vector &payload) { + std::vector f_payload; + for(auto value : payload) { + float f0 = Uint32ToFloat(static_cast(value & 0xFFFFFFFF)); + float f1 = Uint32ToFloat(static_cast((value >> 32 )& 0xFFFFFFFF)); + f_payload.push_back(f0); + f_payload.push_back(f1); + } + return f_payload; +} + +void CUSTOM_CPU::NDPData_FloatVecToTrans(Address_IO addr_io,std::vector f_vec) { + std::vector payload_vec = convertFloatToUint64(f_vec); + + while(payload_vec.size()%8 != 0) { + payload_vec.push_back(0); + } + + assert(payload_vec.size()%8 == 0); + + for(uint32_t i=0;i payload; + for(uint32_t j=0;j<8;j++) payload.push_back(payload_vec[i*8+j]); + ndp_data.push_back(Transaction(config_->MergedAddress(addr_io.addr),true,false,payload)); + addr_io++; + } +} + +} // namespace dramsim3 diff --git a/src/custom_cpu.h b/src/custom_cpu.h new file mode 100644 index 00000000..5e531f4d --- /dev/null +++ b/src/custom_cpu.h @@ -0,0 +1,144 @@ +#ifndef __CCPU_H +#define __CCPU_H + +#include +#include +#include +#include +#include "memory_system.h" +#include +#include "common.h" +#include "configuration.h" +#include +#include "ndp_address_table.h" + +namespace dramsim3 { + +class CUSTOM_CPU { + public: + CUSTOM_CPU(const std::string& config_file, const std::string& output_dir, + const std::string& gen_type) + : memory_system_( + config_file, output_dir, + std::bind(&CUSTOM_CPU::ReadCallBack, this, std::placeholders::_1), + std::bind(&CUSTOM_CPU::WriteCallBack, this, std::placeholders::_1)), + clk_(0), gen_type_(gen_type), address_table(memory_system_.GetNumChannel()) { + + use_data = memory_system_.isLRDIMM(); + config_ = memory_system_.GetConfig(); + + std::cout<<"================================"<dimms<ranks<< + " ( "<ranks_per_dimm<<" ranks per DIMM)"<channel_size/1024<<" GB"< &payload); // Store Write Transaction to Compare Read Data + bool CheckRD(uint64_t hex_addr, std::vector &payload); // Check RD Response Data + void printResult(); + + //DQ mapping function + std::vector wr_DQMapping(std::vector &payload, uint64_t rank_address); + std::vector rd_DQMapping(std::vector &payload, uint64_t rank_address); + + // NDP-related Function + void genRefData(const std::string& kernal_type); // Generate Refererece Data for Tatget kernel Type + void genNDPInst(const std::string& kernal_type); // Generate NDP Instruction for Tatget kernel Type + void genNDPData(const std::string& kernal_type); // Generate NDP Data for Tatget kernel Type + void genNDPConfig(const std::string& kernal_type); // Generate NDP Configuration Request + void genNDPExec(const std::string& kernal_type); // Generate NDP Execution Memory Request + void genNDPReadResult(const std::string& kernal_type); // Generate NDP Result Read Request + void checkNDPResult(const std::string& kernal_type); // Check NDP Result with Reference Data + bool simDone(); // Check Simulation for NDP is Done + + // convert Data format (FP32 -> UINT64) + uint32_t FloattoUint32(float float_value); + float Uint32ToFloat(uint32_t uint_value); + std::vector convertFloatToUint64(std::vector &f_payload); + std::vector convertUint64ToFloat(std::vector &payload); + + // Data Convert (only Using NDP Data Phase) + void NDPData_FloatVecToTrans(Address_IO addr_io,std::vector f_vec); + + private: + //Store Access History + std::vector access_history; + + // To check Read Response Data, Store WR Data + std::unordered_map> wr_req; + + // Transation Vector for Write NDP Instruction and Data + std::vector ndp_config; + std::vector ndp_insts; + std::vector ndp_data; + std::vector ndp_exec; + std::vector ndp_read_result; + + // Reference Data Format (support BLAS-1 function) z=axpy + float scalar_alpha; + std::vector vector_x,vector_y,vector_z; + + // NDP RD Result + std::vector resp_data; + + // Simulation State + bool run_state_ndp_config = false; + bool run_state_ndp_preloading_data = false; + bool run_state_ndp_preloading_inst = false; + bool run_state_ndp_kernel_exec = false; + bool run_state_ndp_read_result = false; + bool sim_done = false; + + // DRAMSim3 Configuration Object + Config *config_; + + protected: + MemorySystem memory_system_; + uint64_t clk_; + std::string gen_type_; + All_Ch_Addrss_Table address_table; + + + std::mt19937_64 gen; + bool get_next_ = true; + bool use_data; + Transaction trans; + + // Use Memory Transaction Generation (Stream and Random) + uint32_t trans_cnt; + uint32_t hex_addr; + + // Counters for Memory Transactions + uint64_t wr_cnt; + uint64_t rd_cnt; + uint64_t wr_resp_cnt; + uint64_t rd_resp_cnt; + uint64_t rd_pass_cnt; + uint64_t rd_fail_cnt; + uint64_t ndp_pass_cnt; + uint64_t ndp_fail_cnt; + + +}; + + +} // namespace dramsim3 +#endif diff --git a/src/dram_system.cc b/src/dram_system.cc index f181db02..1b37fa7b 100644 --- a/src/dram_system.cc +++ b/src/dram_system.cc @@ -20,10 +20,14 @@ BaseDRAMSystem::BaseDRAMSystem(Config &config, const std::string &output_dir, thermal_calc_(config_), #endif // THERMAL clk_(0) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor"<WillAcceptTransaction(hex_addr, is_write); + return ctrls_[channel]->WillAcceptTransaction(hex_addr, is_write); +} + +bool JedecDRAMSystem::WillAcceptTransaction(uint64_t hex_addr, + bool is_write, bool is_MRS) const { + int channel = GetChannel(hex_addr); + return ctrls_[channel]->WillAcceptTransaction(hex_addr, is_write, is_MRS); +} + +bool JedecDRAMSystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) { +// Record trace - Record address trace for debugging or other purposes +#ifdef ADDR_TRACE + address_trace_ << std::hex << hex_addr << std::dec << " " + << (is_write ? "WRITE " : "READ ") << clk_ << std::endl; +#endif + + int channel = GetChannel(hex_addr); + bool ok = ctrls_[channel]->WillAcceptTransaction(hex_addr, is_write, is_MRS); + + assert(ok); + if (ok) { + // Transaction trans = Transaction(hex_addr, is_write); + Transaction trans = Transaction(hex_addr, is_write, is_MRS); + ctrls_[channel]->AddTransaction(trans); + } + last_req_clk_ = clk_; + return ok; } -bool JedecDRAMSystem::AddTransaction(uint64_t hex_addr, bool is_write) { +bool JedecDRAMSystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) { // Record trace - Record address trace for debugging or other purposes -#ifdef ADDRESS_TRACE - address_trace_ << left << setw(18) << clk_ << " " << setw(6) << std::hex - << (is_write ? "WRITE " : "READ ") << hex_addr << std::dec - << std::endl; +#ifdef ADDR_TRACE + address_trace_ << std::hex << hex_addr << std::dec << " " + << (is_write ? "WRITE " : "READ ") << clk_ << std::endl; #endif int channel = GetChannel(hex_addr); - bool ok = ctrls_[channel]->WillAcceptTransaction(hex_addr, is_write); + bool ok = ctrls_[channel]->WillAcceptTransaction(hex_addr, is_write, is_MRS); assert(ok); if (ok) { - Transaction trans = Transaction(hex_addr, is_write); + Transaction trans = Transaction(hex_addr, is_write, is_MRS, payload); ctrls_[channel]->AddTransaction(trans); } last_req_clk_ = clk_; @@ -155,11 +178,10 @@ bool JedecDRAMSystem::AddTransaction(uint64_t hex_addr, bool is_write) { } void JedecDRAMSystem::ClockTick() { - uint64_t look_ahead_cycles = clk_ + config_.mega_tick / 2; for (size_t i = 0; i < ctrls_.size(); i++) { // look ahead and return earlier while (true) { - auto pair = ctrls_[i]->ReturnDoneTrans(look_ahead_cycles); + auto pair = ctrls_[i]->ReturnDoneTrans(clk_); if (pair.second == 1) { write_callback_(pair.first); } else if (pair.second == 0) { @@ -169,15 +191,10 @@ void JedecDRAMSystem::ClockTick() { } } } -#ifdef _OPENMP -#pragma omp parallel for schedule(static) -#endif // _OPENMP for (size_t i = 0; i < ctrls_.size(); i++) { - for (int j = 0; j < config_.mega_tick; j++) { - ctrls_[i]->ClockTick(); - } + ctrls_[i]->ClockTick(); } - clk_ += config_.mega_tick; + clk_++; if (clk_ % config_.epoch_period == 0) { PrintEpochStats(); @@ -185,6 +202,11 @@ void JedecDRAMSystem::ClockTick() { return; } +std::vector JedecDRAMSystem::GetRespData(uint64_t hex_addr) { + int channel = GetChannel(hex_addr); + return ctrls_[channel]->GetRespData(); +} + IdealDRAMSystem::IdealDRAMSystem(Config &config, const std::string &output_dir, std::function read_callback, std::function write_callback) @@ -193,8 +215,17 @@ IdealDRAMSystem::IdealDRAMSystem(Config &config, const std::string &output_dir, IdealDRAMSystem::~IdealDRAMSystem() {} -bool IdealDRAMSystem::AddTransaction(uint64_t hex_addr, bool is_write) { - auto trans = Transaction(hex_addr, is_write); +bool IdealDRAMSystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) { + // auto trans = Transaction(hex_addr, is_write); + auto trans = Transaction(hex_addr, is_write, is_MRS); + trans.added_cycle = clk_; + infinite_buffer_q_.push_back(trans); + return true; +} + +bool IdealDRAMSystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) { + // auto trans = Transaction(hex_addr, is_write); + auto trans = Transaction(hex_addr, is_write, is_MRS, payload); trans.added_cycle = clk_; infinite_buffer_q_.push_back(trans); return true; @@ -220,4 +251,11 @@ void IdealDRAMSystem::ClockTick() { return; } +std::vector IdealDRAMSystem::GetRespData(uint64_t hex_addr) { + // int channel = GetChannel(hex_addr); + //TBD + std::vector payload; + return payload; +} + } // namespace dramsim3 diff --git a/src/dram_system.h b/src/dram_system.h index 40a64260..486ab3f5 100644 --- a/src/dram_system.h +++ b/src/dram_system.h @@ -10,10 +10,6 @@ #include "controller.h" #include "timing.h" -#ifdef _OPENMP -#include "omp.h" -#endif // _OPENMP - #ifdef THERMAL #include "thermal.h" #endif // THERMAL @@ -34,9 +30,13 @@ class BaseDRAMSystem { virtual bool WillAcceptTransaction(uint64_t hex_addr, bool is_write) const = 0; - virtual bool AddTransaction(uint64_t hex_addr, bool is_write) = 0; + virtual bool WillAcceptTransaction(uint64_t hex_addr, + bool is_write, bool is_MRS) const = 0; + virtual bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) = 0; + virtual bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) = 0; virtual void ClockTick() = 0; int GetChannel(uint64_t hex_addr) const; + virtual std::vector GetRespData(uint64_t hex_addr) = 0; std::function read_callback_, write_callback_; static int total_channels_; @@ -56,9 +56,9 @@ class BaseDRAMSystem { uint64_t clk_; std::vector ctrls_; -#ifdef ADDRESS_TRACE +#ifdef ADDR_TRACE std::ofstream address_trace_; -#endif // ADDRESS_TRACE +#endif // ADDR_TRACE }; // hmmm not sure this is the best naming... @@ -69,8 +69,11 @@ class JedecDRAMSystem : public BaseDRAMSystem { std::function write_callback); ~JedecDRAMSystem(); bool WillAcceptTransaction(uint64_t hex_addr, bool is_write) const override; - bool AddTransaction(uint64_t hex_addr, bool is_write) override; + bool WillAcceptTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) const override; + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) override; + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) override; void ClockTick() override; + std::vector GetRespData(uint64_t hex_addr) override; }; // Model a memorysystem with an infinite bandwidth and a fixed latency (possibly @@ -86,8 +89,15 @@ class IdealDRAMSystem : public BaseDRAMSystem { bool is_write) const override { return true; }; - bool AddTransaction(uint64_t hex_addr, bool is_write) override; + bool WillAcceptTransaction(uint64_t hex_addr, + bool is_write, bool is_MRS) const override { + return true; + }; + + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) override; + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) override; void ClockTick() override; + std::vector GetRespData(uint64_t hex_addr) override; private: int latency_; diff --git a/src/half.hpp b/src/half.hpp new file mode 100644 index 00000000..f4d86146 --- /dev/null +++ b/src/half.hpp @@ -0,0 +1,4601 @@ +// half - IEEE 754-based half-precision floating-point library. +// +// Copyright (c) 2012-2021 Christian Rau +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, +// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// Version 2.2.0 + +/// \file +/// Main header file for half-precision functionality. + +#ifndef HALF_HALF_HPP +#define HALF_HALF_HPP + +#define HALF_GCC_VERSION (__GNUC__*100+__GNUC_MINOR__) + +#if defined(__INTEL_COMPILER) + #define HALF_ICC_VERSION __INTEL_COMPILER +#elif defined(__ICC) + #define HALF_ICC_VERSION __ICC +#elif defined(__ICL) + #define HALF_ICC_VERSION __ICL +#else + #define HALF_ICC_VERSION 0 +#endif + +// check C++11 language features +#if defined(__clang__) // clang + #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if __has_feature(cxx_thread_local) && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif +#elif HALF_ICC_VERSION && defined(__INTEL_CXX11_MODE__) // Intel C++ + #if HALF_ICC_VERSION >= 1500 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if HALF_ICC_VERSION >= 1500 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if HALF_ICC_VERSION >= 1400 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if HALF_ICC_VERSION >= 1400 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if HALF_ICC_VERSION >= 1110 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if HALF_ICC_VERSION >= 1110 && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif +#elif defined(__GNUC__) // gcc + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L + #if HALF_GCC_VERSION >= 408 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if HALF_GCC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if HALF_GCC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if HALF_GCC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #endif + #define HALF_TWOS_COMPLEMENT_INT 1 +#elif defined(_MSC_VER) // Visual C++ + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #define HALF_TWOS_COMPLEMENT_INT 1 + #define HALF_POP_WARNINGS 1 + #pragma warning(push) + #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned +#endif + +// check C++11 library features +#include +#if defined(_LIBCPP_VERSION) // libc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #ifndef HALF_ENABLE_CPP11_CSTDINT + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #ifndef HALF_ENABLE_CPP11_CMATH + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #ifndef HALF_ENABLE_CPP11_HASH + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #ifndef HALF_ENABLE_CPP11_CFENV + #define HALF_ENABLE_CPP11_CFENV 1 + #endif + #endif +#elif defined(__GLIBCXX__) // libstdc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifdef __clang__ + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CFENV) + #define HALF_ENABLE_CPP11_CFENV 1 + #endif + #else + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CFENV) + #define HALF_ENABLE_CPP11_CFENV 1 + #endif + #endif + #endif +#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++ + #if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #if _CPPLIB_VER >= 610 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if _CPPLIB_VER >= 610 && !defined(HALF_ENABLE_CPP11_CFENV) + #define HALF_ENABLE_CPP11_CFENV 1 + #endif +#endif +#undef HALF_GCC_VERSION +#undef HALF_ICC_VERSION + +// any error throwing C++ exceptions? +#if defined(HALF_ERRHANDLING_THROW_INVALID) || defined(HALF_ERRHANDLING_THROW_DIVBYZERO) || defined(HALF_ERRHANDLING_THROW_OVERFLOW) || defined(HALF_ERRHANDLING_THROW_UNDERFLOW) || defined(HALF_ERRHANDLING_THROW_INEXACT) +#define HALF_ERRHANDLING_THROWS 1 +#endif + +// any error handling enabled? +#define HALF_ERRHANDLING (HALF_ERRHANDLING_FLAGS||HALF_ERRHANDLING_ERRNO||HALF_ERRHANDLING_FENV||HALF_ERRHANDLING_THROWS) + +#if HALF_ERRHANDLING + #define HALF_UNUSED_NOERR(name) name +#else + #define HALF_UNUSED_NOERR(name) +#endif + +// support constexpr +#if HALF_ENABLE_CPP11_CONSTEXPR + #define HALF_CONSTEXPR constexpr + #define HALF_CONSTEXPR_CONST constexpr + #if HALF_ERRHANDLING + #define HALF_CONSTEXPR_NOERR + #else + #define HALF_CONSTEXPR_NOERR constexpr + #endif +#else + #define HALF_CONSTEXPR + #define HALF_CONSTEXPR_CONST const + #define HALF_CONSTEXPR_NOERR +#endif + +// support noexcept +#if HALF_ENABLE_CPP11_NOEXCEPT + #define HALF_NOEXCEPT noexcept + #define HALF_NOTHROW noexcept +#else + #define HALF_NOEXCEPT + #define HALF_NOTHROW throw() +#endif + +// support thread storage +#if HALF_ENABLE_CPP11_THREAD_LOCAL + #define HALF_THREAD_LOCAL thread_local +#else + #define HALF_THREAD_LOCAL static +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HALF_ENABLE_CPP11_TYPE_TRAITS + #include +#endif +#if HALF_ENABLE_CPP11_CSTDINT + #include +#endif +#if HALF_ERRHANDLING_ERRNO + #include +#endif +#if HALF_ENABLE_CPP11_CFENV + #include +#endif +#if HALF_ENABLE_CPP11_HASH + #include +#endif + + +#ifndef HALF_ENABLE_F16C_INTRINSICS + /// Enable F16C intruction set intrinsics. + /// Defining this to 1 enables the use of [F16C compiler intrinsics](https://en.wikipedia.org/wiki/F16C) for converting between + /// half-precision and single-precision values which may result in improved performance. This will not perform additional checks + /// for support of the F16C instruction set, so an appropriate target platform is required when enabling this feature. + /// + /// Unless predefined it will be enabled automatically when the `__F16C__` symbol is defined, which some compilers do on supporting platforms. + #define HALF_ENABLE_F16C_INTRINSICS __F16C__ +#endif +#if HALF_ENABLE_F16C_INTRINSICS + #include +#endif + +#ifdef HALF_DOXYGEN_ONLY +/// Type for internal floating-point computations. +/// This can be predefined to a built-in floating-point type (`float`, `double` or `long double`) to override the internal +/// half-precision implementation to use this type for computing arithmetic operations and mathematical function (if available). +/// This can result in improved performance for arithmetic operators and mathematical functions but might cause results to +/// deviate from the specified half-precision rounding mode and inhibits proper detection of half-precision exceptions. +#define HALF_ARITHMETIC_TYPE (undefined) + +/// Enable internal exception flags. +/// Defining this to 1 causes operations on half-precision values to raise internal floating-point exception flags according to +/// the IEEE 754 standard. These can then be cleared and checked with clearexcept(), testexcept(). +#define HALF_ERRHANDLING_FLAGS 0 + +/// Enable exception propagation to `errno`. +/// Defining this to 1 causes operations on half-precision values to propagate floating-point exceptions to +/// [errno](https://en.cppreference.com/w/cpp/error/errno) from ``. Specifically this will propagate domain errors as +/// [EDOM](https://en.cppreference.com/w/cpp/error/errno_macros) and pole, overflow and underflow errors as +/// [ERANGE](https://en.cppreference.com/w/cpp/error/errno_macros). Inexact errors won't be propagated. +#define HALF_ERRHANDLING_ERRNO 0 + +/// Enable exception propagation to built-in floating-point platform. +/// Defining this to 1 causes operations on half-precision values to propagate floating-point exceptions to the built-in +/// single- and double-precision implementation's exception flags using the +/// [C++11 floating-point environment control](https://en.cppreference.com/w/cpp/numeric/fenv) from ``. However, this +/// does not work in reverse and single- or double-precision exceptions will not raise the corresponding half-precision +/// exception flags, nor will explicitly clearing flags clear the corresponding built-in flags. +#define HALF_ERRHANDLING_FENV 0 + +/// Throw C++ exception on domain errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::domain_error](https://en.cppreference.com/w/cpp/error/domain_error) with the specified message on domain errors. +#define HALF_ERRHANDLING_THROW_INVALID (undefined) + +/// Throw C++ exception on pole errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::domain_error](https://en.cppreference.com/w/cpp/error/domain_error) with the specified message on pole errors. +#define HALF_ERRHANDLING_THROW_DIVBYZERO (undefined) + +/// Throw C++ exception on overflow errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::overflow_error](https://en.cppreference.com/w/cpp/error/overflow_error) with the specified message on overflows. +#define HALF_ERRHANDLING_THROW_OVERFLOW (undefined) + +/// Throw C++ exception on underflow errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::underflow_error](https://en.cppreference.com/w/cpp/error/underflow_error) with the specified message on underflows. +#define HALF_ERRHANDLING_THROW_UNDERFLOW (undefined) + +/// Throw C++ exception on rounding errors. +/// Defining this to 1 causes operations on half-precision values to throw a +/// [std::range_error](https://en.cppreference.com/w/cpp/error/range_error) with the specified message on general rounding errors. +#define HALF_ERRHANDLING_THROW_INEXACT (undefined) +#endif + +#ifndef HALF_ERRHANDLING_OVERFLOW_TO_INEXACT +/// Raise INEXACT exception on overflow. +/// Defining this to 1 (default) causes overflow errors to automatically raise inexact exceptions in addition. +/// These will be raised after any possible handling of the underflow exception. +#define HALF_ERRHANDLING_OVERFLOW_TO_INEXACT 1 +#endif + +#ifndef HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT +/// Raise INEXACT exception on underflow. +/// Defining this to 1 (default) causes underflow errors to automatically raise inexact exceptions in addition. +/// These will be raised after any possible handling of the underflow exception. +/// +/// **Note:** This will actually cause underflow (and the accompanying inexact) exceptions to be raised *only* when the result +/// is inexact, while if disabled bare underflow errors will be raised for *any* (possibly exact) subnormal result. +#define HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT 1 +#endif + +/// Default rounding mode. +/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and more precise types +/// (unless using half_cast() and specifying the rounding mode directly) as well as in arithmetic operations and mathematical +/// functions. It can be redefined (before including half.hpp) to one of the standard rounding modes using their respective +/// constants or the equivalent values of +/// [std::float_round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/float_round_style): +/// +/// `std::float_round_style` | value | rounding +/// ---------------------------------|-------|------------------------- +/// `std::round_indeterminate` | -1 | fastest +/// `std::round_toward_zero` | 0 | toward zero +/// `std::round_to_nearest` | 1 | to nearest (default) +/// `std::round_toward_infinity` | 2 | toward positive infinity +/// `std::round_toward_neg_infinity` | 3 | toward negative infinity +/// +/// By default this is set to `1` (`std::round_to_nearest`), which rounds results to the nearest representable value. It can even +/// be set to [std::numeric_limits::round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/round_style) to synchronize +/// the rounding mode with that of the built-in single-precision implementation (which is likely `std::round_to_nearest`, though). +#ifndef HALF_ROUND_STYLE + #define HALF_ROUND_STYLE 1 // = std::round_to_nearest +#endif + +/// Value signaling overflow. +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an +/// operation, in particular it just evaluates to positive infinity. +/// +/// **See also:** Documentation for [HUGE_VAL](https://en.cppreference.com/w/cpp/numeric/math/HUGE_VAL) +#define HUGE_VALH std::numeric_limits::infinity() + +/// Fast half-precision fma function. +/// This symbol is defined if the fma() function generally executes as fast as, or faster than, a separate +/// half-precision multiplication followed by an addition, which is always the case. +/// +/// **See also:** Documentation for [FP_FAST_FMA](https://en.cppreference.com/w/cpp/numeric/math/fma) +#define FP_FAST_FMAH 1 + +/// Half rounding mode. +/// In correspondence with `FLT_ROUNDS` from `` this symbol expands to the rounding mode used for +/// half-precision operations. It is an alias for [HALF_ROUND_STYLE](\ref HALF_ROUND_STYLE). +/// +/// **See also:** Documentation for [FLT_ROUNDS](https://en.cppreference.com/w/cpp/types/climits/FLT_ROUNDS) +#define HLF_ROUNDS HALF_ROUND_STYLE + +#ifndef FP_ILOGB0 + #define FP_ILOGB0 INT_MIN +#endif +#ifndef FP_ILOGBNAN + #define FP_ILOGBNAN INT_MAX +#endif +#ifndef FP_SUBNORMAL + #define FP_SUBNORMAL 0 +#endif +#ifndef FP_ZERO + #define FP_ZERO 1 +#endif +#ifndef FP_NAN + #define FP_NAN 2 +#endif +#ifndef FP_INFINITE + #define FP_INFINITE 3 +#endif +#ifndef FP_NORMAL + #define FP_NORMAL 4 +#endif + +#if !HALF_ENABLE_CPP11_CFENV && !defined(FE_ALL_EXCEPT) + #define FE_INVALID 0x10 + #define FE_DIVBYZERO 0x08 + #define FE_OVERFLOW 0x04 + #define FE_UNDERFLOW 0x02 + #define FE_INEXACT 0x01 + #define FE_ALL_EXCEPT (FE_INVALID|FE_DIVBYZERO|FE_OVERFLOW|FE_UNDERFLOW|FE_INEXACT) +#endif + + +/// Main namespace for half-precision functionality. +/// This namespace contains all the functionality provided by the library. +namespace half_float +{ + class half; + +#if HALF_ENABLE_CPP11_USER_LITERALS + /// Library-defined half-precision literals. + /// Import this namespace to enable half-precision floating-point literals: + /// ~~~~{.cpp} + /// using namespace half_float::literal; + /// half_float::half = 4.2_h; + /// ~~~~ + namespace literal + { + half operator "" _h(long double); + } +#endif + + /// \internal + /// \brief Implementation details. + namespace detail + { + #if HALF_ENABLE_CPP11_TYPE_TRAITS + /// Conditional type. + template struct conditional : std::conditional {}; + + /// Helper for tag dispatching. + template struct bool_type : std::integral_constant {}; + using std::true_type; + using std::false_type; + + /// Type traits for floating-point types. + template struct is_float : std::is_floating_point {}; + #else + /// Conditional type. + template struct conditional { typedef T type; }; + template struct conditional { typedef F type; }; + + /// Helper for tag dispatching. + template struct bool_type {}; + typedef bool_type true_type; + typedef bool_type false_type; + + /// Type traits for floating-point types. + template struct is_float : false_type {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + #endif + + /// Type traits for floating-point bits. + template struct bits { typedef unsigned char type; }; + template struct bits : bits {}; + template struct bits : bits {}; + template struct bits : bits {}; + + #if HALF_ENABLE_CPP11_CSTDINT + /// Unsigned integer of (at least) 16 bits width. + typedef std::uint_least16_t uint16; + + /// Fastest unsigned integer of (at least) 32 bits width. + typedef std::uint_fast32_t uint32; + + /// Fastest signed integer of (at least) 32 bits width. + typedef std::int_fast32_t int32; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits { typedef std::uint_least32_t type; }; + + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef std::uint_least64_t type; }; + #else + /// Unsigned integer of (at least) 16 bits width. + typedef unsigned short uint16; + + /// Fastest unsigned integer of (at least) 32 bits width. + typedef unsigned long uint32; + + /// Fastest unsigned integer of (at least) 32 bits width. + typedef long int32; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits : conditional::digits>=32,unsigned int,unsigned long> {}; + + #if HALF_ENABLE_CPP11_LONG_LONG + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits : conditional::digits>=64,unsigned long,unsigned long long> {}; + #else + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef unsigned long type; }; + #endif + #endif + + #ifdef HALF_ARITHMETIC_TYPE + /// Type to use for arithmetic computations and mathematic functions internally. + typedef HALF_ARITHMETIC_TYPE internal_t; + #endif + + /// Tag type for binary construction. + struct binary_t {}; + + /// Tag for binary construction. + HALF_CONSTEXPR_CONST binary_t binary = binary_t(); + + /// \name Implementation defined classification and arithmetic + /// \{ + + /// Check for infinity. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if infinity + /// \retval false else + template bool builtin_isinf(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isinf(arg); + #elif defined(_MSC_VER) + return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); + #else + return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); + #endif + } + + /// Check for NaN. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if not a number + /// \retval false else + template bool builtin_isnan(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isnan(arg); + #elif defined(_MSC_VER) + return ::_isnan(static_cast(arg)) != 0; + #else + return arg != arg; + #endif + } + + /// Check sign. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if signbit set + /// \retval false else + template bool builtin_signbit(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::signbit(arg); + #else + return arg < T() || (arg == T() && T(1)/arg < T()); + #endif + } + + /// Platform-independent sign mask. + /// \param arg integer value in two's complement + /// \retval -1 if \a arg negative + /// \retval 0 if \a arg positive + inline uint32 sign_mask(uint32 arg) + { + static const int N = std::numeric_limits::digits - 1; + #if HALF_TWOS_COMPLEMENT_INT + return static_cast(arg) >> N; + #else + return -((arg>>N)&1); + #endif + } + + /// Platform-independent arithmetic right shift. + /// \param arg integer value in two's complement + /// \param i shift amount (at most 31) + /// \return \a arg right shifted for \a i bits with possible sign extension + inline uint32 arithmetic_shift(uint32 arg, int i) + { + #if HALF_TWOS_COMPLEMENT_INT + return static_cast(arg) >> i; + #else + return static_cast(arg)/(static_cast(1)<>(std::numeric_limits::digits-1))&1); + #endif + } + + /// \} + /// \name Error handling + /// \{ + + /// Internal exception flags. + /// \return reference to global exception flags + inline int& errflags() { HALF_THREAD_LOCAL int flags = 0; return flags; } + + /// Raise floating-point exception. + /// \param flags exceptions to raise + /// \param cond condition to raise exceptions for + inline void raise(int HALF_UNUSED_NOERR(flags), bool HALF_UNUSED_NOERR(cond) = true) + { + #if HALF_ERRHANDLING + if(!cond) + return; + #if HALF_ERRHANDLING_FLAGS + errflags() |= flags; + #endif + #if HALF_ERRHANDLING_ERRNO + if(flags & FE_INVALID) + errno = EDOM; + else if(flags & (FE_DIVBYZERO|FE_OVERFLOW|FE_UNDERFLOW)) + errno = ERANGE; + #endif + #if HALF_ERRHANDLING_FENV && HALF_ENABLE_CPP11_CFENV + std::feraiseexcept(flags); + #endif + #ifdef HALF_ERRHANDLING_THROW_INVALID + if(flags & FE_INVALID) + throw std::domain_error(HALF_ERRHANDLING_THROW_INVALID); + #endif + #ifdef HALF_ERRHANDLING_THROW_DIVBYZERO + if(flags & FE_DIVBYZERO) + throw std::domain_error(HALF_ERRHANDLING_THROW_DIVBYZERO); + #endif + #ifdef HALF_ERRHANDLING_THROW_OVERFLOW + if(flags & FE_OVERFLOW) + throw std::overflow_error(HALF_ERRHANDLING_THROW_OVERFLOW); + #endif + #ifdef HALF_ERRHANDLING_THROW_UNDERFLOW + if(flags & FE_UNDERFLOW) + throw std::underflow_error(HALF_ERRHANDLING_THROW_UNDERFLOW); + #endif + #ifdef HALF_ERRHANDLING_THROW_INEXACT + if(flags & FE_INEXACT) + throw std::range_error(HALF_ERRHANDLING_THROW_INEXACT); + #endif + #if HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT + if((flags & FE_UNDERFLOW) && !(flags & FE_INEXACT)) + raise(FE_INEXACT); + #endif + #if HALF_ERRHANDLING_OVERFLOW_TO_INEXACT + if((flags & FE_OVERFLOW) && !(flags & FE_INEXACT)) + raise(FE_INEXACT); + #endif + #endif + } + + /// Check and signal for any NaN. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \retval true if either \a x or \a y is NaN + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool compsignal(unsigned int x, unsigned int y) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, (x&0x7FFF)>0x7C00 || (y&0x7FFF)>0x7C00); + #endif + return (x&0x7FFF) > 0x7C00 || (y&0x7FFF) > 0x7C00; + } + + /// Signal and silence signaling NaN. + /// \param nan half-precision NaN value + /// \return quiet NaN + /// \exception FE_INVALID if \a nan is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int nan) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, !(nan&0x200)); + #endif + return nan | 0x200; + } + + /// Signal and silence signaling NaNs. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \return quiet NaN + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int x, unsigned int y) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, ((x&0x7FFF)>0x7C00 && !(x&0x200)) || ((y&0x7FFF)>0x7C00 && !(y&0x200))); + #endif + return ((x&0x7FFF)>0x7C00) ? (x|0x200) : (y|0x200); + } + + /// Signal and silence signaling NaNs. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \param z third half-precision value to check + /// \return quiet NaN + /// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int x, unsigned int y, unsigned int z) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, ((x&0x7FFF)>0x7C00 && !(x&0x200)) || ((y&0x7FFF)>0x7C00 && !(y&0x200)) || ((z&0x7FFF)>0x7C00 && !(z&0x200))); + #endif + return ((x&0x7FFF)>0x7C00) ? (x|0x200) : ((y&0x7FFF)>0x7C00) ? (y|0x200) : (z|0x200); + } + + /// Select value or signaling NaN. + /// \param x preferred half-precision value + /// \param y ignored half-precision value except for signaling NaN + /// \return \a y if signaling NaN, \a x otherwise + /// \exception FE_INVALID if \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int select(unsigned int x, unsigned int HALF_UNUSED_NOERR(y)) + { + #if HALF_ERRHANDLING + return (((y&0x7FFF)>0x7C00) && !(y&0x200)) ? signal(y) : x; + #else + return x; + #endif + } + + /// Raise domain error and return NaN. + /// return quiet NaN + /// \exception FE_INVALID + inline HALF_CONSTEXPR_NOERR unsigned int invalid() + { + #if HALF_ERRHANDLING + raise(FE_INVALID); + #endif + return 0x7FFF; + } + + /// Raise pole error and return infinity. + /// \param sign half-precision value with sign bit only + /// \return half-precision infinity with sign of \a sign + /// \exception FE_DIVBYZERO + inline HALF_CONSTEXPR_NOERR unsigned int pole(unsigned int sign = 0) + { + #if HALF_ERRHANDLING + raise(FE_DIVBYZERO); + #endif + return sign | 0x7C00; + } + + /// Check value for underflow. + /// \param arg non-zero half-precision value to check + /// \return \a arg + /// \exception FE_UNDERFLOW if arg is subnormal + inline HALF_CONSTEXPR_NOERR unsigned int check_underflow(unsigned int arg) + { + #if HALF_ERRHANDLING && !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT + raise(FE_UNDERFLOW, !(arg&0x7C00)); + #endif + return arg; + } + + /// \} + /// \name Conversion and rounding + /// \{ + + /// Half-precision overflow. + /// \tparam R rounding mode to use + /// \param sign half-precision value with sign bit only + /// \return rounded overflowing half-precision value + /// \exception FE_OVERFLOW + template HALF_CONSTEXPR_NOERR unsigned int overflow(unsigned int sign = 0) + { + #if HALF_ERRHANDLING + raise(FE_OVERFLOW); + #endif + return (R==std::round_toward_infinity) ? (sign+0x7C00-(sign>>15)) : + (R==std::round_toward_neg_infinity) ? (sign+0x7BFF+(sign>>15)) : + (R==std::round_toward_zero) ? (sign|0x7BFF) : + (sign|0x7C00); + } + + /// Half-precision underflow. + /// \tparam R rounding mode to use + /// \param sign half-precision value with sign bit only + /// \return rounded underflowing half-precision value + /// \exception FE_UNDERFLOW + template HALF_CONSTEXPR_NOERR unsigned int underflow(unsigned int sign = 0) + { + #if HALF_ERRHANDLING + raise(FE_UNDERFLOW); + #endif + return (R==std::round_toward_infinity) ? (sign+1-(sign>>15)) : + (R==std::round_toward_neg_infinity) ? (sign+(sign>>15)) : + sign; + } + + /// Round half-precision number. + /// \tparam R rounding mode to use + /// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results + /// \param value finite half-precision number to round + /// \param g guard bit (most significant discarded bit) + /// \param s sticky bit (or of all but the most significant discarded bits) + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded or \a I is `true` + template HALF_CONSTEXPR_NOERR unsigned int rounded(unsigned int value, int g, int s) + { + #if HALF_ERRHANDLING + value += (R==std::round_to_nearest) ? (g&(s|value)) : + (R==std::round_toward_infinity) ? (~(value>>15)&(g|s)) : + (R==std::round_toward_neg_infinity) ? ((value>>15)&(g|s)) : 0; + if((value&0x7C00) == 0x7C00) + raise(FE_OVERFLOW); + else if(value & 0x7C00) + raise(FE_INEXACT, I || (g|s)!=0); + else + raise(FE_UNDERFLOW, !(HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT) || I || (g|s)!=0); + return value; + #else + return (R==std::round_to_nearest) ? (value+(g&(s|value))) : + (R==std::round_toward_infinity) ? (value+(~(value>>15)&(g|s))) : + (R==std::round_toward_neg_infinity) ? (value+((value>>15)&(g|s))) : + value; + #endif + } + + /// Round half-precision number to nearest integer value. + /// \tparam R rounding mode to use + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it + /// \param value half-precision value to round + /// \return half-precision bits for nearest integral value + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded and \a I is `true` + template unsigned int integral(unsigned int value) + { + unsigned int abs = value & 0x7FFF; + if(abs < 0x3C00) + { + raise(FE_INEXACT, I); + return ((R==std::round_to_nearest) ? (0x3C00&-static_cast(abs>=(0x3800+E))) : + (R==std::round_toward_infinity) ? (0x3C00&-(~(value>>15)&(abs!=0))) : + (R==std::round_toward_neg_infinity) ? (0x3C00&-static_cast(value>0x8000)) : + 0) | (value&0x8000); + } + if(abs >= 0x6400) + return (abs>0x7C00) ? signal(value) : value; + unsigned int exp = 25 - (abs>>10), mask = (1<>exp)&E)) : + (R==std::round_toward_infinity) ? (mask&((value>>15)-1)) : + (R==std::round_toward_neg_infinity) ? (mask&-(value>>15)) : + 0) + value) & ~mask; + } + + /// Convert fixed point to half-precision floating-point. + /// \tparam R rounding mode to use + /// \tparam F number of fractional bits in [11,31] + /// \tparam S `true` for signed, `false` for unsigned + /// \tparam N `true` for additional normalization step, `false` if already normalized to 1.F + /// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results + /// \param m mantissa in Q1.F fixed point format + /// \param exp biased exponent - 1 + /// \param sign half-precision value with sign bit only + /// \param s sticky bit (or of all but the most significant already discarded bits) + /// \return value converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded or \a I is `true` + template unsigned int fixed2half(uint32 m, int exp = 14, unsigned int sign = 0, int s = 0) + { + if(S) + { + uint32 msign = sign_mask(m); + m = (m^msign) - msign; + sign = msign & 0x8000; + } + if(N) + for(; m<(static_cast(1)<(sign+(m>>(F-10-exp)), (m>>(F-11-exp))&1, s|((m&((static_cast(1)<<(F-11-exp))-1))!=0)); + return rounded(sign+(exp<<10)+(m>>(F-10)), (m>>(F-11))&1, s|((m&((static_cast(1)<<(F-11))-1))!=0)); + } + + /// Convert IEEE single-precision to half-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \tparam R rounding mode to use + /// \param value single-precision value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(float value, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtsi128_si32(_mm_cvtps_ph(_mm_set_ss(value), + (R==std::round_to_nearest) ? _MM_FROUND_TO_NEAREST_INT : + (R==std::round_toward_zero) ? _MM_FROUND_TO_ZERO : + (R==std::round_toward_infinity) ? _MM_FROUND_TO_POS_INF : + (R==std::round_toward_neg_infinity) ? _MM_FROUND_TO_NEG_INF : + _MM_FROUND_CUR_DIRECTION)); + #else + bits::type fbits; + std::memcpy(&fbits, &value, sizeof(float)); + #if 1 + unsigned int sign = (fbits>>16) & 0x8000; + fbits &= 0x7FFFFFFF; + if(fbits >= 0x7F800000) + return sign | 0x7C00 | ((fbits>0x7F800000) ? (0x200|((fbits>>13)&0x3FF)) : 0); + if(fbits >= 0x47800000) + return overflow(sign); + if(fbits >= 0x38800000) + return rounded(sign|(((fbits>>23)-112)<<10)|((fbits>>13)&0x3FF), (fbits>>12)&1, (fbits&0xFFF)!=0); + if(fbits >= 0x33000000) + { + int i = 125 - (fbits>>23); + fbits = (fbits&0x7FFFFF) | 0x800000; + return rounded(sign|(fbits>>(i+1)), (fbits>>i)&1, (fbits&((static_cast(1)<(sign); + return sign; + #else + static const uint16 base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, + 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, + 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7C00, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, + 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, + 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFC00 }; + static const unsigned char shift_table[256] = { + 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; + int sexp = fbits >> 23, exp = sexp & 0xFF, i = shift_table[exp]; + fbits &= 0x7FFFFF; + uint32 m = (fbits|((exp!=0)<<23)) & -static_cast(exp!=0xFF); + return rounded(base_table[sexp]+(fbits>>i), (m>>(i-1))&1, (((static_cast(1)<<(i-1))-1)&m)!=0); + #endif + #endif + } + + /// Convert IEEE double-precision to half-precision. + /// \tparam R rounding mode to use + /// \param value double-precision value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(double value, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + if(R == std::round_indeterminate) + return _mm_cvtsi128_si32(_mm_cvtps_ph(_mm_cvtpd_ps(_mm_set_sd(value)), _MM_FROUND_CUR_DIRECTION)); + #endif + bits::type dbits; + std::memcpy(&dbits, &value, sizeof(double)); + uint32 hi = dbits >> 32, lo = dbits & 0xFFFFFFFF; + unsigned int sign = (hi>>16) & 0x8000; + hi &= 0x7FFFFFFF; + if(hi >= 0x7FF00000) + return sign | 0x7C00 | ((dbits&0xFFFFFFFFFFFFF) ? (0x200|((hi>>10)&0x3FF)) : 0); + if(hi >= 0x40F00000) + return overflow(sign); + if(hi >= 0x3F100000) + return rounded(sign|(((hi>>20)-1008)<<10)|((hi>>10)&0x3FF), (hi>>9)&1, ((hi&0x1FF)|lo)!=0); + if(hi >= 0x3E600000) + { + int i = 1018 - (hi>>20); + hi = (hi&0xFFFFF) | 0x100000; + return rounded(sign|(hi>>(i+1)), (hi>>i)&1, ((hi&((static_cast(1)<(sign); + return sign; + } + + /// Convert non-IEEE floating-point to half-precision. + /// \tparam R rounding mode to use + /// \tparam T source type (builtin floating-point type) + /// \param value floating-point value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(T value, ...) + { + unsigned int hbits = static_cast(builtin_signbit(value)) << 15; + if(value == T()) + return hbits; + if(builtin_isnan(value)) + return hbits | 0x7FFF; + if(builtin_isinf(value)) + return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if(exp > 16) + return overflow(hbits); + if(exp < -13) + value = std::ldexp(value, 25); + else + { + value = std::ldexp(value, 12-exp); + hbits |= ((exp+13)<<10); + } + T ival, frac = std::modf(value, &ival); + int m = std::abs(static_cast(ival)); + return rounded(hbits+(m>>1), m&1, frac!=T()); + } + + /// Convert floating-point to half-precision. + /// \tparam R rounding mode to use + /// \tparam T source type (builtin floating-point type) + /// \param value floating-point value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half(T value) + { + return float2half_impl(value, bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert integer to half-precision floating-point. + /// \tparam R rounding mode to use + /// \tparam T type to convert (builtin integer type) + /// \param value integral value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int int2half(T value) + { + unsigned int bits = static_cast(value<0) << 15; + if(!value) + return bits; + if(bits) + value = -value; + if(value > 0xFFFF) + return overflow(bits); + unsigned int m = static_cast(value), exp = 24; + for(; m<0x400; m<<=1,--exp) ; + for(; m>0x7FF; m>>=1,++exp) ; + bits |= (exp<<10) + m; + return (exp>24) ? rounded(bits, (value>>(exp-25))&1, (((1<<(exp-25))-1)&value)!=0) : bits; + } + + /// Convert half-precision to IEEE single-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \param value half-precision value to convert + /// \return single-precision value + inline float half2float_impl(unsigned int value, float, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(value))); + #else + #if 0 + bits::type fbits = static_cast::type>(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + fbits |= 0x38000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,fbits-=0x800000) ; + fbits += static_cast::type>(abs) << 13; + } + #else + static const bits::type mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, + 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, + 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, + 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, + 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, + 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, + 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, + 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, + 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, + 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, + 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, + 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, + 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, + 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, + 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, + 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, + 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, + 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, + 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, + 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, + 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, + 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, + 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, + 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, + 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, + 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, + 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, + 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, + 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, + 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, + 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, + 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, + 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, + 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, + 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, + 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, + 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, + 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, + 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, + 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, + 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, + 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, + 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, + 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, + 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, + 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, + 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, + 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, + 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, + 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, + 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, + 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, + 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, + 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, + 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, + 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, + 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, + 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, + 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, + 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, + 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, + 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, + 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, + 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, + 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, + 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, + 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, + 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, + 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, + 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, + 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, + 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, + 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, + 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, + 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, + 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, + 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, + 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, + 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, + 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, + 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, + 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, + 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, + 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, + 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, + 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, + 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, + 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, + 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, + 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, + 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, + 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, + 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, + 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, + 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, + 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, + 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, + 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, + 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, + 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, + 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, + 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, + 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, + 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, + 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, + 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, + 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, + 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, + 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, + 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, + 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, + 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, + 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, + 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, + 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, + 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, + 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, + 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, + 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, + 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; + static const bits::type exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, + 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, + 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; + bits::type fbits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; + #endif + float out; + std::memcpy(&out, &fbits, sizeof(float)); + return out; + #endif + } + + /// Convert half-precision to IEEE double-precision. + /// \param value half-precision value to convert + /// \return double-precision value + inline double half2float_impl(unsigned int value, double, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtsd_f64(_mm_cvtps_pd(_mm_cvtph_ps(_mm_cvtsi32_si128(value)))); + #else + uint32 hi = static_cast(value&0x8000) << 16; + unsigned int abs = value & 0x7FFF; + if(abs) + { + hi |= 0x3F000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,hi-=0x100000) ; + hi += static_cast(abs) << 10; + } + bits::type dbits = static_cast::type>(hi) << 32; + double out; + std::memcpy(&out, &dbits, sizeof(double)); + return out; + #endif + } + + /// Convert half-precision to non-IEEE floating-point. + /// \tparam T type to convert to (builtin integer type) + /// \param value half-precision value to convert + /// \return floating-point value + template T half2float_impl(unsigned int value, T, ...) + { + T out; + unsigned int abs = value & 0x7FFF; + if(abs > 0x7C00) + out = (std::numeric_limits::has_signaling_NaN && !(abs&0x200)) ? std::numeric_limits::signaling_NaN() : + std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); + else if(abs == 0x7C00) + out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); + else if(abs > 0x3FF) + out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); + else + out = std::ldexp(static_cast(abs), -24); + return (value&0x8000) ? -out : out; + } + + /// Convert half-precision to floating-point. + /// \tparam T type to convert to (builtin integer type) + /// \param value half-precision value to convert + /// \return floating-point value + template T half2float(unsigned int value) + { + return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert half-precision floating-point to integer. + /// \tparam R rounding mode to use + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value half-precision value to convert + /// \return rounded integer value + /// \exception FE_INVALID if value is not representable in type \a T + /// \exception FE_INEXACT if value had to be rounded and \a I is `true` + template T half2int(unsigned int value) + { + unsigned int abs = value & 0x7FFF; + if(abs >= 0x7C00) + { + raise(FE_INVALID); + return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); + } + if(abs < 0x3800) + { + raise(FE_INEXACT, I); + return (R==std::round_toward_infinity) ? T(~(value>>15)&(abs!=0)) : + (R==std::round_toward_neg_infinity) ? -T(value>0x8000) : + T(); + } + int exp = 25 - (abs>>10); + unsigned int m = (value&0x3FF) | 0x400; + int32 i = static_cast((exp<=0) ? (m<<-exp) : ((m+( + (R==std::round_to_nearest) ? ((1<<(exp-1))-(~(m>>exp)&E)) : + (R==std::round_toward_infinity) ? (((1<>15)-1)) : + (R==std::round_toward_neg_infinity) ? (((1<>15)) : 0))>>exp)); + if((!std::numeric_limits::is_signed && (value&0x8000)) || (std::numeric_limits::digits<16 && + ((value&0x8000) ? (-i::min()) : (i>std::numeric_limits::max())))) + raise(FE_INVALID); + else if(I && exp > 0 && (m&((1<((value&0x8000) ? -i : i); + } + + /// \} + /// \name Mathematics + /// \{ + + /// upper part of 64-bit multiplication. + /// \tparam R rounding mode to use + /// \param x first factor + /// \param y second factor + /// \return upper 32 bit of \a x * \a y + template uint32 mulhi(uint32 x, uint32 y) + { + uint32 xy = (x>>16) * (y&0xFFFF), yx = (x&0xFFFF) * (y>>16), c = (xy&0xFFFF) + (yx&0xFFFF) + (((x&0xFFFF)*(y&0xFFFF))>>16); + return (x>>16)*(y>>16) + (xy>>16) + (yx>>16) + (c>>16) + + ((R==std::round_to_nearest) ? ((c>>15)&1) : (R==std::round_toward_infinity) ? ((c&0xFFFF)!=0) : 0); + } + + /// 64-bit multiplication. + /// \param x first factor + /// \param y second factor + /// \return upper 32 bit of \a x * \a y rounded to nearest + inline uint32 multiply64(uint32 x, uint32 y) + { + #if HALF_ENABLE_CPP11_LONG_LONG + return static_cast((static_cast(x)*static_cast(y)+0x80000000)>>32); + #else + return mulhi(x, y); + #endif + } + + /// 64-bit division. + /// \param x upper 32 bit of dividend + /// \param y divisor + /// \param s variable to store sticky bit for rounding + /// \return (\a x << 32) / \a y + inline uint32 divide64(uint32 x, uint32 y, int &s) + { + #if HALF_ENABLE_CPP11_LONG_LONG + unsigned long long xx = static_cast(x) << 32; + return s = (xx%y!=0), static_cast(xx/y); + #else + y >>= 1; + uint32 rem = x, div = 0; + for(unsigned int i=0; i<32; ++i) + { + div <<= 1; + if(rem >= y) + { + rem -= y; + div |= 1; + } + rem <<= 1; + } + return s = rem > 1, div; + #endif + } + + /// Half precision positive modulus. + /// \tparam Q `true` to compute full quotient, `false` else + /// \tparam R `true` to compute signed remainder, `false` for positive remainder + /// \param x first operand as positive finite half-precision value + /// \param y second operand as positive finite half-precision value + /// \param quo adress to store quotient at, `nullptr` if \a Q `false` + /// \return modulus of \a x / \a y + template unsigned int mod(unsigned int x, unsigned int y, int *quo = NULL) + { + unsigned int q = 0; + if(x > y) + { + int absx = x, absy = y, expx = 0, expy = 0; + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + expx += absx >> 10; + expy += absy >> 10; + int mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + for(int d=expx-expy; d; --d) + { + if(!Q && mx == my) + return 0; + if(mx >= my) + { + mx -= my; + q += Q; + } + mx <<= 1; + q <<= static_cast(Q); + } + if(!Q && mx == my) + return 0; + if(mx >= my) + { + mx -= my; + ++q; + } + if(Q) + { + q &= (1<<(std::numeric_limits::digits-1)) - 1; + if(!mx) + return *quo = q, 0; + } + for(; mx<0x400; mx<<=1,--expy) ; + x = (expy>0) ? ((expy<<10)|(mx&0x3FF)) : (mx>>(1-expy)); + } + if(R) + { + unsigned int a, b; + if(y < 0x800) + { + a = (x<0x400) ? (x<<1) : (x+0x400); + b = y; + } + else + { + a = x; + b = y - 0x400; + } + if(a > b || (a == b && (q&1))) + { + int exp = (y>>10) + (y<=0x3FF), d = exp - (x>>10) - (x<=0x3FF); + int m = (((y&0x3FF)|((y>0x3FF)<<10))<<1) - (((x&0x3FF)|((x>0x3FF)<<10))<<(1-d)); + for(; m<0x800 && exp>1; m<<=1,--exp) ; + x = 0x8000 + ((exp-1)<<10) + (m>>1); + q += Q; + } + } + if(Q) + *quo = q; + return x; + } + + /// Fixed point square root. + /// \tparam F number of fractional bits + /// \param r radicand in Q1.F fixed point format + /// \param exp exponent + /// \return square root as Q1.F/2 + template uint32 sqrt(uint32 &r, int &exp) + { + int i = exp & 1; + r <<= i; + exp = (exp-i) / 2; + uint32 m = 0; + for(uint32 bit=static_cast(1)<>=2) + { + if(r < m+bit) + m >>= 1; + else + { + r -= m + bit; + m = (m>>1) + bit; + } + } + return m; + } + + /// Fixed point binary exponential. + /// This uses the BKM algorithm in E-mode. + /// \param m exponent in [0,1) as Q0.31 + /// \param n number of iterations (at most 32) + /// \return 2 ^ \a m as Q1.31 + inline uint32 exp2(uint32 m, unsigned int n = 32) + { + static const uint32 logs[] = { + 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, 0x016FE50B, + 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, 0x0002E2A3, 0x00017153, + 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, 0x000005C5, 0x000002E3, 0x00000171, + 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, 0x0000000C, 0x00000006, 0x00000003, 0x00000001 }; + if(!m) + return 0x80000000; + uint32 mx = 0x80000000, my = 0; + for(unsigned int i=1; i> i; + } + } + return mx; + } + + /// Fixed point binary logarithm. + /// This uses the BKM algorithm in L-mode. + /// \param m mantissa in [1,2) as Q1.30 + /// \param n number of iterations (at most 32) + /// \return log2(\a m) as Q0.31 + inline uint32 log2(uint32 m, unsigned int n = 32) + { + static const uint32 logs[] = { + 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, 0x016FE50B, + 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, 0x0002E2A3, 0x00017153, + 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, 0x000005C5, 0x000002E3, 0x00000171, + 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, 0x0000000C, 0x00000006, 0x00000003, 0x00000001 }; + if(m == 0x40000000) + return 0; + uint32 mx = 0x40000000, my = 0; + for(unsigned int i=1; i>i); + if(mz <= m) + { + mx = mz; + my += logs[i]; + } + } + return my; + } + + /// Fixed point sine and cosine. + /// This uses the CORDIC algorithm in rotation mode. + /// \param mz angle in [-pi/2,pi/2] as Q1.30 + /// \param n number of iterations (at most 31) + /// \return sine and cosine of \a mz as Q1.30 + inline std::pair sincos(uint32 mz, unsigned int n = 31) + { + static const uint32 angles[] = { + 0x3243F6A9, 0x1DAC6705, 0x0FADBAFD, 0x07F56EA7, 0x03FEAB77, 0x01FFD55C, 0x00FFFAAB, 0x007FFF55, + 0x003FFFEB, 0x001FFFFD, 0x00100000, 0x00080000, 0x00040000, 0x00020000, 0x00010000, 0x00008000, + 0x00004000, 0x00002000, 0x00001000, 0x00000800, 0x00000400, 0x00000200, 0x00000100, 0x00000080, + 0x00000040, 0x00000020, 0x00000010, 0x00000008, 0x00000004, 0x00000002, 0x00000001 }; + uint32 mx = 0x26DD3B6A, my = 0; + for(unsigned int i=0; i0x3FF)<<10); + int exp = (abs>>10) + (abs<=0x3FF) - 15; + if(abs < 0x3A48) + return k = 0, m << (exp+20); + #if HALF_ENABLE_CPP11_LONG_LONG + unsigned long long y = m * 0xA2F9836E4E442, mask = (1ULL<<(62-exp)) - 1, yi = (y+(mask>>1)) & ~mask, f = y - yi; + uint32 sign = -static_cast(f>>63); + k = static_cast(yi>>(62-exp)); + return (multiply64(static_cast((sign ? -f : f)>>(31-exp)), 0xC90FDAA2)^sign) - sign; + #else + uint32 yh = m*0xA2F98 + mulhi(m, 0x36E4E442), yl = (m*0x36E4E442) & 0xFFFFFFFF; + uint32 mask = (static_cast(1)<<(30-exp)) - 1, yi = (yh+(mask>>1)) & ~mask, sign = -static_cast(yi>yh); + k = static_cast(yi>>(30-exp)); + uint32 fh = (yh^sign) + (yi^~sign) - ~sign, fl = (yl^sign) - sign; + return (multiply64((exp>-1) ? (((fh<<(1+exp))&0xFFFFFFFF)|((fl&0xFFFFFFFF)>>(31-exp))) : fh, 0xC90FDAA2)^sign) - sign; + #endif + } + + /// Get arguments for atan2 function. + /// \param abs half-precision floating-point value + /// \return \a abs and sqrt(1 - \a abs^2) as Q0.30 + inline std::pair atan2_args(unsigned int abs) + { + int exp = -15; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + uint32 my = ((abs&0x3FF)|0x400) << 5, r = my * my; + int rexp = 2 * exp; + r = 0x40000000 - ((rexp>-31) ? ((r>>-rexp)|((r&((static_cast(1)<<-rexp)-1))!=0)) : 1); + for(rexp=0; r<0x40000000; r<<=1,--rexp) ; + uint32 mx = sqrt<30>(r, rexp); + int d = exp - rexp; + if(d < 0) + return std::make_pair((d<-14) ? ((my>>(-d-14))+((my>>(-d-15))&1)) : (my<<(14+d)), (mx<<14)+(r<<13)/mx); + if(d > 0) + return std::make_pair(my<<14, (d>14) ? ((mx>>(d-14))+((mx>>(d-15))&1)) : ((d==14) ? mx : ((mx<<(14-d))+(r<<(13-d))/mx))); + return std::make_pair(my<<13, (mx<<13)+(r<<12)/mx); + } + + /// Get exponentials for hyperbolic computation + /// \param abs half-precision floating-point value + /// \param exp variable to take unbiased exponent of larger result + /// \param n number of BKM iterations (at most 32) + /// \return exp(abs) and exp(-\a abs) as Q1.31 with same exponent + inline std::pair hyperbolic_args(unsigned int abs, int &exp, unsigned int n = 32) + { + uint32 mx = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29), my; + int e = (abs>>10) + (abs<=0x3FF); + if(e < 14) + { + exp = 0; + mx >>= 14 - e; + } + else + { + exp = mx >> (45-e); + mx = (mx<<(e-14)) & 0x7FFFFFFF; + } + mx = exp2(mx, n); + int d = exp << 1, s; + if(mx > 0x80000000) + { + my = divide64(0x80000000, mx, s); + my |= s; + ++d; + } + else + my = mx; + return std::make_pair(mx, (d<31) ? ((my>>d)|((my&((static_cast(1)< unsigned int exp2_post(uint32 m, int exp, bool esign, unsigned int sign = 0, unsigned int n = 32) + { + if(esign) + { + exp = -exp - (m!=0); + if(exp < -25) + return underflow(sign); + else if(exp == -25) + return rounded(sign, 1, m!=0); + } + else if(exp > 15) + return overflow(sign); + if(!m) + return sign | (((exp+=15)>0) ? (exp<<10) : check_underflow(0x200>>-exp)); + m = exp2(m, n); + int s = 0; + if(esign) + m = divide64(0x80000000, m, s); + return fixed2half(m, exp+14, sign, s); + } + + /// Postprocessing for binary logarithm. + /// \tparam R rounding mode to use + /// \tparam L logarithm for base transformation as Q1.31 + /// \param m fractional part of logarithm as Q0.31 + /// \param ilog signed integer part of logarithm + /// \param exp biased exponent of result + /// \param sign sign bit of result + /// \return value base-transformed and converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int log2_post(uint32 m, int ilog, int exp, unsigned int sign = 0) + { + uint32 msign = sign_mask(ilog); + m = (((static_cast(ilog)<<27)+(m>>4))^msign) - msign; + if(!m) + return 0; + for(; m<0x80000000; m<<=1,--exp) ; + int i = m >= L, s; + exp += i; + m >>= 1 + i; + sign ^= msign & 0x8000; + if(exp < -11) + return underflow(sign); + m = divide64(m, L, s); + return fixed2half(m, exp, sign, 1); + } + + /// Hypotenuse square root and postprocessing. + /// \tparam R rounding mode to use + /// \param r mantissa as Q2.30 + /// \param exp biased exponent + /// \return square root converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int hypot_post(uint32 r, int exp) + { + int i = r >> 31; + if((exp+=i) > 46) + return overflow(); + if(exp < -34) + return underflow(); + r = (r>>i) | (r&i); + uint32 m = sqrt<30>(r, exp+=15); + return fixed2half(m, exp-1, 0, r!=0); + } + + /// Division and postprocessing for tangents. + /// \tparam R rounding mode to use + /// \param my dividend as Q1.31 + /// \param mx divisor as Q1.31 + /// \param exp biased exponent of result + /// \param sign sign bit of result + /// \return quotient converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int tangent_post(uint32 my, uint32 mx, int exp, unsigned int sign = 0) + { + int i = my >= mx, s; + exp += i; + if(exp > 29) + return overflow(sign); + if(exp < -11) + return underflow(sign); + uint32 m = divide64(my>>(i+1), mx, s); + return fixed2half(m, exp, sign, s); + } + + /// Area function and postprocessing. + /// This computes the value directly in Q2.30 using the representation `asinh|acosh(x) = log(x+sqrt(x^2+|-1))`. + /// \tparam R rounding mode to use + /// \tparam S `true` for asinh, `false` for acosh + /// \param arg half-precision argument + /// \return asinh|acosh(\a arg) converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int area(unsigned int arg) + { + int abs = arg & 0x7FFF, expx = (abs>>10) + (abs<=0x3FF) - 15, expy = -15, ilog, i; + uint32 mx = static_cast((abs&0x3FF)|((abs>0x3FF)<<10)) << 20, my, r; + for(; abs<0x400; abs<<=1,--expy) ; + expy += abs >> 10; + r = ((abs&0x3FF)|0x400) << 5; + r *= r; + i = r >> 31; + expy = 2*expy + i; + r >>= i; + if(S) + { + if(expy < 0) + { + r = 0x40000000 + ((expy>-30) ? ((r>>-expy)|((r&((static_cast(1)<<-expy)-1))!=0)) : 1); + expy = 0; + } + else + { + r += 0x40000000 >> expy; + i = r >> 31; + r = (r>>i) | (r&i); + expy += i; + } + } + else + { + r -= 0x40000000 >> expy; + for(; r<0x40000000; r<<=1,--expy) ; + } + my = sqrt<30>(r, expy); + my = (my<<15) + (r<<14)/my; + if(S) + { + mx >>= expy - expx; + ilog = expy; + } + else + { + my >>= expx - expy; + ilog = expx; + } + my += mx; + i = my >> 31; + static const int G = S && (R==std::round_to_nearest); + return log2_post(log2(my>>i, 26+S+G)+(G<<3), ilog+i, 17, arg&(static_cast(S)<<15)); + } + + /// Class for 1.31 unsigned floating-point computation + struct f31 + { + /// Constructor. + /// \param mant mantissa as 1.31 + /// \param e exponent + HALF_CONSTEXPR f31(uint32 mant, int e) : m(mant), exp(e) {} + + /// Constructor. + /// \param abs unsigned half-precision value + f31(unsigned int abs) : exp(-15) + { + for(; abs<0x400; abs<<=1,--exp) ; + m = static_cast((abs&0x3FF)|0x400) << 21; + exp += (abs>>10); + } + + /// Addition operator. + /// \param a first operand + /// \param b second operand + /// \return \a a + \a b + friend f31 operator+(f31 a, f31 b) + { + if(b.exp > a.exp) + std::swap(a, b); + int d = a.exp - b.exp; + uint32 m = a.m + ((d<32) ? (b.m>>d) : 0); + int i = (m&0xFFFFFFFF) < a.m; + return f31(((m+i)>>i)|0x80000000, a.exp+i); + } + + /// Subtraction operator. + /// \param a first operand + /// \param b second operand + /// \return \a a - \a b + friend f31 operator-(f31 a, f31 b) + { + int d = a.exp - b.exp, exp = a.exp; + uint32 m = a.m - ((d<32) ? (b.m>>d) : 0); + if(!m) + return f31(0, -32); + for(; m<0x80000000; m<<=1,--exp) ; + return f31(m, exp); + } + + /// Multiplication operator. + /// \param a first operand + /// \param b second operand + /// \return \a a * \a b + friend f31 operator*(f31 a, f31 b) + { + uint32 m = multiply64(a.m, b.m); + int i = m >> 31; + return f31(m<<(1-i), a.exp + b.exp + i); + } + + /// Division operator. + /// \param a first operand + /// \param b second operand + /// \return \a a / \a b + friend f31 operator/(f31 a, f31 b) + { + int i = a.m >= b.m, s; + uint32 m = divide64((a.m+i)>>i, b.m, s); + return f31(m, a.exp - b.exp + i - 1); + } + + uint32 m; ///< mantissa as 1.31. + int exp; ///< exponent. + }; + + /// Error function and postprocessing. + /// This computes the value directly in Q1.31 using the approximations given + /// [here](https://en.wikipedia.org/wiki/Error_function#Approximation_with_elementary_functions). + /// \tparam R rounding mode to use + /// \tparam C `true` for comlementary error function, `false` else + /// \param arg half-precision function argument + /// \return approximated value of error function in half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int erf(unsigned int arg) + { + unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; + f31 x(abs), x2 = x * x * f31(0xB8AA3B29, 0), t = f31(0x80000000, 0) / (f31(0x80000000, 0)+f31(0xA7BA054A, -2)*x), t2 = t * t; + f31 e = ((f31(0x87DC2213, 0)*t2+f31(0xB5F0E2AE, 0))*t2+f31(0x82790637, -2)-(f31(0xBA00E2B8, 0)*t2+f31(0x91A98E62, -2))*t) * t / + ((x2.exp<0) ? f31(exp2((x2.exp>-32) ? (x2.m>>-x2.exp) : 0, 30), 0) : f31(exp2((x2.m<>(31-x2.exp))); + return (!C || sign) ? fixed2half(0x80000000-(e.m>>(C-e.exp)), 14+C, sign&(C-1U)) : + (e.exp<-25) ? underflow() : fixed2half(e.m>>1, e.exp+14, 0, e.m&1); + } + + /// Gamma function and postprocessing. + /// This approximates the value of either the gamma function or its logarithm directly in Q1.31. + /// \tparam R rounding mode to use + /// \tparam L `true` for lograithm of gamma function, `false` for gamma function + /// \param arg half-precision floating-point value + /// \return lgamma/tgamma(\a arg) in half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if \a arg is not a positive integer + template unsigned int gamma(unsigned int arg) + { +/* static const double p[] ={ 2.50662827563479526904, 225.525584619175212544, -268.295973841304927459, 80.9030806934622512966, -5.00757863970517583837, 0.0114684895434781459556 }; + double t = arg + 4.65, s = p[0]; + for(unsigned int i=0; i<5; ++i) + s += p[i+1] / (arg+i); + return std::log(s) + (arg-0.5)*std::log(t) - t; +*/ static const f31 pi(0xC90FDAA2, 1), lbe(0xB8AA3B29, 0); + unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; + bool bsign = sign != 0; + f31 z(abs), x = sign ? (z+f31(0x80000000, 0)) : z, t = x + f31(0x94CCCCCD, 2), s = + f31(0xA06C9901, 1) + f31(0xBBE654E2, -7)/(x+f31(0x80000000, 2)) + f31(0xA1CE6098, 6)/(x+f31(0x80000000, 1)) + + f31(0xE1868CB7, 7)/x - f31(0x8625E279, 8)/(x+f31(0x80000000, 0)) - f31(0xA03E158F, 2)/(x+f31(0xC0000000, 1)); + int i = (s.exp>=2) + (s.exp>=4) + (s.exp>=8) + (s.exp>=16); + s = f31((static_cast(s.exp)<<(31-i))+(log2(s.m>>1, 28)>>i), i) / lbe; + if(x.exp != -1 || x.m != 0x80000000) + { + i = (t.exp>=2) + (t.exp>=4) + (t.exp>=8); + f31 l = f31((static_cast(t.exp)<<(31-i))+(log2(t.m>>1, 30)>>i), i) / lbe; + s = (x.exp<-1) ? (s-(f31(0x80000000, -1)-x)*l) : (s+(x-f31(0x80000000, -1))*l); + } + s = x.exp ? (s-t) : (t-s); + if(bsign) + { + if(z.exp >= 0) + { + sign &= (L|((z.m>>(31-z.exp))&1)) - 1; + for(z=f31((z.m<<(1+z.exp))&0xFFFFFFFF, -1); z.m<0x80000000; z.m<<=1,--z.exp) ; + } + if(z.exp == -1) + z = f31(0x80000000, 0) - z; + if(z.exp < -1) + { + z = z * pi; + z.m = sincos(z.m>>(1-z.exp), 30).first; + for(z.exp=1; z.m<0x80000000; z.m<<=1,--z.exp) ; + } + else + z = f31(0x80000000, 0); + } + if(L) + { + if(bsign) + { + f31 l(0x92868247, 0); + if(z.exp < 0) + { + uint32 m = log2((z.m+1)>>1, 27); + z = f31(-((static_cast(z.exp)<<26)+(m>>5)), 5); + for(; z.m<0x80000000; z.m<<=1,--z.exp) ; + l = l + z / lbe; + } + sign = static_cast(x.exp&&(l.exp(x.exp==0) << 15; + if(s.exp < -24) + return underflow(sign); + if(s.exp > 15) + return overflow(sign); + } + } + else + { + s = s * lbe; + uint32 m; + if(s.exp < 0) + { + m = s.m >> -s.exp; + s.exp = 0; + } + else + { + m = (s.m<>(31-s.exp)); + } + s.m = exp2(m, 27); + if(!x.exp) + s = f31(0x80000000, 0) / s; + if(bsign) + { + if(z.exp < 0) + s = s * z; + s = pi / s; + if(s.exp < -24) + return underflow(sign); + } + else if(z.exp > 0 && !(z.m&((1<<(31-z.exp))-1))) + return ((s.exp+14)<<10) + (s.m>>21); + if(s.exp > 15) + return overflow(sign); + } + return fixed2half(s.m, s.exp+14, sign); + } + /// \} + + template struct half_caster; + } + + /// Half-precision floating-point type. + /// This class implements an IEEE-conformant half-precision floating-point type with the usual arithmetic + /// operators and conversions. It is implicitly convertible to single-precision floating-point, which makes artihmetic + /// expressions and functions with mixed-type operands to be of the most precise operand type. + /// + /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and + /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which + /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the + /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of + /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most + /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit + /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if + /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on + /// nearly any reasonable platform. + /// + /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable + /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. + class half + { + public: + /// \name Construction and assignment + /// \{ + + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics + /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. + HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} + + /// Conversion constructor. + /// \param rhs float to convert + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + explicit half(float rhs) : data_(static_cast(detail::float2half(rhs))) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + half& operator=(float rhs) { data_ = static_cast(detail::float2half(rhs)); return *this; } + + /// \} + /// \name Arithmetic updates + /// \{ + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + /// \exception FE_... according to operator+(half,half) + half& operator+=(half rhs) { return *this = *this + rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + /// \exception FE_... according to operator-(half,half) + half& operator-=(half rhs) { return *this = *this - rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + /// \exception FE_... according to operator*(half,half) + half& operator*=(half rhs) { return *this = *this * rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + /// \exception FE_... according to operator/(half,half) + half& operator/=(half rhs) { return *this = *this / rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator+=(float rhs) { return *this = *this + rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator-=(float rhs) { return *this = *this - rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator*=(float rhs) { return *this = *this * rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator/=(float rhs) { return *this = *this / rhs; } + + /// \} + /// \name Increment and decrement + /// \{ + + /// Prefix increment. + /// \return incremented half value + /// \exception FE_... according to operator+(half,half) + half& operator++() { return *this = *this + half(detail::binary, 0x3C00); } + + /// Prefix decrement. + /// \return decremented half value + /// \exception FE_... according to operator-(half,half) + half& operator--() { return *this = *this + half(detail::binary, 0xBC00); } + + /// Postfix increment. + /// \return non-incremented half value + /// \exception FE_... according to operator+(half,half) + half operator++(int) { half out(*this); ++*this; return out; } + + /// Postfix decrement. + /// \return non-decremented half value + /// \exception FE_... according to operator-(half,half) + half operator--(int) { half out(*this); --*this; return out; } + /// \} + + private: + /// Rounding mode to use + static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + HALF_CONSTEXPR half(detail::binary_t, unsigned int bits) HALF_NOEXCEPT : data_(static_cast(bits)) {} + + /// Internal binary representation + detail::uint16 data_; + + #ifndef HALF_DOXYGEN_ONLY + friend HALF_CONSTEXPR_NOERR bool operator==(half, half); + friend HALF_CONSTEXPR_NOERR bool operator!=(half, half); + friend HALF_CONSTEXPR_NOERR bool operator<(half, half); + friend HALF_CONSTEXPR_NOERR bool operator>(half, half); + friend HALF_CONSTEXPR_NOERR bool operator<=(half, half); + friend HALF_CONSTEXPR_NOERR bool operator>=(half, half); + friend HALF_CONSTEXPR half operator-(half); + friend half operator+(half, half); + friend half operator-(half, half); + friend half operator*(half, half); + friend half operator/(half, half); + template friend std::basic_ostream& operator<<(std::basic_ostream&, half); + template friend std::basic_istream& operator>>(std::basic_istream&, half&); + friend HALF_CONSTEXPR half fabs(half); + friend half fmod(half, half); + friend half remainder(half, half); + friend half remquo(half, half, int*); + friend half fma(half, half, half); + friend HALF_CONSTEXPR_NOERR half fmax(half, half); + friend HALF_CONSTEXPR_NOERR half fmin(half, half); + friend half fdim(half, half); + friend half nanh(const char*); + friend half exp(half); + friend half exp2(half); + friend half expm1(half); + friend half log(half); + friend half log10(half); + friend half log2(half); + friend half log1p(half); + friend half sqrt(half); + friend half rsqrt(half); + friend half cbrt(half); + friend half hypot(half, half); + friend half hypot(half, half, half); + friend half pow(half, half); + friend void sincos(half, half*, half*); + friend half sin(half); + friend half cos(half); + friend half tan(half); + friend half asin(half); + friend half acos(half); + friend half atan(half); + friend half atan2(half, half); + friend half sinh(half); + friend half cosh(half); + friend half tanh(half); + friend half asinh(half); + friend half acosh(half); + friend half atanh(half); + friend half erf(half); + friend half erfc(half); + friend half lgamma(half); + friend half tgamma(half); + friend half ceil(half); + friend half floor(half); + friend half trunc(half); + friend half round(half); + friend long lround(half); + friend half rint(half); + friend long lrint(half); + friend half nearbyint(half); + #ifdef HALF_ENABLE_CPP11_LONG_LONG + friend long long llround(half); + friend long long llrint(half); + #endif + friend half frexp(half, int*); + friend half scalbln(half, long); + friend half modf(half, half*); + friend int ilogb(half); + friend half logb(half); + friend half nextafter(half, half); + friend half nexttoward(half, long double); + friend HALF_CONSTEXPR half copysign(half, half); + friend HALF_CONSTEXPR int fpclassify(half); + friend HALF_CONSTEXPR bool isfinite(half); + friend HALF_CONSTEXPR bool isinf(half); + friend HALF_CONSTEXPR bool isnan(half); + friend HALF_CONSTEXPR bool isnormal(half); + friend HALF_CONSTEXPR bool signbit(half); + friend HALF_CONSTEXPR bool isgreater(half, half); + friend HALF_CONSTEXPR bool isgreaterequal(half, half); + friend HALF_CONSTEXPR bool isless(half, half); + friend HALF_CONSTEXPR bool islessequal(half, half); + friend HALF_CONSTEXPR bool islessgreater(half, half); + template friend struct detail::half_caster; + friend class std::numeric_limits; + #if HALF_ENABLE_CPP11_HASH + friend struct std::hash; + #endif + #if HALF_ENABLE_CPP11_USER_LITERALS + friend half literal::operator "" _h(long double); + #endif + #endif + }; + +#if HALF_ENABLE_CPP11_USER_LITERALS + namespace literal + { + /// Half literal. + /// While this returns a properly rounded half-precision value, half literals can unfortunately not be constant + /// expressions due to rather involved conversions. So don't expect this to be a literal literal without involving + /// conversion operations at runtime. It is a convenience feature, not a performance optimization. + /// \param value literal value + /// \return half with of given value (possibly rounded) + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator "" _h(long double value) { return half(detail::binary, detail::float2half(value)); } + } +#endif + + namespace detail + { + /// Helper class for half casts. + /// This class template has to be specialized for all valid cast arguments to define an appropriate static + /// `cast` member function and a corresponding `type` member denoting its return type. + /// \tparam T destination type + /// \tparam U source type + /// \tparam R rounding mode to use + template struct half_caster {}; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); + #endif + + static half cast(U arg) { return cast_impl(arg, is_float()); }; + + private: + static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } + static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } + }; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); + #endif + + static T cast(half arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } + }; + template struct half_caster + { + static half cast(half arg) { return arg; } + }; + } +} + +/// Extensions to the C++ standard library. +namespace std +{ + /// Numeric limits for half-precision floats. + /// **See also:** Documentation for [std::numeric_limits](https://en.cppreference.com/w/cpp/types/numeric_limits) + template<> class numeric_limits + { + public: + /// Is template specialization. + static HALF_CONSTEXPR_CONST bool is_specialized = true; + + /// Supports signed values. + static HALF_CONSTEXPR_CONST bool is_signed = true; + + /// Is not an integer type. + static HALF_CONSTEXPR_CONST bool is_integer = false; + + /// Is not exact. + static HALF_CONSTEXPR_CONST bool is_exact = false; + + /// Doesn't provide modulo arithmetic. + static HALF_CONSTEXPR_CONST bool is_modulo = false; + + /// Has a finite set of values. + static HALF_CONSTEXPR_CONST bool is_bounded = true; + + /// IEEE conformant. + static HALF_CONSTEXPR_CONST bool is_iec559 = true; + + /// Supports infinity. + static HALF_CONSTEXPR_CONST bool has_infinity = true; + + /// Supports quiet NaNs. + static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + + /// Supports signaling NaNs. + static HALF_CONSTEXPR_CONST bool has_signaling_NaN = true; + + /// Supports subnormal values. + static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + + /// Supports no denormalization detection. + static HALF_CONSTEXPR_CONST bool has_denorm_loss = false; + + #if HALF_ERRHANDLING_THROWS + static HALF_CONSTEXPR_CONST bool traps = true; + #else + /// Traps only if [HALF_ERRHANDLING_THROW_...](\ref HALF_ERRHANDLING_THROW_INVALID) is acitvated. + static HALF_CONSTEXPR_CONST bool traps = false; + #endif + + /// Does not support no pre-rounding underflow detection. + static HALF_CONSTEXPR_CONST bool tinyness_before = false; + + /// Rounding mode. + static HALF_CONSTEXPR_CONST float_round_style round_style = half_float::half::round_style; + + /// Significant digits. + static HALF_CONSTEXPR_CONST int digits = 11; + + /// Significant decimal digits. + static HALF_CONSTEXPR_CONST int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static HALF_CONSTEXPR_CONST int max_digits10 = 5; + + /// Number base. + static HALF_CONSTEXPR_CONST int radix = 2; + + /// One more than smallest exponent. + static HALF_CONSTEXPR_CONST int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static HALF_CONSTEXPR_CONST int min_exponent10 = -4; + + /// One more than largest exponent + static HALF_CONSTEXPR_CONST int max_exponent = 16; + + /// Largest finitely representable power of 10. + static HALF_CONSTEXPR_CONST int max_exponent10 = 4; + + /// Smallest positive normal value. + static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); } + + /// Smallest finite value. + static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); } + + /// Largest finite value. + static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); } + + /// Difference between 1 and next representable value. + static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); } + + /// Maximum rounding error in ULP (units in the last place). + static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW + { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } + + /// Positive infinity. + static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); } + + /// Quiet NaN. + static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); } + + /// Signaling NaN. + static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); } + + /// Smallest positive subnormal value. + static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); } + }; + +#if HALF_ENABLE_CPP11_HASH + /// Hash function for half-precision floats. + /// This is only defined if C++11 `std::hash` is supported and enabled. + /// + /// **See also:** Documentation for [std::hash](https://en.cppreference.com/w/cpp/utility/hash) + template<> struct hash + { + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const { return hash()(arg.data_&-static_cast(arg.data_!=0x8000)); } + }; +#endif +} + +namespace half_float +{ + /// \anchor compop + /// \name Comparison operators + /// \{ + + /// Comparison for equality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator==(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)); + } + + /// Comparison for inequality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator!=(half x, half y) + { + return detail::compsignal(x.data_, y.data_) || (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)); + } + + /// Comparison for less than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator<(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) < ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// Comparison for greater than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator>(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) > ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// Comparison for less equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator<=(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) <= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// Comparison for greater equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator>=(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) >= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// \} + /// \anchor arithmetics + /// \name Arithmetic operators + /// \{ + + /// Identity. + /// \param arg operand + /// \return unchanged operand + inline HALF_CONSTEXPR half operator+(half arg) { return arg; } + + /// Negation. + /// \param arg operand + /// \return negated operand + inline HALF_CONSTEXPR half operator-(half arg) { return half(detail::binary, arg.data_^0x8000); } + + /// Addition. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return sum of half expressions + /// \exception FE_INVALID if \a x and \a y are infinities with different signs or signaling NaNs + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator+(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)+detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF; + bool sub = ((x.data_^y.data_)&0x8000) != 0; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : (absy!=0x7C00) ? x.data_ : + (sub && absx==0x7C00) ? detail::invalid() : y.data_); + if(!absx) + return absy ? y : half(detail::binary, (half::round_style==std::round_toward_neg_infinity) ? (x.data_|y.data_) : (x.data_&y.data_)); + if(!absy) + return x; + unsigned int sign = ((sub && absy>absx) ? y.data_ : x.data_) & 0x8000; + if(absy > absx) + std::swap(absx, absy); + int exp = (absx>>10) + (absx<=0x3FF), d = exp - (absy>>10) - (absy<=0x3FF), mx = ((absx&0x3FF)|((absx>0x3FF)<<10)) << 3, my; + if(d < 13) + { + my = ((absy&0x3FF)|((absy>0x3FF)<<10)) << 3; + my = (my>>d) | ((my&((1<(half::round_style==std::round_toward_neg_infinity)<<15); + for(; mx<0x2000 && exp>1; mx<<=1,--exp) ; + } + else + { + mx += my; + int i = mx >> 14; + if((exp+=i) > 30) + return half(detail::binary, detail::overflow(sign)); + mx = (mx>>i) | (mx&i); + } + return half(detail::binary, detail::rounded(sign+((exp-1)<<10)+(mx>>3), (mx>>2)&1, (mx&0x3)!=0)); + #endif + } + + /// Subtraction. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return difference of half expressions + /// \exception FE_INVALID if \a x and \a y are infinities with equal signs or signaling NaNs + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator-(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)-detail::half2float(y.data_))); + #else + return x + -y; + #endif + } + + /// Multiplication. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return product of half expressions + /// \exception FE_INVALID if multiplying 0 with infinity or if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator*(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)*detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -16; + unsigned int sign = (x.data_^y.data_) & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + ((absx==0x7C00 && !absy)||(absy==0x7C00 && !absx)) ? detail::invalid() : (sign|0x7C00)); + if(!absx || !absy) + return half(detail::binary, sign); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + detail::uint32 m = static_cast((absx&0x3FF)|0x400) * static_cast((absy&0x3FF)|0x400); + int i = m >> 21, s = m & i; + exp += (absx>>10) + (absy>>10) + i; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -11) + return half(detail::binary, detail::underflow(sign)); + return half(detail::binary, detail::fixed2half(m>>i, exp, sign, s)); + #endif + } + + /// Division. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return quotient of half expressions + /// \exception FE_INVALID if dividing 0s or infinities with each other or if \a x or \a y is signaling NaN + /// \exception FE_DIVBYZERO if dividing finite value by 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator/(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)/detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = 14; + unsigned int sign = (x.data_^y.data_) & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==absy) ? detail::invalid() : (sign|((absx==0x7C00) ? 0x7C00 : 0))); + if(!absx) + return half(detail::binary, absy ? sign : detail::invalid()); + if(!absy) + return half(detail::binary, detail::pole(sign)); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,++exp) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + int i = mx < my; + exp += (absx>>10) - (absy>>10) - i; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -11) + return half(detail::binary, detail::underflow(sign)); + mx <<= 12 + i; + my <<= 1; + return half(detail::binary, detail::fixed2half(mx/my, exp, sign, mx%my!=0)); + #endif + } + + /// \} + /// \anchor streaming + /// \name Input and output + /// \{ + + /// Output operator. + /// This uses the built-in functionality for streaming out floating-point numbers. + /// \param out output stream to write into + /// \param arg half expression to write + /// \return reference to output stream + template std::basic_ostream& operator<<(std::basic_ostream &out, half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return out << detail::half2float(arg.data_); + #else + return out << detail::half2float(arg.data_); + #endif + } + + /// Input operator. + /// This uses the built-in functionality for streaming in floating-point numbers, specifically double precision floating + /// point numbers (unless overridden with [HALF_ARITHMETIC_TYPE](\ref HALF_ARITHMETIC_TYPE)). So the input string is first + /// rounded to double precision using the underlying platform's current floating-point rounding mode before being rounded + /// to half-precision using the library's half-precision rounding mode. + /// \param in input stream to read from + /// \param arg half to read into + /// \return reference to input stream + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template std::basic_istream& operator>>(std::basic_istream &in, half &arg) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t f; + #else + double f; + #endif + if(in >> f) + arg.data_ = detail::float2half(f); + return in; + } + + /// \} + /// \anchor basic + /// \name Basic mathematical operations + /// \{ + + /// Absolute value. + /// **See also:** Documentation for [std::fabs](https://en.cppreference.com/w/cpp/numeric/math/fabs). + /// \param arg operand + /// \return absolute value of \a arg + inline HALF_CONSTEXPR half fabs(half arg) { return half(detail::binary, arg.data_&0x7FFF); } + + /// Absolute value. + /// **See also:** Documentation for [std::abs](https://en.cppreference.com/w/cpp/numeric/math/fabs). + /// \param arg operand + /// \return absolute value of \a arg + inline HALF_CONSTEXPR half abs(half arg) { return fabs(arg); } + + /// Remainder of division. + /// **See also:** Documentation for [std::fmod](https://en.cppreference.com/w/cpp/numeric/math/fmod). + /// \param x first operand + /// \param y second operand + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half fmod(half x, half y) + { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : x.data_); + if(!absy) + return half(detail::binary, detail::invalid()); + if(!absx) + return x; + if(absx == absy) + return half(detail::binary, sign); + return half(detail::binary, sign|detail::mod(absx, absy)); + } + + /// Remainder of division. + /// **See also:** Documentation for [std::remainder](https://en.cppreference.com/w/cpp/numeric/math/remainder). + /// \param x first operand + /// \param y second operand + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half remainder(half x, half y) + { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : x.data_); + if(!absy) + return half(detail::binary, detail::invalid()); + if(absx == absy) + return half(detail::binary, sign); + return half(detail::binary, sign^detail::mod(absx, absy)); + } + + /// Remainder of division. + /// **See also:** Documentation for [std::remquo](https://en.cppreference.com/w/cpp/numeric/math/remquo). + /// \param x first operand + /// \param y second operand + /// \param quo address to store some bits of quotient at + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half remquo(half x, half y, int *quo) + { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, value = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : (*quo = 0, x.data_)); + if(!absy) + return half(detail::binary, detail::invalid()); + bool qsign = ((value^y.data_)&0x8000) != 0; + int q = 1; + if(absx != absy) + value ^= detail::mod(absx, absy, &q); + return *quo = qsign ? -q : q, half(detail::binary, value); + } + + /// Fused multiply add. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::fma](https://en.cppreference.com/w/cpp/numeric/math/fma). + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return ( \a x * \a y ) + \a z rounded as one operation. + /// \exception FE_INVALID according to operator*() and operator+() unless any argument is a quiet NaN and no argument is a signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding the final addition + inline half fma(half x, half y, half z) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_), fz = detail::half2float(z.data_); + #if HALF_ENABLE_CPP11_CMATH && FP_FAST_FMA + return half(detail::binary, detail::float2half(std::fma(fx, fy, fz))); + #else + return half(detail::binary, detail::float2half(fx*fy+fz)); + #endif + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, exp = -15; + unsigned int sign = (x.data_^y.data_) & 0x8000; + bool sub = ((sign^z.data_)&0x8000) != 0; + if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) + return (absx>0x7C00 || absy>0x7C00 || absz>0x7C00) ? half(detail::binary, detail::signal(x.data_, y.data_, z.data_)) : + (absx==0x7C00) ? half(detail::binary, (!absy || (sub && absz==0x7C00)) ? detail::invalid() : (sign|0x7C00)) : + (absy==0x7C00) ? half(detail::binary, (!absx || (sub && absz==0x7C00)) ? detail::invalid() : (sign|0x7C00)) : z; + if(!absx || !absy) + return absz ? z : half(detail::binary, (half::round_style==std::round_toward_neg_infinity) ? (z.data_|sign) : (z.data_&sign)); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + detail::uint32 m = static_cast((absx&0x3FF)|0x400) * static_cast((absy&0x3FF)|0x400); + int i = m >> 21; + exp += (absx>>10) + (absy>>10) + i; + m <<= 3 - i; + if(absz) + { + int expz = 0; + for(; absz<0x400; absz<<=1,--expz) ; + expz += absz >> 10; + detail::uint32 mz = static_cast((absz&0x3FF)|0x400) << 13; + if(expz > exp || (expz == exp && mz > m)) + { + std::swap(m, mz); + std::swap(exp, expz); + if(sub) + sign = z.data_ & 0x8000; + } + int d = exp - expz; + mz = (d<23) ? ((mz>>d)|((mz&((static_cast(1)<(half::round_style==std::round_toward_neg_infinity)<<15); + for(; m<0x800000; m<<=1,--exp) ; + } + else + { + m += mz; + i = m >> 24; + m = (m>>i) | (m&i); + exp += i; + } + } + if(exp > 30) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -10) + return half(detail::binary, detail::underflow(sign)); + return half(detail::binary, detail::fixed2half(m, exp-1, sign)); + #endif + } + + /// Maximum of half expressions. + /// **See also:** Documentation for [std::fmax](https://en.cppreference.com/w/cpp/numeric/math/fmax). + /// \param x first operand + /// \param y second operand + /// \return maximum of operands, ignoring quiet NaNs + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR half fmax(half x, half y) + { + return half(detail::binary, (!isnan(y) && (isnan(x) || (x.data_^(0x8000|(0x8000-(x.data_>>15)))) < + (y.data_^(0x8000|(0x8000-(y.data_>>15)))))) ? detail::select(y.data_, x.data_) : detail::select(x.data_, y.data_)); + } + + /// Minimum of half expressions. + /// **See also:** Documentation for [std::fmin](https://en.cppreference.com/w/cpp/numeric/math/fmin). + /// \param x first operand + /// \param y second operand + /// \return minimum of operands, ignoring quiet NaNs + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR half fmin(half x, half y) + { + return half(detail::binary, (!isnan(y) && (isnan(x) || (x.data_^(0x8000|(0x8000-(x.data_>>15)))) > + (y.data_^(0x8000|(0x8000-(y.data_>>15)))))) ? detail::select(y.data_, x.data_) : detail::select(x.data_, y.data_)); + } + + /// Positive difference. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::fdim](https://en.cppreference.com/w/cpp/numeric/math/fdim). + /// \param x first operand + /// \param y second operand + /// \return \a x - \a y or 0 if difference negative + /// \exception FE_... according to operator-(half,half) + inline half fdim(half x, half y) + { + if(isnan(x) || isnan(y)) + return half(detail::binary, detail::signal(x.data_, y.data_)); + return (x.data_^(0x8000|(0x8000-(x.data_>>15)))) <= (y.data_^(0x8000|(0x8000-(y.data_>>15)))) ? half(detail::binary, 0) : (x-y); + } + + /// Get NaN value. + /// **See also:** Documentation for [std::nan](https://en.cppreference.com/w/cpp/numeric/math/nan). + /// \param arg string code + /// \return quiet NaN + inline half nanh(const char *arg) + { + unsigned int value = 0x7FFF; + while(*arg) + value ^= static_cast(*arg++) & 0xFF; + return half(detail::binary, value); + } + + /// \} + /// \anchor exponential + /// \name Exponential functions + /// \{ + + /// Exponential function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::exp](https://en.cppreference.com/w/cpp/numeric/math/exp). + /// \param arg function argument + /// \return e raised to \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half exp(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::exp(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, e = (abs>>10) + (abs<=0x3FF), exp; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00&((arg.data_>>15)-1U)) : detail::signal(arg.data_)); + if(abs >= 0x4C80) + return half(detail::binary, (arg.data_&0x8000) ? detail::underflow() : detail::overflow()); + detail::uint32 m = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29); + if(e < 14) + { + exp = 0; + m >>= 14 - e; + } + else + { + exp = m >> (45-e); + m = (m<<(e-14)) & 0x7FFFFFFF; + } + return half(detail::binary, detail::exp2_post(m, exp, (arg.data_&0x8000)!=0, 0, 26)); + #endif + } + + /// Binary exponential. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::exp2](https://en.cppreference.com/w/cpp/numeric/math/exp2). + /// \param arg function argument + /// \return 2 raised to \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half exp2(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::exp2(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, e = (abs>>10) + (abs<=0x3FF), exp = (abs&0x3FF) + ((abs>0x3FF)<<10); + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00&((arg.data_>>15)-1U)) : detail::signal(arg.data_)); + if(abs >= 0x4E40) + return half(detail::binary, (arg.data_&0x8000) ? detail::underflow() : detail::overflow()); + return half(detail::binary, detail::exp2_post( + (static_cast(exp)<<(6+e))&0x7FFFFFFF, exp>>(25-e), (arg.data_&0x8000)!=0, 0, 28)); + #endif + } + + /// Exponential minus one. + /// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for `std::round_to_nearest` + /// and in <1% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::expm1](https://en.cppreference.com/w/cpp/numeric/math/expm1). + /// \param arg function argument + /// \return e raised to \a arg and subtracted by 1 + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half expm1(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::expm1(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000, e = (abs>>10) + (abs<=0x3FF), exp; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00+(sign>>1)) : detail::signal(arg.data_)); + if(abs >= 0x4A00) + return half(detail::binary, (arg.data_&0x8000) ? detail::rounded(0xBBFF, 1, 1) : detail::overflow()); + detail::uint32 m = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29); + if(e < 14) + { + exp = 0; + m >>= 14 - e; + } + else + { + exp = m >> (45-e); + m = (m<<(e-14)) & 0x7FFFFFFF; + } + m = detail::exp2(m); + if(sign) + { + int s = 0; + if(m > 0x80000000) + { + ++exp; + m = detail::divide64(0x80000000, m, s); + } + m = 0x80000000 - ((m>>exp)|((m&((static_cast(1)<>exp) : 1; + for(exp+=14; m<0x80000000 && exp; m<<=1,--exp) ; + if(exp > 29) + return half(detail::binary, detail::overflow()); + return half(detail::binary, detail::rounded(sign+(exp<<10)+(m>>21), (m>>20)&1, (m&0xFFFFF)!=0)); + #endif + } + + /// Natural logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log](https://en.cppreference.com/w/cpp/numeric/math/log). + /// \param arg function argument + /// \return logarithm of \a arg to base e + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::log(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + return half(detail::binary, detail::log2_post( + detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 27)+8, exp, 17)); + #endif + } + + /// Common logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log10](https://en.cppreference.com/w/cpp/numeric/math/log10). + /// \param arg function argument + /// \return logarithm of \a arg to base 10 + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log10(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::log10(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + switch(abs) + { + case 0x4900: return half(detail::binary, 0x3C00); + case 0x5640: return half(detail::binary, 0x4000); + case 0x63D0: return half(detail::binary, 0x4200); + case 0x70E2: return half(detail::binary, 0x4400); + } + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + return half(detail::binary, detail::log2_post( + detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 27)+8, exp, 16)); + #endif + } + + /// Binary logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log2](https://en.cppreference.com/w/cpp/numeric/math/log2). + /// \param arg function argument + /// \return logarithm of \a arg to base 2 + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log2(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::log2(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15, s = 0; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + if(abs == 0x3C00) + return half(detail::binary, 0); + for(; abs<0x400; abs<<=1,--exp) ; + exp += (abs>>10); + if(!(abs&0x3FF)) + { + unsigned int value = static_cast(exp<0) << 15, m = std::abs(exp) << 6; + for(exp=18; m<0x400; m<<=1,--exp) ; + return half(detail::binary, value+(exp<<10)+m); + } + detail::uint32 ilog = exp, sign = detail::sign_mask(ilog), m = + (((ilog<<27)+(detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 28)>>4))^sign) - sign; + if(!m) + return half(detail::binary, 0); + for(exp=14; m<0x8000000 && exp; m<<=1,--exp) ; + for(; m>0xFFFFFFF; m>>=1,++exp) + s |= m & 1; + return half(detail::binary, detail::fixed2half(m, exp, sign&0x8000, s)); + #endif + } + + /// Natural logarithm plus one. + /// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for `std::round_to_nearest` + /// and in ~1% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::log1p](https://en.cppreference.com/w/cpp/numeric/math/log1p). + /// \param arg function argument + /// \return logarithm of \a arg plus 1 to base e + /// \exception FE_INVALID for signaling NaN or argument <-1 + /// \exception FE_DIVBYZERO for -1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log1p(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::log1p(detail::half2float(arg.data_)))); + #else + if(arg.data_ >= 0xBC00) + return half(detail::binary, (arg.data_==0xBC00) ? detail::pole(0x8000) : (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + detail::uint32 m = static_cast((abs&0x3FF)|0x400) << 20; + if(arg.data_ & 0x8000) + { + m = 0x40000000 - (m>>-exp); + for(exp=0; m<0x40000000; m<<=1,--exp) ; + } + else + { + if(exp < 0) + { + m = 0x40000000 + (m>>-exp); + exp = 0; + } + else + { + m += 0x40000000 >> exp; + int i = m >> 31; + m >>= i; + exp += i; + } + } + return half(detail::binary, detail::log2_post(detail::log2(m), exp, 17)); + #endif + } + + /// \} + /// \anchor power + /// \name Power functions + /// \{ + + /// Square root. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sqrt](https://en.cppreference.com/w/cpp/numeric/math/sqrt). + /// \param arg function argument + /// \return square root of \a arg + /// \exception FE_INVALID for signaling NaN and negative arguments + /// \exception FE_INEXACT according to rounding + inline half sqrt(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sqrt(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 15; + if(!abs || arg.data_ >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_>0x8000) ? detail::invalid() : arg.data_); + for(; abs<0x400; abs<<=1,--exp) ; + detail::uint32 r = static_cast((abs&0x3FF)|0x400) << 10, m = detail::sqrt<20>(r, exp+=abs>>10); + return half(detail::binary, detail::rounded((exp<<10)+(m&0x3FF), r>m, r!=0)); + #endif + } + + /// Inverse square root. + /// This function is exact to rounding for all rounding modes and thus generally more accurate than directly computing + /// 1 / sqrt(\a arg) in half-precision, in addition to also being faster. + /// \param arg function argument + /// \return reciprocal of square root of \a arg + /// \exception FE_INVALID for signaling NaN and negative arguments + /// \exception FE_INEXACT according to rounding + inline half rsqrt(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::internal_t(1)/std::sqrt(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, bias = 0x4000; + if(!abs || arg.data_ >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_>0x8000) ? + detail::invalid() : !abs ? detail::pole(arg.data_&0x8000) : 0); + for(; abs<0x400; abs<<=1,bias-=0x400) ; + unsigned int frac = (abs+=bias) & 0x7FF; + if(frac == 0x400) + return half(detail::binary, 0x7A00-(abs>>1)); + if((half::round_style == std::round_to_nearest && (frac == 0x3FE || frac == 0x76C)) || + (half::round_style != std::round_to_nearest && (frac == 0x15A || frac == 0x3FC || frac == 0x401 || frac == 0x402 || frac == 0x67B))) + return pow(arg, half(detail::binary, 0xB800)); + detail::uint32 f = 0x17376 - abs, mx = (abs&0x3FF) | 0x400, my = ((f>>1)&0x3FF) | 0x400, mz = my * my; + int expy = (f>>11) - 31, expx = 32 - (abs>>10), i = mz >> 21; + for(mz=0x60000000-(((mz>>i)*mx)>>(expx-2*expy-i)); mz<0x40000000; mz<<=1,--expy) ; + i = (my*=mz>>10) >> 31; + expy += i; + my = (my>>(20+i)) + 1; + i = (mz=my*my) >> 21; + for(mz=0x60000000-(((mz>>i)*mx)>>(expx-2*expy-i)); mz<0x40000000; mz<<=1,--expy) ; + i = (my*=(mz>>10)+1) >> 31; + return half(detail::binary, detail::fixed2half(my>>i, expy+i+14)); + #endif + } + + /// Cubic root. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cbrt](https://en.cppreference.com/w/cpp/numeric/math/cbrt). + /// \param arg function argument + /// \return cubic root of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT according to rounding + inline half cbrt(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::cbrt(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs || abs == 0x3C00 || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1, --exp); + detail::uint32 ilog = exp + (abs>>10), sign = detail::sign_mask(ilog), f, m = + (((ilog<<27)+(detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 24)>>4))^sign) - sign; + for(exp=2; m<0x80000000; m<<=1,--exp) ; + m = detail::multiply64(m, 0xAAAAAAAB); + int i = m >> 31, s; + exp += i; + m <<= 1 - i; + if(exp < 0) + { + f = m >> -exp; + exp = 0; + } + else + { + f = (m<> (31-exp); + } + m = detail::exp2(f, (half::round_style==std::round_to_nearest) ? 29 : 26); + if(sign) + { + if(m > 0x80000000) + { + m = detail::divide64(0x80000000, m, s); + ++exp; + } + exp = -exp; + } + return half(detail::binary, (half::round_style==std::round_to_nearest) ? + detail::fixed2half(m, exp+14, arg.data_&0x8000) : + detail::fixed2half((m+0x80)>>8, exp+14, arg.data_&0x8000)); + #endif + } + + /// Hypotenuse function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). + /// \param x first argument + /// \param y second argument + /// \return square root of sum of squares without internal over- or underflows + /// \exception FE_INVALID if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root + inline half hypot(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_); + #if HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::hypot(fx, fy))); + #else + return half(detail::binary, detail::float2half(std::sqrt(fx*fx+fy*fy))); + #endif + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, expx = 0, expy = 0; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx==0x7C00) ? detail::select(0x7C00, y.data_) : + (absy==0x7C00) ? detail::select(0x7C00, x.data_) : detail::signal(x.data_, y.data_)); + if(!absx) + return half(detail::binary, absy ? detail::check_underflow(absy) : 0); + if(!absy) + return half(detail::binary, detail::check_underflow(absx)); + if(absy > absx) + std::swap(absx, absy); + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + mx *= mx; + my *= my; + int ix = mx >> 21, iy = my >> 21; + expx = 2*(expx+(absx>>10)) - 15 + ix; + expy = 2*(expy+(absy>>10)) - 15 + iy; + mx <<= 10 - ix; + my <<= 10 - iy; + int d = expx - expy; + my = (d<30) ? ((my>>d)|((my&((static_cast(1)<(mx+my, expx)); + #endif + } + + /// Hypotenuse function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). + /// \param x first argument + /// \param y second argument + /// \param z third argument + /// \return square root of sum of squares without internal over- or underflows + /// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root + inline half hypot(half x, half y, half z) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_), fz = detail::half2float(z.data_); + return half(detail::binary, detail::float2half(std::sqrt(fx*fx+fy*fy+fz*fz))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, expx = 0, expy = 0, expz = 0; + if(!absx) + return hypot(y, z); + if(!absy) + return hypot(x, z); + if(!absz) + return hypot(x, y); + if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) + return half(detail::binary, (absx==0x7C00) ? detail::select(0x7C00, detail::select(y.data_, z.data_)) : + (absy==0x7C00) ? detail::select(0x7C00, detail::select(x.data_, z.data_)) : + (absz==0x7C00) ? detail::select(0x7C00, detail::select(x.data_, y.data_)) : + detail::signal(x.data_, y.data_, z.data_)); + if(absz > absy) + std::swap(absy, absz); + if(absy > absx) + std::swap(absx, absy); + if(absz > absy) + std::swap(absy, absz); + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + for(; absz<0x400; absz<<=1,--expz) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400, mz = (absz&0x3FF) | 0x400; + mx *= mx; + my *= my; + mz *= mz; + int ix = mx >> 21, iy = my >> 21, iz = mz >> 21; + expx = 2*(expx+(absx>>10)) - 15 + ix; + expy = 2*(expy+(absy>>10)) - 15 + iy; + expz = 2*(expz+(absz>>10)) - 15 + iz; + mx <<= 10 - ix; + my <<= 10 - iy; + mz <<= 10 - iz; + int d = expy - expz; + mz = (d<30) ? ((mz>>d)|((mz&((static_cast(1)<>1) | (my&1); + if(++expy > expx) + { + std::swap(mx, my); + std::swap(expx, expy); + } + } + d = expx - expy; + my = (d<30) ? ((my>>d)|((my&((static_cast(1)<(mx+my, expx)); + #endif + } + + /// Power function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in ~0.00025% of inputs. + /// + /// **See also:** Documentation for [std::pow](https://en.cppreference.com/w/cpp/numeric/math/pow). + /// \param x base + /// \param y exponent + /// \return \a x raised to \a y + /// \exception FE_INVALID if \a x or \a y is signaling NaN or if \a x is finite an negative and \a y is finite and not integral + /// \exception FE_DIVBYZERO if \a x is 0 and \a y is negative + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half pow(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::pow(detail::half2float(x.data_), detail::half2float(y.data_)))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -15; + if(!absy || x.data_ == 0x3C00) + return half(detail::binary, detail::select(0x3C00, (x.data_==0x3C00) ? y.data_ : x.data_)); + bool is_int = absy >= 0x6400 || (absy>=0x3C00 && !(absy&((1<<(25-(absy>>10)))-1))); + unsigned int sign = x.data_ & (static_cast((absy<0x6800)&&is_int&&((absy>>(25-(absy>>10)))&1))<<15); + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absy==0x7C00) ? ((absx==0x3C00) ? 0x3C00 : (!absx && y.data_==0xFC00) ? detail::pole() : + (0x7C00&-((y.data_>>15)^(absx>0x3C00)))) : (sign|(0x7C00&((y.data_>>15)-1U)))); + if(!absx) + return half(detail::binary, (y.data_&0x8000) ? detail::pole(sign) : sign); + if((x.data_&0x8000) && !is_int) + return half(detail::binary, detail::invalid()); + if(x.data_ == 0xBC00) + return half(detail::binary, sign|0x3C00); + switch(y.data_) + { + case 0x3800: return sqrt(x); + case 0x3C00: return half(detail::binary, detail::check_underflow(x.data_)); + case 0x4000: return x * x; + case 0xBC00: return half(detail::binary, 0x3C00) / x; + } + for(; absx<0x400; absx<<=1,--exp) ; + detail::uint32 ilog = exp + (absx>>10), msign = detail::sign_mask(ilog), f, m = + (((ilog<<27)+((detail::log2(static_cast((absx&0x3FF)|0x400)<<20)+8)>>4))^msign) - msign; + for(exp=-11; m<0x80000000; m<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + m = detail::multiply64(m, static_cast((absy&0x3FF)|0x400)<<21); + int i = m >> 31; + exp += (absy>>10) + i; + m <<= 1 - i; + if(exp < 0) + { + f = m >> -exp; + exp = 0; + } + else + { + f = (m<> (31-exp); + } + return half(detail::binary, detail::exp2_post(f, exp, ((msign&1)^(y.data_>>15))!=0, sign)); + #endif + } + + /// \} + /// \anchor trigonometric + /// \name Trigonometric functions + /// \{ + + /// Compute sine and cosine simultaneously. + /// This returns the same results as sin() and cos() but is faster than calling each function individually. + /// + /// This function is exact to rounding for all rounding modes. + /// \param arg function argument + /// \param sin variable to take sine of \a arg + /// \param cos variable to take cosine of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline void sincos(half arg, half *sin, half *cos) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t f = detail::half2float(arg.data_); + *sin = half(detail::binary, detail::float2half(std::sin(f))); + *cos = half(detail::binary, detail::float2half(std::cos(f))); + #else + int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15, k; + if(abs >= 0x7C00) + *sin = *cos = half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + else if(!abs) + { + *sin = arg; + *cos = half(detail::binary, 0x3C00); + } + else if(abs < 0x2500) + { + *sin = half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + } + else + { + if(half::round_style != std::round_to_nearest) + { + switch(abs) + { + case 0x48B7: + *sin = half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x1D07, 1, 1)); + *cos = half(detail::binary, detail::rounded(0xBBFF, 1, 1)); + return; + case 0x598C: + *sin = half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x80FC, 1, 1)); + return; + case 0x6A64: + *sin = half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x3BFE, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x27FF, 1, 1)); + return; + case 0x6D8C: + *sin = half(detail::binary, detail::rounded((arg.data_&0x8000)|0x0FE6, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + return; + } + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + switch(k & 3) + { + case 1: sc = std::make_pair(sc.second, -sc.first); break; + case 2: sc = std::make_pair(-sc.first, -sc.second); break; + case 3: sc = std::make_pair(-sc.second, sc.first); break; + } + *sin = half(detail::binary, detail::fixed2half((sc.first^-static_cast(sign))+sign)); + *cos = half(detail::binary, detail::fixed2half(sc.second)); + } + #endif + } + + /// Sine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sin](https://en.cppreference.com/w/cpp/numeric/math/sin). + /// \param arg function argument + /// \return sine value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half sin(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sin(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, k; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2900) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) + { + case 0x48B7: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x1D07, 1, 1)); + case 0x6A64: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x3BFE, 1, 1)); + case 0x6D8C: return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x0FE6, 1, 1)); + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + detail::uint32 sign = -static_cast(((k>>1)&1)^(arg.data_>>15)); + return half(detail::binary, detail::fixed2half((((k&1) ? sc.second : sc.first)^sign) - sign)); + #endif + } + + /// Cosine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cos](https://en.cppreference.com/w/cpp/numeric/math/cos). + /// \param arg function argument + /// \return cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half cos(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::cos(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, k; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2500) + return half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + if(half::round_style != std::round_to_nearest && abs == 0x598C) + return half(detail::binary, detail::rounded(0x80FC, 1, 1)); + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + detail::uint32 sign = -static_cast(((k>>1)^k)&1); + return half(detail::binary, detail::fixed2half((((k&1) ? sc.first : sc.second)^sign) - sign)); + #endif + } + + /// Tangent function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::tan](https://en.cppreference.com/w/cpp/numeric/math/tan). + /// \param arg function argument + /// \return tangent value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tan(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::tan(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 13, k; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) + { + case 0x658C: return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x07E6, 1, 1)); + case 0x7330: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x4B62, 1, 1)); + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 30); + if(k & 1) + sc = std::make_pair(-sc.second, sc.first); + detail::uint32 signy = detail::sign_mask(sc.first), signx = detail::sign_mask(sc.second); + detail::uint32 my = (sc.first^signy) - signy, mx = (sc.second^signx) - signx; + for(; my<0x80000000; my<<=1,--exp) ; + for(; mx<0x80000000; mx<<=1,++exp) ; + return half(detail::binary, detail::tangent_post(my, mx, exp, (signy^signx^arg.data_)&0x8000)); + #endif + } + + /// Arc sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::asin](https://en.cppreference.com/w/cpp/numeric/math/asin). + /// \param arg function argument + /// \return arc sine value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half asin(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::asin(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(!abs) + return arg; + if(abs >= 0x3C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (abs>0x3C00) ? detail::invalid() : + detail::rounded(sign|0x3E48, 0, 1)); + if(abs < 0x2900) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + if(half::round_style != std::round_to_nearest && (abs == 0x2B44 || abs == 0x2DC3)) + return half(detail::binary, detail::rounded(arg.data_+1, 1, 1)); + std::pair sc = detail::atan2_args(abs); + detail::uint32 m = detail::atan2(sc.first, sc.second, (half::round_style==std::round_to_nearest) ? 27 : 26); + return half(detail::binary, detail::fixed2half(m, 14, sign)); + #endif + } + + /// Arc cosine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::acos](https://en.cppreference.com/w/cpp/numeric/math/acos). + /// \param arg function argument + /// \return arc cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half acos(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::acos(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15; + if(!abs) + return half(detail::binary, detail::rounded(0x3E48, 0, 1)); + if(abs >= 0x3C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (abs>0x3C00) ? detail::invalid() : + sign ? detail::rounded(0x4248, 0, 1) : 0); + std::pair cs = detail::atan2_args(abs); + detail::uint32 m = detail::atan2(cs.second, cs.first, 28); + return half(detail::binary, detail::fixed2half(sign ? (0xC90FDAA2-m) : m, 15, 0, sign)); + #endif + } + + /// Arc tangent function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::atan](https://en.cppreference.com/w/cpp/numeric/math/atan). + /// \param arg function argument + /// \return arc tangent value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atan(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::atan(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::rounded(sign|0x3E48, 0, 1) : detail::signal(arg.data_)); + if(abs <= 0x2700) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + int exp = (abs>>10) + (abs<=0x3FF); + detail::uint32 my = (abs&0x3FF) | ((abs>0x3FF)<<10); + detail::uint32 m = (exp>15) ? detail::atan2(my<<19, 0x20000000>>(exp-15), (half::round_style==std::round_to_nearest) ? 26 : 24) : + detail::atan2(my<<(exp+4), 0x20000000, (half::round_style==std::round_to_nearest) ? 30 : 28); + return half(detail::binary, detail::fixed2half(m, 14, sign)); + #endif + } + + /// Arc tangent function. + /// This function may be 1 ULP off the correctly rounded exact result in ~0.005% of inputs for `std::round_to_nearest`, + /// in ~0.1% of inputs for `std::round_toward_zero` and in ~0.02% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::atan2](https://en.cppreference.com/w/cpp/numeric/math/atan2). + /// \param y numerator + /// \param x denominator + /// \return arc tangent value + /// \exception FE_INVALID if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atan2(half y, half x) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::atan2(detail::half2float(y.data_), detail::half2float(x.data_)))); + #else + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, signx = x.data_ >> 15, signy = y.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + { + if(absx > 0x7C00 || absy > 0x7C00) + return half(detail::binary, detail::signal(x.data_, y.data_)); + if(absy == 0x7C00) + return half(detail::binary, (absx<0x7C00) ? detail::rounded(signy|0x3E48, 0, 1) : + signx ? detail::rounded(signy|0x40B6, 0, 1) : + detail::rounded(signy|0x3A48, 0, 1)); + return (x.data_==0x7C00) ? half(detail::binary, signy) : half(detail::binary, detail::rounded(signy|0x4248, 0, 1)); + } + if(!absy) + return signx ? half(detail::binary, detail::rounded(signy|0x4248, 0, 1)) : y; + if(!absx) + return half(detail::binary, detail::rounded(signy|0x3E48, 0, 1)); + int d = (absy>>10) + (absy<=0x3FF) - (absx>>10) - (absx<=0x3FF); + if(d > (signx ? 18 : 12)) + return half(detail::binary, detail::rounded(signy|0x3E48, 0, 1)); + if(signx && d < -11) + return half(detail::binary, detail::rounded(signy|0x4248, 0, 1)); + if(!signx && d < ((half::round_style==std::round_toward_zero) ? -15 : -9)) + { + for(; absy<0x400; absy<<=1,--d) ; + detail::uint32 mx = ((absx<<1)&0x7FF) | 0x800, my = ((absy<<1)&0x7FF) | 0x800; + int i = my < mx; + d -= i; + if(d < -25) + return half(detail::binary, detail::underflow(signy)); + my <<= 11 + i; + return half(detail::binary, detail::fixed2half(my/mx, d+14, signy, my%mx!=0)); + } + detail::uint32 m = detail::atan2( ((absy&0x3FF)|((absy>0x3FF)<<10))<<(19+((d<0) ? d : (d>0) ? 0 : -1)), + ((absx&0x3FF)|((absx>0x3FF)<<10))<<(19-((d>0) ? d : (d<0) ? 0 : 1))); + return half(detail::binary, detail::fixed2half(signx ? (0xC90FDAA2-m) : m, 15, signy, signx)); + #endif + } + + /// \} + /// \anchor hyperbolic + /// \name Hyperbolic functions + /// \{ + + /// Hyperbolic sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sinh](https://en.cppreference.com/w/cpp/numeric/math/sinh). + /// \param arg function argument + /// \return hyperbolic sine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half sinh(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sinh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + if(abs <= 0x2900) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + std::pair mm = detail::hyperbolic_args(abs, exp, (half::round_style==std::round_to_nearest) ? 29 : 27); + detail::uint32 m = mm.first - mm.second; + for(exp+=13; m<0x80000000 && exp; m<<=1,--exp) ; + unsigned int sign = arg.data_ & 0x8000; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + return half(detail::binary, detail::fixed2half(m, exp, sign)); + #endif + } + + /// Hyperbolic cosine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cosh](https://en.cppreference.com/w/cpp/numeric/math/cosh). + /// \param arg function argument + /// \return hyperbolic cosine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half cosh(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::cosh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : 0x7C00); + std::pair mm = detail::hyperbolic_args(abs, exp, (half::round_style==std::round_to_nearest) ? 23 : 26); + detail::uint32 m = mm.first + mm.second, i = (~m&0xFFFFFFFF) >> 31; + m = (m>>i) | (m&i) | 0x80000000; + if((exp+=13+i) > 29) + return half(detail::binary, detail::overflow()); + return half(detail::binary, detail::fixed2half(m, exp)); + #endif + } + + /// Hyperbolic tangent. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::tanh](https://en.cppreference.com/w/cpp/numeric/math/tanh). + /// \param arg function argument + /// \return hyperbolic tangent value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tanh(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::tanh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_-0x4000)); + if(abs >= 0x4500) + return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest && abs == 0x2D3F) + return half(detail::binary, detail::rounded(arg.data_-3, 0, 1)); + std::pair mm = detail::hyperbolic_args(abs, exp, 27); + detail::uint32 my = mm.first - mm.second - (half::round_style!=std::round_to_nearest), mx = mm.first + mm.second, i = (~mx&0xFFFFFFFF) >> 31; + for(exp=13; my<0x80000000; my<<=1,--exp) ; + mx = (mx>>i) | 0x80000000; + return half(detail::binary, detail::tangent_post(my, mx, exp-i, arg.data_&0x8000)); + #endif + } + + /// Hyperbolic area sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::asinh](https://en.cppreference.com/w/cpp/numeric/math/asinh). + /// \param arg function argument + /// \return area sine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half asinh(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::asinh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + if(abs <= 0x2900) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) + { + case 0x32D4: return half(detail::binary, detail::rounded(arg.data_-13, 1, 1)); + case 0x3B5B: return half(detail::binary, detail::rounded(arg.data_-197, 1, 1)); + } + return half(detail::binary, detail::area(arg.data_)); + #endif + } + + /// Hyperbolic area cosine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::acosh](https://en.cppreference.com/w/cpp/numeric/math/acosh). + /// \param arg function argument + /// \return area cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or arguments <1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half acosh(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::acosh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if((arg.data_&0x8000) || abs < 0x3C00) + return half(detail::binary, (abs<=0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs == 0x3C00) + return half(detail::binary, 0); + if(arg.data_ >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + return half(detail::binary, detail::area(arg.data_)); + #endif + } + + /// Hyperbolic area tangent. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::atanh](https://en.cppreference.com/w/cpp/numeric/math/atanh). + /// \param arg function argument + /// \return area tangent value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_DIVBYZERO for +/-1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atanh(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::atanh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 0; + if(!abs) + return arg; + if(abs >= 0x3C00) + return half(detail::binary, (abs==0x3C00) ? detail::pole(arg.data_&0x8000) : (abs<=0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + detail::uint32 m = static_cast((abs&0x3FF)|((abs>0x3FF)<<10)) << ((abs>>10)+(abs<=0x3FF)+6), my = 0x80000000 + m, mx = 0x80000000 - m; + for(; mx<0x80000000; mx<<=1,++exp) ; + int i = my >= mx, s; + return half(detail::binary, detail::log2_post(detail::log2( + (detail::divide64(my>>i, mx, s)+1)>>1, 27)+0x10, exp+i-1, 16, arg.data_&0x8000)); + #endif + } + + /// \} + /// \anchor special + /// \name Error and gamma functions + /// \{ + + /// Error function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% of inputs. + /// + /// **See also:** Documentation for [std::erf](https://en.cppreference.com/w/cpp/numeric/math/erf). + /// \param arg function argument + /// \return error function value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half erf(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::erf(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF; + if(!abs || abs >= 0x7C00) + return (abs>=0x7C00) ? half(detail::binary, (abs==0x7C00) ? (arg.data_-0x4000) : detail::signal(arg.data_)) : arg; + if(abs >= 0x4200) + return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + return half(detail::binary, detail::erf(arg.data_)); + #endif + } + + /// Complementary error function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% of inputs. + /// + /// **See also:** Documentation for [std::erfc](https://en.cppreference.com/w/cpp/numeric/math/erfc). + /// \param arg function argument + /// \return 1 minus error function value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half erfc(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::erfc(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(abs >= 0x7C00) + return (abs>=0x7C00) ? half(detail::binary, (abs==0x7C00) ? (sign>>1) : detail::signal(arg.data_)) : arg; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x4400) + return half(detail::binary, detail::rounded((sign>>1)-(sign>>15), sign>>15, 1)); + return half(detail::binary, detail::erf(arg.data_)); + #endif + } + + /// Natural logarithm of gamma function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in ~0.025% of inputs. + /// + /// **See also:** Documentation for [std::lgamma](https://en.cppreference.com/w/cpp/numeric/math/lgamma). + /// \param arg function argument + /// \return natural logarith of gamma function for \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_DIVBYZERO for 0 or negative integer arguments + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half lgamma(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::lgamma(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? 0x7C00 : detail::signal(arg.data_)); + if(!abs || arg.data_ >= 0xE400 || (arg.data_ >= 0xBC00 && !(abs&((1<<(25-(abs>>10)))-1)))) + return half(detail::binary, detail::pole()); + if(arg.data_ == 0x3C00 || arg.data_ == 0x4000) + return half(detail::binary, 0); + return half(detail::binary, detail::gamma(arg.data_)); + #endif + } + + /// Gamma function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.25% of inputs. + /// + /// **See also:** Documentation for [std::tgamma](https://en.cppreference.com/w/cpp/numeric/math/tgamma). + /// \param arg function argument + /// \return gamma function value of \a arg + /// \exception FE_INVALID for signaling NaN, negative infinity or negative integer arguments + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tgamma(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::tgamma(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF; + if(!abs) + return half(detail::binary, detail::pole(arg.data_)); + if(abs >= 0x7C00) + return (arg.data_==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + if(arg.data_ >= 0xE400 || (arg.data_ >= 0xBC00 && !(abs&((1<<(25-(abs>>10)))-1)))) + return half(detail::binary, detail::invalid()); + if(arg.data_ >= 0xCA80) + return half(detail::binary, detail::underflow((1-((abs>>(25-(abs>>10)))&1))<<15)); + if(arg.data_ <= 0x100 || (arg.data_ >= 0x4900 && arg.data_ < 0x8000)) + return half(detail::binary, detail::overflow()); + if(arg.data_ == 0x3C00) + return arg; + return half(detail::binary, detail::gamma(arg.data_)); + #endif + } + + /// \} + /// \anchor rounding + /// \name Rounding + /// \{ + + /// Nearest integer not less than half value. + /// **See also:** Documentation for [std::ceil](https://en.cppreference.com/w/cpp/numeric/math/ceil). + /// \param arg half to round + /// \return nearest integer not less than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half ceil(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer not greater than half value. + /// **See also:** Documentation for [std::floor](https://en.cppreference.com/w/cpp/numeric/math/floor). + /// \param arg half to round + /// \return nearest integer not greater than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half floor(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer not greater in magnitude than half value. + /// **See also:** Documentation for [std::trunc](https://en.cppreference.com/w/cpp/numeric/math/trunc). + /// \param arg half to round + /// \return nearest integer not greater in magnitude than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half trunc(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer. + /// **See also:** Documentation for [std::round](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half round(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer. + /// **See also:** Documentation for [std::lround](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID if value is not representable as `long` + inline long lround(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::rint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half rint(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::lrint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID if value is not representable as `long` + /// \exception FE_INEXACT if value had to be rounded + inline long lrint(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::nearbyint](https://en.cppreference.com/w/cpp/numeric/math/nearbyint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID for signaling NaN + inline half nearbyint(half arg) { return half(detail::binary, detail::integral(arg.data_)); } +#if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer. + /// **See also:** Documentation for [std::llround](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID if value is not representable as `long long` + inline long long llround(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::llrint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID if value is not representable as `long long` + /// \exception FE_INEXACT if value had to be rounded + inline long long llrint(half arg) { return detail::half2int(arg.data_); } +#endif + + /// \} + /// \anchor float + /// \name Floating point manipulation + /// \{ + + /// Decompress floating-point number. + /// **See also:** Documentation for [std::frexp](https://en.cppreference.com/w/cpp/numeric/math/frexp). + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return significant in range [0.5, 1) + /// \exception FE_INVALID for signaling NaN + inline half frexp(half arg, int *exp) + { + *exp = 0; + unsigned int abs = arg.data_ & 0x7FFF; + if(abs >= 0x7C00 || !abs) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--*exp) ; + *exp += (abs>>10) - 14; + return half(detail::binary, (arg.data_&0x8000)|0x3800|(abs&0x3FF)); + } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::scalbln](https://en.cppreference.com/w/cpp/numeric/math/scalbn). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half scalbln(half arg, long exp) + { + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(abs >= 0x7C00 || !abs) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + if(exp > 30) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -10) + return half(detail::binary, detail::underflow(sign)); + else if(exp > 0) + return half(detail::binary, sign|(exp<<10)|(abs&0x3FF)); + unsigned int m = (abs&0x3FF) | 0x400; + return half(detail::binary, detail::rounded(sign|(m>>(1-exp)), (m>>-exp)&1, (m&((1<<-exp)-1))!=0)); + } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::scalbn](https://en.cppreference.com/w/cpp/numeric/math/scalbn). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half scalbn(half arg, int exp) { return scalbln(arg, exp); } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::ldexp](https://en.cppreference.com/w/cpp/numeric/math/ldexp). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half ldexp(half arg, int exp) { return scalbln(arg, exp); } + + /// Extract integer and fractional parts. + /// **See also:** Documentation for [std::modf](https://en.cppreference.com/w/cpp/numeric/math/modf). + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + /// \exception FE_INVALID for signaling NaN + inline half modf(half arg, half *iptr) + { + unsigned int abs = arg.data_ & 0x7FFF; + if(abs > 0x7C00) + { + arg = half(detail::binary, detail::signal(arg.data_)); + return *iptr = arg, arg; + } + if(abs >= 0x6400) + return *iptr = arg, half(detail::binary, arg.data_&0x8000); + if(abs < 0x3C00) + return iptr->data_ = arg.data_ & 0x8000, arg; + unsigned int exp = abs >> 10, mask = (1<<(25-exp)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if(!m) + return half(detail::binary, arg.data_&0x8000); + for(; m<0x400; m<<=1,--exp) ; + return half(detail::binary, (arg.data_&0x8000)|(exp<<10)|(m&0x3FF)); + } + + /// Extract exponent. + /// **See also:** Documentation for [std::ilogb](https://en.cppreference.com/w/cpp/numeric/math/ilogb). + /// \param arg number to query + /// \return floating-point exponent + /// \retval FP_ILOGB0 for zero + /// \retval FP_ILOGBNAN for NaN + /// \retval INT_MAX for infinity + /// \exception FE_INVALID for 0 or infinite values + inline int ilogb(half arg) + { + int abs = arg.data_ & 0x7FFF, exp; + if(!abs || abs >= 0x7C00) + { + detail::raise(FE_INVALID); + return !abs ? FP_ILOGB0 : (abs==0x7C00) ? INT_MAX : FP_ILOGBNAN; + } + for(exp=(abs>>10)-15; abs<0x200; abs<<=1,--exp) ; + return exp; + } + + /// Extract exponent. + /// **See also:** Documentation for [std::logb](https://en.cppreference.com/w/cpp/numeric/math/logb). + /// \param arg number to query + /// \return floating-point exponent + /// \exception FE_INVALID for signaling NaN + /// \exception FE_DIVBYZERO for 0 + inline half logb(half arg) + { + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? 0x7C00 : detail::signal(arg.data_)); + for(exp=(abs>>10)-15; abs<0x200; abs<<=1,--exp) ; + unsigned int value = static_cast(exp<0) << 15; + if(exp) + { + unsigned int m = std::abs(exp) << 6; + for(exp=18; m<0x400; m<<=1,--exp) ; + value |= (exp<<10) + m; + } + return half(detail::binary, value); + } + + /// Next representable value. + /// **See also:** Documentation for [std::nextafter](https://en.cppreference.com/w/cpp/numeric/math/nextafter). + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW for infinite result from finite argument + /// \exception FE_UNDERFLOW for subnormal result + inline half nextafter(half from, half to) + { + int fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if(fabs > 0x7C00 || tabs > 0x7C00) + return half(detail::binary, detail::signal(from.data_, to.data_)); + if(from.data_ == to.data_ || !(fabs|tabs)) + return to; + if(!fabs) + { + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); + return half(detail::binary, (to.data_&0x8000)+1); + } + unsigned int out = from.data_ + (((from.data_>>15)^static_cast( + (from.data_^(0x8000|(0x8000-(from.data_>>15))))<(to.data_^(0x8000|(0x8000-(to.data_>>15))))))<<1) - 1; + detail::raise(FE_OVERFLOW, fabs<0x7C00 && (out&0x7C00)==0x7C00); + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT && (out&0x7C00)<0x400); + return half(detail::binary, out); + } + + /// Next representable value. + /// **See also:** Documentation for [std::nexttoward](https://en.cppreference.com/w/cpp/numeric/math/nexttoward). + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW for infinite result from finite argument + /// \exception FE_UNDERFLOW for subnormal result + inline half nexttoward(half from, long double to) + { + int fabs = from.data_ & 0x7FFF; + if(fabs > 0x7C00) + return half(detail::binary, detail::signal(from.data_)); + long double lfrom = static_cast(from); + if(detail::builtin_isnan(to) || lfrom == to) + return half(static_cast(to)); + if(!fabs) + { + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); + return half(detail::binary, (static_cast(detail::builtin_signbit(to))<<15)+1); + } + unsigned int out = from.data_ + (((from.data_>>15)^static_cast(lfrom 0x7C00; } + + /// Check if normal number. + /// **See also:** Documentation for [std::isnormal](https://en.cppreference.com/w/cpp/numeric/math/isnormal). + /// \param arg number to check + /// \retval true if normal number + /// \retval false if either subnormal, zero, infinity or NaN + inline HALF_CONSTEXPR bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } + + /// Check sign. + /// **See also:** Documentation for [std::signbit](https://en.cppreference.com/w/cpp/numeric/math/signbit). + /// \param arg number to check + /// \retval true for negative number + /// \retval false for positive number + inline HALF_CONSTEXPR bool signbit(half arg) { return (arg.data_&0x8000) != 0; } + + /// \} + /// \anchor compfunc + /// \name Comparison + /// \{ + + /// Quiet comparison for greater than. + /// **See also:** Documentation for [std::isgreater](https://en.cppreference.com/w/cpp/numeric/math/isgreater). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + inline HALF_CONSTEXPR bool isgreater(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) > ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for greater equal. + /// **See also:** Documentation for [std::isgreaterequal](https://en.cppreference.com/w/cpp/numeric/math/isgreaterequal). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + inline HALF_CONSTEXPR bool isgreaterequal(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) >= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for less than. + /// **See also:** Documentation for [std::isless](https://en.cppreference.com/w/cpp/numeric/math/isless). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + inline HALF_CONSTEXPR bool isless(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) < ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for less equal. + /// **See also:** Documentation for [std::islessequal](https://en.cppreference.com/w/cpp/numeric/math/islessequal). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + inline HALF_CONSTEXPR bool islessequal(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) <= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comarison for less or greater. + /// **See also:** Documentation for [std::islessgreater](https://en.cppreference.com/w/cpp/numeric/math/islessgreater). + /// \param x first operand + /// \param y second operand + /// \retval true if either less or greater + /// \retval false else + inline HALF_CONSTEXPR bool islessgreater(half x, half y) + { + return x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF) && !isnan(x) && !isnan(y); + } + + /// Quiet check if unordered. + /// **See also:** Documentation for [std::isunordered](https://en.cppreference.com/w/cpp/numeric/math/isunordered). + /// \param x first operand + /// \param y second operand + /// \retval true if unordered (one or two NaN operands) + /// \retval false else + inline HALF_CONSTEXPR bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + + /// \} + /// \anchor casting + /// \name Casting + /// \{ + + /// Cast to or from half-precision floating-point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the default rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s returns the argument unmodified. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + /// \exception FE_INVALID if \a T is integer type and result is not representable as \a T + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template T half_cast(U arg) { return detail::half_caster::cast(arg); } + + /// Cast to or from half-precision floating-point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the specified rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s returns the argument unmodified. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam R rounding mode to use. + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + /// \exception FE_INVALID if \a T is integer type and result is not representable as \a T + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template T half_cast(U arg) { return detail::half_caster::cast(arg); } + /// \} + + /// \} + /// \anchor errors + /// \name Error handling + /// \{ + + /// Clear exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::feclearexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feclearexcept). + /// \param excepts OR of exceptions to clear + /// \retval 0 all selected flags cleared successfully + inline int feclearexcept(int excepts) { detail::errflags() &= ~excepts; return 0; } + + /// Test exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fetestexcept](https://en.cppreference.com/w/cpp/numeric/fenv/fetestexcept). + /// \param excepts OR of exceptions to test + /// \return OR of selected exceptions if raised + inline int fetestexcept(int excepts) { return detail::errflags() & excepts; } + + /// Raise exception flags. + /// This raises the specified floating point exceptions and also invokes any additional automatic exception handling as + /// configured with the [HALF_ERRHANDLIG_...](\ref HALF_ERRHANDLING_ERRNO) preprocessor symbols. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::feraiseexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feraiseexcept). + /// \param excepts OR of exceptions to raise + /// \retval 0 all selected exceptions raised successfully + inline int feraiseexcept(int excepts) { detail::errflags() |= excepts; detail::raise(excepts); return 0; } + + /// Save exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fegetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). + /// \param flagp adress to store flag state at + /// \param excepts OR of flags to save + /// \retval 0 for success + inline int fegetexceptflag(int *flagp, int excepts) { *flagp = detail::errflags() & excepts; return 0; } + + /// Restore exception flags. + /// This only copies the specified exception state (including unset flags) without incurring any additional exception handling. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fesetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). + /// \param flagp adress to take flag state from + /// \param excepts OR of flags to restore + /// \retval 0 for success + inline int fesetexceptflag(const int *flagp, int excepts) { detail::errflags() = (detail::errflags()|(*flagp&excepts)) & (*flagp|~excepts); return 0; } + + /// Throw C++ exceptions based on set exception flags. + /// This function manually throws a corresponding C++ exception if one of the specified flags is set, + /// no matter if automatic throwing (via [HALF_ERRHANDLING_THROW_...](\ref HALF_ERRHANDLING_THROW_INVALID)) is enabled or not. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// \param excepts OR of exceptions to test + /// \param msg error message to use for exception description + /// \throw std::domain_error if `FE_INVALID` or `FE_DIVBYZERO` is selected and set + /// \throw std::overflow_error if `FE_OVERFLOW` is selected and set + /// \throw std::underflow_error if `FE_UNDERFLOW` is selected and set + /// \throw std::range_error if `FE_INEXACT` is selected and set + inline void fethrowexcept(int excepts, const char *msg = "") + { + excepts &= detail::errflags(); + if(excepts & (FE_INVALID|FE_DIVBYZERO)) + throw std::domain_error(msg); + if(excepts & FE_OVERFLOW) + throw std::overflow_error(msg); + if(excepts & FE_UNDERFLOW) + throw std::underflow_error(msg); + if(excepts & FE_INEXACT) + throw std::range_error(msg); + } + /// \} +} + + +#undef HALF_UNUSED_NOERR +#undef HALF_CONSTEXPR +#undef HALF_CONSTEXPR_CONST +#undef HALF_CONSTEXPR_NOERR +#undef HALF_NOEXCEPT +#undef HALF_NOTHROW +#undef HALF_THREAD_LOCAL +#undef HALF_TWOS_COMPLEMENT_INT +#ifdef HALF_POP_WARNINGS + #pragma warning(pop) + #undef HALF_POP_WARNINGS +#endif + +#endif diff --git a/src/hmc.cc b/src/hmc.cc index 1b9d54a6..3e9f8627 100644 --- a/src/hmc.cc +++ b/src/hmc.cc @@ -305,8 +305,8 @@ HMCMemorySystem::~HMCMemorySystem() { void HMCMemorySystem::SetClockRatio() { // There are 3 clock domains here, Link (super fast), logic (fast), DRAM // (slow) We assume the logic process 1 flit per logic cycle and since the - // link takes several cycles to process 1 flit, we can deduce logic speed - // according to link speed + // link takes several cycles to process 1 flit (128b), we can deduce logic + // speed according to link speed ps_per_dram_ = 800; // 800 ps int link_cycles_per_flit = 128 / config_.link_width; int logic_speed = config_.link_speed / link_cycles_per_flit; // MHz @@ -340,7 +340,28 @@ bool HMCMemorySystem::WillAcceptTransaction(uint64_t hex_addr, return insertable; } -bool HMCMemorySystem::AddTransaction(uint64_t hex_addr, bool is_write) { +bool HMCMemorySystem::WillAcceptTransaction(uint64_t hex_addr, + bool is_write, bool is_MRS) const { + // We DO NOT USE HMC Model, So Not implement MRS Command of HMC Model + bool insertable = false; + /* + for (auto link_queue = link_req_queues_.begin(); + link_queue != link_req_queues_.end(); link_queue++) { + if ((*link_queue).size() < queue_depth_) { + insertable = true; + break; + } + } + */ + return insertable; +} + +bool HMCMemorySystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) { + // WE DO NOT USE HMC MODEL + return false; +} +bool HMCMemorySystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) { + // WE DO NOT USE HMC MODEL // to be compatible with other protocol we have this interface // when using this intreface the size of each transaction will be block_size HMCReqType req_type; @@ -441,35 +462,7 @@ bool HMCMemorySystem::InsertHMCReq(HMCRequest *req) { } } -void HMCMemorySystem::LogicClockTickPre() { - // I just need to note this somewhere: - // the links of HMC are full duplex, e.g. in full width links, - // each link has 16 input lanes AND 16 output lanes - // therefore it makes no sense to have just 1 layer of xbar - // for both requests and responses. - // so 2 layers just sounds about right - - // return responses to CPU - for (int i = 0; i < links_; i++) { - if (!link_resp_queues_[i].empty()) { - HMCResponse *resp = link_resp_queues_[i].front(); - if (resp->exit_time <= logic_clk_) { - if (resp->type == HMCRespType::RD_RS) { - read_callback_(resp->resp_id); - } else { - write_callback_(resp->resp_id); - } - delete (resp); - link_resp_queues_[i].erase(link_resp_queues_[i].begin()); - } - } - } - return; -} - -void HMCMemorySystem::LogicClockTickPost() { - // This MUST happen after DRAMClockTick - +void HMCMemorySystem::DrainRequests() { // drain quad request queue to vaults for (int i = 0; i < 4; i++) { if (!quad_req_queues_[i].empty() && @@ -486,84 +479,13 @@ void HMCMemorySystem::LogicClockTickPost() { } } - // step xbar - // the decremented values here basically represents the internal BW of - // a xbar, to match the external link, transfering 2 flits are - // usually be needed - for (auto &&i : link_busy_) { - if (i > 0) { - i -= 2; - } - } - + // drain xbar for (auto &&i : quad_busy_) { if (i > 0) { i -= 2; } } - // xbar arbitrate using age/FIFO arbitration - // What is set/updated here: - // - link_busy_, quad_busy_ indicators - // - link req, resp queues, quad req, resp queues - // - age counter - XbarArbitrate(); - return; -} - -void HMCMemorySystem::DRAMClockTick() { - uint64_t look_ahead_cycles = clk_ + config_.mega_tick / 2; - for (size_t i = 0; i < ctrls_.size(); i++) { - // look ahead and return earlier - while (true) { - auto pair = ctrls_[i]->ReturnDoneTrans(look_ahead_cycles); - if (pair.second == 1) { // write - VaultCallback(pair.first); - } else if (pair.second == 0) { // read - VaultCallback(pair.first); - } else { - break; - } - } - } -#ifdef _OPENMP -#pragma omp parallel for schedule(static) -#endif // _OPENMP - for (size_t i = 0; i < ctrls_.size(); i++) { - for (int j = 0; j < config_.mega_tick; j++) { - ctrls_[i]->ClockTick(); - } - } - clk_ += config_.mega_tick; - - if (clk_ % config_.epoch_period == 0) { - PrintEpochStats(); - } - return; -} - -void HMCMemorySystem::ClockTick() { - if (dram_ps_ == logic_ps_) { - LogicClockTickPre(); - DRAMClockTick(); - LogicClockTickPost(); - logic_ps_ += ps_per_logic_; - logic_clk_ += 1; - } else { - DRAMClockTick(); - } - while (logic_ps_ < dram_ps_ + ps_per_dram_) { - LogicClockTickPre(); - LogicClockTickPost(); - logic_ps_ += ps_per_logic_; - logic_clk_ += 1; - } - dram_ps_ += ps_per_dram_; - return; -} - -void HMCMemorySystem::XbarArbitrate() { - // arbitrage based on age / FIFO // drain requests from link to quad buffers std::vector age_queue = BuildAgeQueue(link_age_counter_); while (!age_queue.empty()) { @@ -588,9 +510,34 @@ void HMCMemorySystem::XbarArbitrate() { age_queue.erase(age_queue.begin()); } age_queue.clear(); +} + +void HMCMemorySystem::DrainResponses() { + // Link resp to CPU + for (int i = 0; i < links_; i++) { + if (!link_resp_queues_[i].empty()) { + HMCResponse *resp = link_resp_queues_[i].front(); + if (resp->exit_time <= logic_clk_) { + if (resp->type == HMCRespType::RD_RS) { + read_callback_(resp->resp_id); + } else { + write_callback_(resp->resp_id); + } + delete (resp); + link_resp_queues_[i].erase(link_resp_queues_[i].begin()); + } + } + } + + // drain xbar + for (auto &&i : link_busy_) { + if (i > 0) { + i -= 2; + } + } // drain responses from quad to link buffers - age_queue = BuildAgeQueue(quad_age_counter_); + auto age_queue = BuildAgeQueue(quad_age_counter_); while (!age_queue.empty()) { int src_quad = age_queue.front(); int dest_link = quad_resp_queues_[src_quad].front()->link; @@ -615,6 +562,51 @@ void HMCMemorySystem::XbarArbitrate() { age_queue.clear(); } +void HMCMemorySystem::DRAMClockTick() { + for (size_t i = 0; i < ctrls_.size(); i++) { + // look ahead and return earlier + while (true) { + auto pair = ctrls_[i]->ReturnDoneTrans(clk_); + if (pair.second == 1) { // write + VaultCallback(pair.first); + } else if (pair.second == 0) { // read + VaultCallback(pair.first); + } else { + break; + } + } + } + for (size_t i = 0; i < ctrls_.size(); i++) { + ctrls_[i]->ClockTick(); + } + clk_++; + + if (clk_ % config_.epoch_period == 0) { + PrintEpochStats(); + } + return; +} + +void HMCMemorySystem::ClockTick() { + if (dram_ps_ == logic_ps_) { + DrainResponses(); + DRAMClockTick(); + DrainRequests(); + logic_ps_ += ps_per_logic_; + logic_clk_ += 1; + } else { + DRAMClockTick(); + } + while (logic_ps_ < dram_ps_ + ps_per_dram_) { + DrainResponses(); + DrainRequests(); + logic_ps_ += ps_per_logic_; + logic_clk_ += 1; + } + dram_ps_ += ps_per_dram_; + return; +} + std::vector HMCMemorySystem::BuildAgeQueue(std::vector &age_counter) { // return a vector of indices sorted in decending order // meaning that the oldest age link/quad should be processed first @@ -647,6 +639,12 @@ void HMCMemorySystem::InsertReqToDRAM(HMCRequest *req) { return; } + +std::vector HMCMemorySystem::GetRespData(uint64_t hex_addr) { + std::vector a; + return a; +} + void HMCMemorySystem::VaultCallback(uint64_t req_id) { // we will use hex addr as the req_id and use a multimap to lookup the // requests the vaults cannot directly talk to the CPU so this callback will diff --git a/src/hmc.h b/src/hmc.h index 911fd911..655c63d7 100644 --- a/src/hmc.h +++ b/src/hmc.h @@ -112,20 +112,22 @@ class HMCMemorySystem : public BaseDRAMSystem { // we can unify them as one but then we'll have to convert all the // slow dram time units to faster logic units... void ClockTick() override; - void LogicClockTickPre(); - void LogicClockTickPost(); - void DRAMClockTick(); - // had to have 3 insert interfaces cuz HMC is so different... + // had to have 3 insert interfaces cuz HMC is so different... bool WillAcceptTransaction(uint64_t hex_addr, bool is_write) const override; - bool AddTransaction(uint64_t hex_addr, bool is_write) override; + bool WillAcceptTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) const override; + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) override; + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) override; bool InsertReqToLink(HMCRequest* req, int link); bool InsertHMCReq(HMCRequest* req); - + std::vector GetRespData(uint64_t hex_addr) override; private: uint64_t logic_clk_, ps_per_dram_, ps_per_logic_, logic_ps_, dram_ps_; void SetClockRatio(); + void DRAMClockTick(); + void DrainRequests(); + void DrainResponses(); void InsertReqToDRAM(HMCRequest* req); void VaultCallback(uint64_t req_id); std::vector BuildAgeQueue(std::vector& age_counter); @@ -136,6 +138,9 @@ class HMCMemorySystem : public BaseDRAMSystem { int links_; size_t queue_depth_; + // number of flits xbar can process per logic cycle + const int xbar_bandwidth_ = 2; + // had to use a multimap because the controller callback return hex addr // instead of unique id std::multimap resp_lookup_table_; diff --git a/src/main.cc b/src/main.cc index 9ca252e2..f660c1c9 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,16 +1,25 @@ #include +#include +#include #include "./../ext/headers/args.hxx" #include "cpu.h" +#include "custom_cpu.h" + using namespace dramsim3; int main(int argc, const char **argv) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<" main "< num_cycles_arg(parser, "num_cycles", "Number of cycles to simulate", @@ -21,6 +30,12 @@ int main(int argc, const char **argv) { args::ValueFlag stream_arg( parser, "stream_type", "address stream generator - (random), stream", {'s', "stream"}, ""); + args::ValueFlag custom_gen_arg( + parser, "generator_type", "transaction generator, random(RANDOM), stream (STREAM)", + {'g', "generator"}, ""); + args::ValueFlag kernel_type_arg( + parser, "kernel_type", "NDP kernel typess - EWA, EWM,..", + {'k', "kernel"}, ""); args::ValueFlag trace_file_arg( parser, "trace", "Trace file, setting this option will ignore -s option", @@ -49,24 +64,97 @@ int main(int argc, const char **argv) { std::string output_dir = args::get(output_dir_arg); std::string trace_file = args::get(trace_file_arg); std::string stream_type = args::get(stream_arg); + std::string generator_type = args::get(custom_gen_arg); + std::string kernel_type = args::get(kernel_type_arg); + + if(generator_type != "") { + CUSTOM_CPU *cpu = new CUSTOM_CPU(config_file, output_dir, generator_type); + if(generator_type == "kernel" || generator_type =="KERNEL") { + /* + Generate memory requests for NDP kernel exeuction + Simulation Steps + Before Start Simulation + - Generatation Reference Data + - Generation NDP Instruction + - Generation NDP Data + Start Simulation + - Preloading NDP Data + - Preloading NDP Instruction + - NDP Kernel Execution + - Read Result + End Simulation + - check the result + */ + + std::cout<<"================================"<genNDPConfig(kernel_type); + + // Generatation Reference Data + cpu->genRefData(kernel_type); + + // Generation NDP Instruction + cpu->genNDPInst(kernel_type); + + // Generation NDP Data + cpu->genNDPData(kernel_type); + + // Generation NDP Execution + cpu->genNDPExec(kernel_type); + + // Generate NDP Result Read Request + cpu->genNDPReadResult(kernel_type); + + std::cout<<"================================"<ClockTick(); + if(cpu->simDone()) break; + } + std::cout<<"================================"<checkNDPResult(kernel_type); - CPU *cpu; - if (!trace_file.empty()) { - cpu = new TraceBasedCPU(config_file, output_dir, trace_file); - } else { - if (stream_type == "stream" || stream_type == "s") { - cpu = new StreamCPU(config_file, output_dir); + } + else { + // Generate memory requests randomly or sequentially + for (uint64_t clk = 0; clk < cycles; clk++) { + cpu->ClockTick(); + } + } + + // Check Result + cpu->PrintStats(); + + // delete CPU object + cpu->printResult(); + delete cpu; + } + else { + CPU *cpu; + if (!trace_file.empty()) { + cpu = new TraceBasedCPU(config_file, output_dir, trace_file); } else { - cpu = new RandomCPU(config_file, output_dir); + if (stream_type == "stream" || stream_type == "s") { + cpu = new StreamCPU(config_file, output_dir); + } else { + cpu = new RandomCPU(config_file, output_dir); + } } - } - for (uint64_t clk = 0; clk < cycles; clk++) { - cpu->ClockTick(); - } - cpu->PrintStats(); + for (uint64_t clk = 0; clk < cycles; clk++) { + cpu->ClockTick(); + } + cpu->PrintStats(); - delete cpu; + delete cpu; + } return 0; } diff --git a/src/memory_system.cc b/src/memory_system.cc index 74412bb4..0aedb394 100644 --- a/src/memory_system.cc +++ b/src/memory_system.cc @@ -1,4 +1,7 @@ #include "memory_system.h" +#include +#include +#include namespace dramsim3 { MemorySystem::MemorySystem(const std::string &config_file, @@ -6,6 +9,10 @@ MemorySystem::MemorySystem(const std::string &config_file, std::function read_callback, std::function write_callback) : config_(new Config(config_file, output_dir)) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor"<IsHMC()) { dram_system_ = new HMCMemorySystem(*config_, output_dir, read_callback, @@ -23,9 +30,7 @@ MemorySystem::~MemorySystem() { void MemorySystem::ClockTick() { dram_system_->ClockTick(); } -double MemorySystem::GetTCK() const { return config_->tCK * config_->mega_tick; } - -int MemorySystem::GetMegaTick() const { return config_->mega_tick; } +double MemorySystem::GetTCK() const { return config_->tCK; } int MemorySystem::GetBusBits() const { return config_->bus_width; } @@ -33,6 +38,18 @@ int MemorySystem::GetBurstLength() const { return config_->BL; } int MemorySystem::GetQueueSize() const { return config_->trans_queue_size; } +int MemorySystem::GetRank(uint64_t hex_addr) const { + auto addr = config_->AddressMapping(hex_addr); + return addr.rank; +} + +int MemorySystem::GetNumChannel() const { + std::cout<<__func__<channels<channels; +} + + void MemorySystem::RegisterCallbacks( std::function read_callback, std::function write_callback) { @@ -44,19 +61,46 @@ bool MemorySystem::WillAcceptTransaction(uint64_t hex_addr, return dram_system_->WillAcceptTransaction(hex_addr, is_write); } +bool MemorySystem::WillAcceptTransaction(uint64_t hex_addr, + bool is_write, bool is_mrs) const { + return dram_system_->WillAcceptTransaction(hex_addr, is_write, is_mrs); +} + bool MemorySystem::AddTransaction(uint64_t hex_addr, bool is_write) { - return dram_system_->AddTransaction(hex_addr, is_write); + return dram_system_->AddTransaction(hex_addr, is_write, false); +} + +bool MemorySystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS) { + return dram_system_->AddTransaction(hex_addr, is_write, is_MRS); +} + +bool MemorySystem::AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload) { + return dram_system_->AddTransaction(hex_addr, is_write, is_MRS, payload); } void MemorySystem::PrintStats() const { dram_system_->PrintStats(); } void MemorySystem::ResetStats() { dram_system_->ResetStats(); } +bool MemorySystem::isLRDIMM() { + return config_->is_LRDIMM; +} + +Config* MemorySystem::GetConfig() { + return config_; +} + + MemorySystem* GetMemorySystem(const std::string &config_file, const std::string &output_dir, std::function read_callback, std::function write_callback) { return new MemorySystem(config_file, output_dir, read_callback, write_callback); } + +std::vector MemorySystem::GetRespData(uint64_t hex_addr) { + return dram_system_->GetRespData(hex_addr); +} + } // namespace dramsim3 // This function can be used by autoconf AC_CHECK_LIB since diff --git a/src/memory_system.h b/src/memory_system.h index 39ccd0cb..3f1e3ccd 100644 --- a/src/memory_system.h +++ b/src/memory_system.h @@ -21,15 +21,23 @@ class MemorySystem { void RegisterCallbacks(std::function read_callback, std::function write_callback); double GetTCK() const; - int GetMegaTick() const; int GetBusBits() const; int GetBurstLength() const; int GetQueueSize() const; void PrintStats() const; void ResetStats(); + int GetRank(uint64_t hex_addr) const; + int GetNumChannel() const; bool WillAcceptTransaction(uint64_t hex_addr, bool is_write) const; + bool WillAcceptTransaction(uint64_t hex_addr, bool is_write, bool is_mrs) const; bool AddTransaction(uint64_t hex_addr, bool is_write); + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS); + bool AddTransaction(uint64_t hex_addr, bool is_write, bool is_MRS, std::vector &payload); + std::vector GetRespData(uint64_t hex_addr); + bool isLRDIMM(); + + Config* GetConfig(); private: // These have to be pointers because Gem5 will try to push this object diff --git a/src/ndp_address_table.cc b/src/ndp_address_table.cc new file mode 100644 index 00000000..5aaea1ed --- /dev/null +++ b/src/ndp_address_table.cc @@ -0,0 +1,62 @@ +#include "ndp_address_table.h" + +namespace dramsim3 { + Address_Table::Address_Table(uint64_t channel) : channel(channel) { + /* + Insert New Address and Configuration + */ + ADDR_REGION_START_NDP_DRAM = Address(channel,0,0,0,START_RO_BOTH_NDP_DRAM,0); + ADDR_REGION_START_NDP = Address(channel,0,0,0,START_RO_ONLY_NDP,0); + ADDR_NDP_CONFIG = Address(channel,0,0,0,START_RO_ONLY_NDP,0); + ADDR_NDP_DB_CONFIG = Address(channel,0,0,0,START_RO_ONLY_NDP,0); + ADDR_NDP_RCD_CONFIG = Address(channel,0,2,0,START_RO_ONLY_NDP,0); + ADDR_NDP_INST_MEM = Address(channel,1,0,0,START_RO_ONLY_NDP,0); + ADDR_NDP_DATA_MEM = Address(channel,2,0,0,START_RO_ONLY_NDP,0); + + address_map.insert({"ADDR_REGION_START_NDP_DRAM",ADDR_REGION_START_NDP_DRAM}); + address_map.insert({"ADDR_REGION_START_NDP",ADDR_REGION_START_NDP}); + address_map.insert({"ADDR_NDP_CONFIG",ADDR_NDP_CONFIG}); + address_map.insert({"ADDR_NDP_DB_CONFIG",ADDR_NDP_DB_CONFIG}); + address_map.insert({"ADDR_NDP_RCD_CONFIG",ADDR_NDP_RCD_CONFIG}); + address_map.insert({"ADDR_NDP_INST_MEM",ADDR_NDP_INST_MEM}); + address_map.insert({"ADDR_NDP_DATA_MEM",ADDR_NDP_DATA_MEM}); + } + + Address Address_Table::get_address(std::string address) { + if(address_map.find(address)!=address_map.end()) { + return address_map[address]; + } + else { + std::cerr<<"WRONG ADDRESS"< element: address_map) { + std::cout< +#include +#include "common.h" +#include +#include + +#define START_RK_ONLY_NDP 0 +#define START_BG_ONLY_NDP 0 +#define START_BK_ONLY_NDP 0 +#define START_RO_ONLY_NDP 0x3FFFF +#define NDP_CONFIG_COL 0 +#define START_RO_BOTH_NDP_DRAM 0x20000 + +namespace dramsim3 { +class Address_Table { + public: + Address_Table(uint64_t channel); + void print_table(); + + Address get_address(std::string address); + Address operator[](std::string address); + + private: + // Each DIMM has unique channel index + uint64_t channel; + std::unordered_map address_map; + + // Address Space + Address ADDR_REGION_START_NDP_DRAM; + Address ADDR_REGION_START_NDP; + Address ADDR_NDP_CONFIG; + Address ADDR_NDP_DB_CONFIG; + Address ADDR_NDP_RCD_CONFIG; + Address ADDR_NDP_INST_MEM; + Address ADDR_NDP_DATA_MEM; +}; + +class All_Ch_Addrss_Table { + public: + All_Ch_Addrss_Table(uint64_t num_channel); + // Address get_address(std::string address); + + Address_Table operator[](uint64_t ch_idx); + private: + uint64_t num_channel; + std::vector address_tables; +}; + +} // namespace dramsim3 +#endif diff --git a/src/refresh.cc b/src/refresh.cc index 31cbc9f4..406fd8bb 100644 --- a/src/refresh.cc +++ b/src/refresh.cc @@ -9,6 +9,10 @@ Refresh::Refresh(const Config &config, ChannelState &channel_state) next_rank_(0), next_bg_(0), next_bank_(0) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor"<= 1) { @@ -124,7 +147,9 @@ void SimpleStats::PrintFinalStats() { } if (config_.output_level >= 1) { - std::ofstream txt_out(config_.txt_stats_name, std::ofstream::app); + // HACK: overwrite existing file if this is first channel + auto perm = channel_id_ == 0 ? std::ofstream::out : std::ofstream::app; + std::ofstream txt_out(config_.txt_stats_name, perm); txt_out << GetTextHeader(true); for (const auto& it : print_pairs_) { PrintStatText(txt_out, it.first, it.second, diff --git a/src/simple_stats.h b/src/simple_stats.h index 8ed30971..2e03892c 100644 --- a/src/simple_stats.h +++ b/src/simple_stats.h @@ -30,6 +30,9 @@ class SimpleStats { // add historgram value void AddValue(const std::string name, const int value); + // return per rank background energy + double RankBackgroundEnergy(const int r) const; + // Epoch update void PrintEpochStats(); diff --git a/src/thermal_replay.cc b/src/thermal_replay.cc index de90017e..6832aedf 100644 --- a/src/thermal_replay.cc +++ b/src/thermal_replay.cc @@ -75,7 +75,8 @@ void ThermalReplay::Run() { } } for (int c = 0; c < config_.channels; c++) { - channel_stats_[c].PrintFinalStats(clk, std::cout, std::cout, std::cout); + //channel_stats_[c].PrintFinalStats(clk, std::cout, std::cout, std::cout); + channel_stats_[c].PrintFinalStats(); } thermal_calc_.PrintFinalPT(clk); } @@ -187,7 +188,7 @@ void ThermalReplay::ProcessCMD(Command &cmd, uint64_t clk) { for (int c = 0; c < config_.channels; c++) { // where to print isn't important here what we really need is the // updated stats - channel_stats_[c].PrintEpochStats(clk, std::cout); + channel_stats_[c].PrintEpochStats(); for (int r = 0; r < config_.ranks; r++) { double bg_energy = channel_stats_[c].RankBackgroundEnergy(r); thermal_calc_.UpdateBackgroundEnergy(c, r, bg_energy); diff --git a/src/timing.cc b/src/timing.cc index b96ba480..07153253 100644 --- a/src/timing.cc +++ b/src/timing.cc @@ -10,16 +10,25 @@ Timing::Timing(const Config& config) other_bankgroups_same_rank(static_cast(CommandType::SIZE)), other_ranks(static_cast(CommandType::SIZE)), same_rank(static_cast(CommandType::SIZE)) { + #ifdef MY_DEBUG + std::cout<<"== "<<__func__<<" == "; + std::cout<<"constructor"<(CommandType::READ_PRECHARGE)] = std::vector >{ {CommandType::READ, read_to_read_l}, @@ -159,7 +174,8 @@ Timing::Timing(const Config& config) {CommandType::ACTIVATE, write_to_activate}, {CommandType::REFRESH, write_to_activate}, {CommandType::REFRESH_BANK, write_to_activate}, - {CommandType::SREF_ENTER, write_to_activate}}; + {CommandType::SREF_ENTER, write_to_activate}, + {CommandType::MRS, write_to_activate}}; other_banks_same_bankgroup[static_cast(CommandType::WRITE_PRECHARGE)] = std::vector >{ {CommandType::READ, write_to_read_l}, @@ -206,7 +222,8 @@ Timing::Timing(const Config& config) {CommandType::ACTIVATE, precharge_to_activate}, {CommandType::REFRESH, precharge_to_activate}, {CommandType::REFRESH_BANK, precharge_to_activate}, - {CommandType::SREF_ENTER, precharge_to_activate}}; + {CommandType::SREF_ENTER, precharge_to_activate}, + {CommandType::MRS, precharge_to_activate}}; // for those who need tPPD if (config.IsGDDR() || config.protocol == DRAMProtocol::LPDDR4) { @@ -227,7 +244,8 @@ Timing::Timing(const Config& config) {CommandType::ACTIVATE, refresh_to_activate_bank}, {CommandType::REFRESH, refresh_to_activate_bank}, {CommandType::REFRESH_BANK, refresh_to_activate_bank}, - {CommandType::SREF_ENTER, refresh_to_activate_bank}}; + {CommandType::SREF_ENTER, refresh_to_activate_bank}, + {CommandType::MRS, refresh_to_activate_bank}}; other_banks_same_bankgroup[static_cast(CommandType::REFRESH_BANK)] = std::vector >{ @@ -247,7 +265,8 @@ Timing::Timing(const Config& config) std::vector >{ {CommandType::ACTIVATE, refresh_to_activate}, {CommandType::REFRESH, refresh_to_activate}, - {CommandType::SREF_ENTER, refresh_to_activate}}; + {CommandType::SREF_ENTER, refresh_to_activate}, + {CommandType::MRS, refresh_to_activate}}; // command SREF_ENTER // TODO: add power down commands @@ -261,7 +280,16 @@ Timing::Timing(const Config& config) {CommandType::ACTIVATE, self_refresh_exit}, {CommandType::REFRESH, self_refresh_exit}, {CommandType::REFRESH_BANK, self_refresh_exit}, - {CommandType::SREF_ENTER, self_refresh_exit}}; + {CommandType::SREF_ENTER, self_refresh_exit}, + {CommandType::MRS, self_refresh_exit}}; + + // MRS Command + same_rank[static_cast(CommandType::MRS)] = + std::vector >{ + {CommandType::ACTIVATE, mrs_to_non_mrs}, + {CommandType::REFRESH, mrs_to_non_mrs}, + {CommandType::SREF_ENTER, mrs_to_non_mrs}, + {CommandType::MRS, mrs_to_mrs}}; } } // namespace dramsim3 diff --git a/tests/test_hmcsys.cc b/tests/test_hmcsys.cc index cd2d4df3..09a741d9 100644 --- a/tests/test_hmcsys.cc +++ b/tests/test_hmcsys.cc @@ -27,8 +27,15 @@ TEST_CASE("HMC System Testing", "[dramsim3][hmc]") { } } - // int tRC = config.tRCDRD + config.CL + config.BL; - int tRC = 42; - REQUIRE(clk == tRC); + // For HMC things are complicated, e.g. for a 64B read request and x2 bandwidth xbar + // takes 1 cycle from CPU to Link + // takes 1 cycle from Link to Quad + // takes 1 cycle from Quad to DRAM + // takes xx cycles for DRAM to finish + // takes 1 cycle from DRAM to quad + // takes multiple cycles from quad to CPU + // (depending on packet size, and contention) + int idle_lat = 52; + REQUIRE(clk == idle_lat); } }