admin/golem-v2-tinycbor
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for IEEE 754 half-precision floating point

Browse Source 
The CBOR spec has support for them, but we didn't take them into account
in the CBOR and JSON dumpers nor the JSON parser.

See: https://en.wikipedia.org/wiki/Half-precision_floating-point_format

Signed-off-by: Thiago Macieira <thiago.macieira@intel.com>
This commit is contained in:
Thiago Macieira
2015-09-29 18:50:27 -07:00
parent 78632b3e5c
commit 57bcf4fec4
8 changed files with 111 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1 -1
View File
@@ -101,7 +101,7 @@ lib/libtinycbor.a: $(TINYCBOR_SOURCES:.c=.o) | lib
$(AR) cqs $@ $^
bin/cbordump: $(CBORDUMP_SOURCES:.c=.o) lib/libtinycbor.a | bin
$(CC) -o $@ $(LDFLAGS) $^ $(LDLIBS)
$(CC) -o $@ $(LDFLAGS) $^ $(LDLIBS) -lm
bin/json2cbor: $(JSON2CBOR_SOURCES:.c=.o) lib/libtinycbor.a | bin
$(CC) -o $@ $(LDFLAGS) $(LDFLAGS_CJSON) $^ $(LDLIBS) -lm
src/cborpretty.c
+10 -10
View File
@@ -325,15 +325,22 @@ static CborError value_to_pretty(FILE *out, CborValue *it)
cbor_value_get_float(it, &f);
val = f;
suffix = "f";
if (isnan(f) || isinf(f))
suffix = "";
} else if (false) {
uint16_t f16;
case CborHalfFloatType:
cbor_value_get_half_float(it, &f16);
val = decode_half(f16);
suffix = "f16";
} else {
cbor_value_get_double(it, &val);
suffix = "";
}
int r = fpclassify(val);
if (r == FP_NAN || r == FP_INFINITE)
suffix = "";
uint64_t ival = (uint64_t)fabs(val);
int r;
if (ival == fabs(val)) {
// this double value fits in a 64-bit integer, so show it as such
// (followed by a floating point suffix, to disambiguate)
@@ -346,13 +353,6 @@ static CborError value_to_pretty(FILE *out, CborValue *it)
return CborErrorIO;
break;
}
case CborHalfFloatType: {
uint16_t val;
cbor_value_get_half_float(it, &val);
if (fprintf(out, "__f16(0x%04" PRIu16 ")", val) < 0)
return CborErrorIO;
break;
}
case CborInvalidType:
if (fprintf(out, "invalid") < 0)
src/cbortojson.c
+10 -7
View File
@@ -469,18 +469,24 @@ static CborError value_to_json(FILE *out, CborValue *it, int flags, CborType typ
status->flags = TypeWasNotNative;
cbor_value_get_float(it, &f);
val = f;
} else if (false) {
uint16_t f16;
case CborHalfFloatType:
status->flags = TypeWasNotNative;
cbor_value_get_half_float(it, &f16);
val = decode_half(f16);
} else {
cbor_value_get_double(it, &val);
}
if (isinf(val) || isnan(val)) {
int r = fpclassify(val);
if (r == FP_NAN || r == FP_INFINITE) {
if (fprintf(out, "null") < 0)
return CborErrorIO;
status->flags |= isnan(val) ? NumberWasNaN :
NumberWasInfinite | (val < 0 ? NumberWasNegative : 0);
status->flags |= r == FP_NAN ? NumberWasNaN :
NumberWasInfinite | (val < 0 ? NumberWasNegative : 0);
} else {
uint64_t ival = (uint64_t)fabs(val);
int r;
if ((double)ival == fabs(val)) {
// print as integer so we get the full precision
r = fprintf(out, "%s%" PRIu64, val < 0 ? "-" : "", ival);
@@ -495,9 +501,6 @@ static CborError value_to_json(FILE *out, CborValue *it, int flags, CborType typ
break;
}
case CborHalfFloatType:
return CborErrorUnsupportedType;
case CborInvalidType:
return CborErrorUnknownType;
}
src/compilersupport_p.h
+55
View File
@@ -30,6 +30,7 @@
#endif
#include <assert.h>
#include <float.h>
#include <math.h>
#include <stddef.h>
#include <stdint.h>
@@ -37,6 +38,10 @@
# include <stdbool.h>
#endif
#ifdef __F16C__
# include <immintrin.h>
#endif
#if __STDC_VERSION__ >= 201112L || __cplusplus >= 201103L || __cpp_static_assert >= 200410
# define cbor_static_assert(x) static_assert(x, #x)
#elif !defined(__cplusplus) && defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 406)
@@ -163,5 +168,55 @@ static inline bool add_check_overflow(size_t v1, size_t v2, size_t *r)
#endif
}
static inline unsigned short encode_half(double val)
{
#ifdef __F16C__
return _cvtss_sh(val, 3);
#else
uint64_t v;
memcpy(&v, &val, sizeof(v));
int sign = v >> 63 << 15;
int exp = (v >> 52) & 0x7ff;
int mant = v << 12 >> 12 >> (53-11); // keep only the 11 most significant bits of the mantissa
exp -= 1023;
if (exp == 1024) {
// infinity or NaN
exp = 16;
mant >>= 1;
} else if (exp >= 16) {
// overflow, as largest number
exp = 15;
mant = 1023;
} else if (exp >= -14) {
// regular normal
} else if (exp >= -24) {
// subnormal
mant |= 1024;
mant >>= -(exp + 14);
exp = -15;
} else {
// underflow, make zero
return 0;
}
return sign | ((exp + 15) << 10) | mant;
#endif
}
// this function was copied & adapted from RFC 7049 Appendix D
static inline double decode_half(unsigned short half)
{
#ifdef __F16C__
return _cvtsh_ss(half);
#else
int exp = (half >> 10) & 0x1f;
int mant = half & 0x3ff;
double val;
if (exp == 0) val = ldexp(mant, -24);
else if (exp != 31) val = ldexp(mant + 1024, exp - 25);
else val = mant == 0 ? INFINITY : NAN;
return half & 0x8000 ? -val : val;
#endif
}
#endif // COMPILERSUPPORT_H
tests/parser/tst_parser.cpp
+10 -5
View File
@@ -197,19 +197,23 @@ void addFixedData()
QTest::newRow("simple255") << raw("\xf8\xff") << "simple(255)";
// floating point
QTest::newRow("0f16") << raw("\xf9\0\0") << "__f16(0x0000)";
QTest::newRow("0.f16") << raw("\xf9\0\0") << "0.f16";
QTest::newRow("0.f") << raw("\xfa\0\0\0\0") << "0.f";
QTest::newRow("0.") << raw("\xfb\0\0\0\0\0\0\0\0") << "0.";
QTest::newRow("-1.f16") << raw("\xf9\xbc\x00") << "-1.f16";
QTest::newRow("-1.f") << raw("\xfa\xbf\x80\0\0") << "-1.f";
QTest::newRow("-1.") << raw("\xfb\xbf\xf0\0\0\0\0\0\0") << "-1.";
QTest::newRow("65504.f16") << raw("\xf9\x7b\xff") << "65504.f16";
QTest::newRow("16777215.f") << raw("\xfa\x4b\x7f\xff\xff") << "16777215.f";
QTest::newRow("16777215.") << raw("\xfb\x41\x6f\xff\xff\xe0\0\0\0") << "16777215.";
QTest::newRow("-16777215.f") << raw("\xfa\xcb\x7f\xff\xff") << "-16777215.f";
QTest::newRow("-16777215.") << raw("\xfb\xc1\x6f\xff\xff\xe0\0\0\0") << "-16777215.";
QTest::newRow("0.5f16") << raw("\xf9\x38\0") << "0.5f16";
QTest::newRow("0.5f") << raw("\xfa\x3f\0\0\0") << "0.5f";
QTest::newRow("0.5") << raw("\xfb\x3f\xe0\0\0\0\0\0\0") << "0.5";
QTest::newRow("2.f16^11-1") << raw("\xf9\x67\xff") << "2047.f16";
QTest::newRow("2.f^24-1") << raw("\xfa\x4b\x7f\xff\xff") << "16777215.f";
QTest::newRow("2.^53-1") << raw("\xfb\x43\x3f\xff\xff""\xff\xff\xff\xff") << "9007199254740991.";
QTest::newRow("2.f^64-epsilon") << raw("\xfa\x5f\x7f\xff\xff") << "18446742974197923840.f";
@@ -217,12 +221,13 @@ void addFixedData()
QTest::newRow("2.f^64") << raw("\xfa\x5f\x80\0\0") << "1.8446744073709552e+19f";
QTest::newRow("2.^64") << raw("\xfb\x43\xf0\0\0\0\0\0\0") << "1.8446744073709552e+19";
QTest::newRow("qnan_f") << raw("\xfa\x7f\xc0\0\0") << "nan";
QTest::newRow("qnan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "nan";
QTest::newRow("snan_f") << raw("\xfa\x7f\xc0\0\0") << "nan";
QTest::newRow("snan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "nan";
QTest::newRow("nan_f16") << raw("\xf9\x7e\x00") << "nan";
QTest::newRow("nan_f") << raw("\xfa\x7f\xc0\0\0") << "nan";
QTest::newRow("nan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "nan";
QTest::newRow("-inf_f16") << raw("\xf9\xfc\x00") << "-inf";
QTest::newRow("-inf_f") << raw("\xfa\xff\x80\0\0") << "-inf";
QTest::newRow("-inf") << raw("\xfb\xff\xf0\0\0\0\0\0\0") << "-inf";
QTest::newRow("+inf_f16") << raw("\xf9\x7c\x00") << "inf";
QTest::newRow("+inf_f") << raw("\xfa\x7f\x80\0\0") << "inf";
QTest::newRow("+inf") << raw("\xfb\x7f\xf0\0\0\0\0\0\0") << "inf";
tests/tojson/tst_tojson.cpp
+19 -10
View File
@@ -102,8 +102,10 @@ void addFixedData()
QTest::newRow("true") << raw("\xf5") << "true";
QTest::newRow("null") << raw("\xf6") << "null";
QTest::newRow("0.f16") << raw("\xf9\0\0") << "0";
QTest::newRow("0.f") << raw("\xfa\0\0\0\0") << "0";
QTest::newRow("0.") << raw("\xfb\0\0\0\0\0\0\0\0") << "0";
QTest::newRow("-1.f16") << raw("\xf9\xbc\x00") << "-1";
QTest::newRow("-1.f") << raw("\xfa\xbf\x80\0\0") << "-1";
QTest::newRow("-1.") << raw("\xfb\xbf\xf0\0\0\0\0\0\0") << "-1";
QTest::newRow("16777215.f") << raw("\xfa\x4b\x7f\xff\xff") << "16777215";
@@ -111,6 +113,7 @@ void addFixedData()
QTest::newRow("-16777215.f") << raw("\xfa\xcb\x7f\xff\xff") << "-16777215";
QTest::newRow("-16777215.") << raw("\xfb\xc1\x6f\xff\xff\xe0\0\0\0") << "-16777215";
QTest::newRow("0.5f16") << raw("\xf9\x38\0") << "0.5";
QTest::newRow("0.5f") << raw("\xfa\x3f\0\0\0") << "0.5";
QTest::newRow("0.5") << raw("\xfb\x3f\xe0\0\0\0\0\0\0") << "0.5";
QTest::newRow("2.f^24-1") << raw("\xfa\x4b\x7f\xff\xff") << "16777215";
@@ -121,13 +124,14 @@ void addFixedData()
QTest::newRow("2.^64") << raw("\xfb\x43\xf0\0\0\0\0\0\0") << "1.8446744073709552e+19";
// infinities and NaN are not supported in JSON, they convert to null
QTest::newRow("qnan_f") << raw("\xfa\x7f\xc0\0\0") << "null";
QTest::newRow("qnan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "null";
QTest::newRow("snan_f") << raw("\xfa\x7f\xc0\0\0") << "null";
QTest::newRow("snan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "null";
QTest::newRow("nan_f16") << raw("\xf9\x7e\x00") << "null";
QTest::newRow("nan_f") << raw("\xfa\x7f\xc0\0\0") << "null";
QTest::newRow("nan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "null";
QTest::newRow("-inf_f") << raw("\xfa\xff\x80\0\0") << "null";
QTest::newRow("-inf_f16") << raw("\xf9\xfc\x00") << "null";
QTest::newRow("-inf") << raw("\xfb\xff\xf0\0\0\0\0\0\0") << "null";
QTest::newRow("+inf_f") << raw("\xfa\x7f\x80\0\0") << "null";
QTest::newRow("+inf_f16") << raw("\xf9\x7c\x00") << "null";
QTest::newRow("+inf") << raw("\xfb\x7f\xf0\0\0\0\0\0\0") << "null";
}
@@ -345,12 +349,13 @@ void tst_ToJson::nonStringKeyMaps_data()
QTest::newRow("simple32") << raw("\xf8\x20") << "simple(32)";
QTest::newRow("simple255") << raw("\xf8\xff") << "simple(255)";
QTest::newRow("0f16") << raw("\xf9\0\0") << "__f16(0x0000)";
QTest::newRow("0.f16") << raw("\xf9\0\0") << "0.f16";
QTest::newRow("0.f") << raw("\xfa\0\0\0\0") << "0.f";
QTest::newRow("0.") << raw("\xfb\0\0\0\0\0\0\0\0") << "0.";
QTest::newRow("-1.f16") << raw("\xf9\xbc\x00") << "-1.f16";
QTest::newRow("-1.f") << raw("\xfa\xbf\x80\0\0") << "-1.f";
QTest::newRow("-1.") << raw("\xfb\xbf\xf0\0\0\0\0\0\0") << "-1.";
QTest::newRow("65504.f16") << raw("\xf9\x7b\xff") << "65504.f16";
QTest::newRow("16777215.f") << raw("\xfa\x4b\x7f\xff\xff") << "16777215.f";
QTest::newRow("16777215.") << raw("\xfb\x41\x6f\xff\xff\xe0\0\0\0") << "16777215.";
QTest::newRow("-16777215.f") << raw("\xfa\xcb\x7f\xff\xff") << "-16777215.f";
@@ -373,6 +378,7 @@ void tst_ToJson::nonStringKeyMaps_data()
QTest::newRow("-inf_f16") << raw("\xf9\xfc\x00") << "-inf";
QTest::newRow("-inf_f") << raw("\xfa\xff\x80\0\0") << "-inf";
QTest::newRow("-inf") << raw("\xfb\xff\xf0\0\0\0\0\0\0") << "-inf";
QTest::newRow("+inf_f16") << raw("\xf9\x7c\x00") << "inf";
QTest::newRow("+inf_f") << raw("\xfa\x7f\x80\0\0") << "inf";
QTest::newRow("+inf") << raw("\xfb\x7f\xf0\0\0\0\0\0\0") << "inf";
@@ -560,6 +566,8 @@ void tst_ToJson::metaData_data()
QTest::newRow("emptybytestring") << raw("\x40") << "\"t\":64";
QTest::newRow("bytestring1") << raw("\x41 ") << "\"t\":64";
QTest::newRow("undefined") << raw("\xf7") << "\"t\":247";
QTest::newRow("0.f16") << raw("\xf9\0\0") << "\"t\":249";
QTest::newRow("-1.f16") << raw("\xf9\xbc\x00") << "\"t\":249";
QTest::newRow("0.f") << raw("\xfa\0\0\0\0") << "\"t\":250";
QTest::newRow("-1.f") << raw("\xfa\xbf\x80\0\0") << "\"t\":250";
QTest::newRow("16777215.f") << raw("\xfa\x4b\x7f\xff\xff") << "\"t\":250";
@@ -588,12 +596,13 @@ void tst_ToJson::metaData_data()
QTest::newRow("simple255") << raw("\xf8\xff") << "\"t\":224,\"v\":255";
// infinities and NaN are not supported in JSON, they convert to null
QTest::newRow("qnan_f") << raw("\xfa\x7f\xc0\0\0") << "\"t\":250,\"v\":\"nan\"";
QTest::newRow("qnan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "\"t\":251,\"v\":\"nan\"";
QTest::newRow("snan_f") << raw("\xfa\x7f\xc0\0\0") << "\"t\":250,\"v\":\"nan\"";
QTest::newRow("snan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "\"t\":251,\"v\":\"nan\"";
QTest::newRow("nan_f16") << raw("\xf9\x7e\x00") << "\"t\":249,\"v\":\"nan\"";
QTest::newRow("nan_f") << raw("\xfa\x7f\xc0\0\0") << "\"t\":250,\"v\":\"nan\"";
QTest::newRow("nan") << raw("\xfb\x7f\xf8\0\0\0\0\0\0") << "\"t\":251,\"v\":\"nan\"";
QTest::newRow("-inf_f16") << raw("\xf9\xfc\x00") << "\"t\":249,\"v\":\"-inf\"";
QTest::newRow("-inf_f") << raw("\xfa\xff\x80\0\0") << "\"t\":250,\"v\":\"-inf\"";
QTest::newRow("-inf") << raw("\xfb\xff\xf0\0\0\0\0\0\0") << "\"t\":251,\"v\":\"-inf\"";
QTest::newRow("+inf_f16") << raw("\xf9\x7c\x00") << "\"t\":249,\"v\":\"inf\"";
QTest::newRow("+inf_f") << raw("\xfa\x7f\x80\0\0") << "\"t\":250,\"v\":\"inf\"";
QTest::newRow("+inf") << raw("\xfb\x7f\xf0\0\0\0\0\0\0") << "\"t\":251,\"v\":\"inf\"";
tinycbor.pc.in
+1
View File
@@ -7,4 +7,5 @@ Name: TinyCBOR
Description: A tiny CBOR encoder and decoder library
Version: @version@
Libs: -L${libdir} -ltinycbor
Libs.private: -lm
Cflags: -I${includedir}/tinycbor
tools/json2cbor/json2cbor.c
+5 -13
View File
@@ -270,19 +270,9 @@ CborError decode_json_with_metadata(cJSON *item, CborEncoder *encoder, struct Me
return cbor_encode_undefined(encoder);
case CborHalfFloatType:
fprintf(stderr, "json2cbor: Unimplemented: encoding to half-float. Encoding as single-precision float instead.\n");
md.t = CborFloatType;
// fall through
case CborFloatType:
if (!md.v) {
// we can't get an OOM here because the metadata makes up for space
// (the smallest metadata is "$cbor":{"t":250} (17 bytes)
return cbor_encode_float(encoder, item->valuedouble);
}
// fall through
case CborDoubleType: {
unsigned short half;
double v;
if (!md.v) {
v = item->valuedouble;
@@ -297,9 +287,11 @@ CborError decode_json_with_metadata(cJSON *item, CborEncoder *encoder, struct Me
break;
}
// no OOM possible either, as the shortest metadata is also 17 bytes
// we can't get an OOM here because the metadata makes up for space
// (the smallest metadata is "$cbor":{"t":250} (17 bytes)
return (md.t == CborDoubleType) ? cbor_encode_double(encoder, v) :
cbor_encode_float(encoder, v);
(md.t == CborFloatType) ? cbor_encode_float(encoder, v) :
(half = encode_half(v), cbor_encode_half_float(encoder, &half));
}
default: