WIP Initial API for delegated streaming in

Signed-off-by: Thiago Macieira <thiago.macieira@intel.com>

.travis.yml

@@ -75,7 +75,7 @@ script:
   - make -s -f Makefile.configure configure | tee .config
   - make -k
         CFLAGS="$CFLAGS -march=native -g1 -Wall -Wextra -Werror"
-        CPPFLAGS="-DNDEBUG"
+        CPPFLAGS="-DNDEBUG -DCBOR_PARSER_READER_CONTROL=-1"
         lib/libtinycbor.a
   - size lib/libtinycbor.a | tee sizes
   - make -s clean

src/cbor.h

@@ -189,6 +189,7 @@ typedef enum CborError {
     CborErrorDataTooLarge = 1024,
     CborErrorNestingTooDeep,
     CborErrorUnsupportedType,
+    CborErrorUnimplementedValidation,
 
     /* errors in converting to JSON */
     CborErrorJsonObjectKeyIsAggregate = 1280,
@@ -267,6 +268,11 @@ CBOR_INLINE_API size_t cbor_encoder_get_extra_bytes_needed(const CborEncoder *en
 
 /* Parser API */
 
+enum CborParserGlobalFlags
+{
+    CborParserFlag_ExternalSource           = 0x01
+};
+
 enum CborParserIteratorFlags
 {
     CborIteratorFlag_IntegerValueIs64Bit    = 0x01,
@@ -278,17 +284,32 @@ enum CborParserIteratorFlags
     CborIteratorFlag_NextIsMapKey           = 0x40
 };
 
+struct CborValue;
+struct CborParserOperations
+{
+    bool (*can_read_bytes)(void *token, size_t len);
+    void *(*read_bytes)(void *token, void *dst, size_t offset, size_t len);
+    void (*advance_bytes)(void *token, size_t len);
+    CborError (*transfer_string)(void *token, const void **userptr, size_t offset, size_t len);
+};
+
 struct CborParser
 {
-    const uint8_t *end;
-    uint32_t flags;
+    union {
+        const uint8_t *end;
+        const struct CborParserOperations *ops;
+    } source;
+    enum CborParserGlobalFlags flags;
 };
 typedef struct CborParser CborParser;
 
 struct CborValue
 {
     const CborParser *parser;
-    const uint8_t *ptr;
+    union {
+        const uint8_t *ptr;
+        void *token;
+    } source;
     uint32_t remaining;
     uint16_t extra;
     uint8_t type;
@@ -298,13 +319,14 @@ typedef struct CborValue CborValue;
 
 #ifndef CBOR_NO_PARSER_API
 CBOR_API CborError cbor_parser_init(const uint8_t *buffer, size_t size, uint32_t flags, CborParser *parser, CborValue *it);
+CBOR_API CborError cbor_parser_init_reader(const struct CborParserOperations *ops, CborParser *parser, CborValue *it, void *token);
 
 CBOR_API CborError cbor_value_validate_basic(const CborValue *it);
 
 CBOR_INLINE_API bool cbor_value_at_end(const CborValue *it)
 { return it->remaining == 0; }
 CBOR_INLINE_API const uint8_t *cbor_value_get_next_byte(const CborValue *it)
-{ return it->ptr; }
+{ return it->source.ptr; }
 CBOR_API CborError cbor_value_advance_fixed(CborValue *it);
 CBOR_API CborError cbor_value_advance(CborValue *it);
 CBOR_INLINE_API bool cbor_value_is_container(const CborValue *it)

src/cborerrorstrings.c

@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 Intel Corporation
+** Copyright (C) 2021 Intel Corporation
 **
 ** Permission is hereby granted, free of charge, to any person obtaining a copy
 ** of this software and associated documentation files (the "Software"), to deal
@@ -169,6 +169,9 @@ const char *cbor_error_string(CborError error)
     case CborErrorUnsupportedType:
         return _("unsupported type");
 
+    case CborErrorUnimplementedValidation:
+        return _("validation not implemented for the current parser state");
+
     case CborErrorJsonObjectKeyIsAggregate:
         return _("conversion to JSON failed: key in object is an array or map");
 

src/cborinternal_p.h

@@ -106,6 +106,9 @@ static inline double decode_half(unsigned short half)
 #  define CBOR_PARSER_MAX_RECURSIONS 1024
 #endif
 
+#ifndef CBOR_PARSER_READER_CONTROL
+#  define CBOR_PARSER_READER_CONTROL    0
+#endif
 /*
  * CBOR Major types
  * Encoded in the high 3 bits of the descriptor byte
@@ -158,25 +161,82 @@ CBOR_INTERNAL_API CborError CBOR_INTERNAL_API_CC _cbor_value_prepare_string_iter
 
 static inline void copy_current_position(CborValue *dst, const CborValue *src)
 {
-    dst->ptr = src->ptr;
+    /* This "if" is here for pedantry only: the two branches should perform
+     * the same memory operation. */
+    if (src->parser->flags & CborParserFlag_ExternalSource)
+        dst->source.token = src->source.token;
+    else
+        dst->source.ptr = src->source.ptr;
 }
 
 static inline bool can_read_bytes(const CborValue *it, size_t n)
 {
+    if (CBOR_PARSER_READER_CONTROL >= 0) {
+        if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
+#ifdef CBOR_PARSER_CAN_READ_BYTES_FUNCTION
+            return CBOR_PARSER_CAN_READ_BYTES_FUNCTION(it->source.token, n);
+#else
+            return it->parser->source.ops->can_read_bytes(it->source.token, n);
+#endif
+        }
+    }
+
     /* Convert the pointer subtraction to size_t since end >= ptr
      * (this prevents issues with (ptrdiff_t)n becoming negative).
      */
-    return (size_t)(it->parser->end - it->ptr) >= n;
+    return (size_t)(it->parser->source.end - it->source.ptr) >= n;
 }
 
 static inline void advance_bytes(CborValue *it, size_t n)
 {
-    it->ptr += n;
+    if (CBOR_PARSER_READER_CONTROL >= 0) {
+        if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
+#ifdef CBOR_PARSER_ADVANCE_BYTES_FUNCTION
+            CBOR_PARSER_ADVANCE_BYTES_FUNCTION(it->source.token, n);
+#else
+            it->parser->source.ops->advance_bytes(it->source.token, n);
+#endif
+            return;
+        }
+    }
+
+    it->source.ptr += n;
+}
+
+static inline CborError transfer_string(CborValue *it, const void **ptr, size_t offset, size_t len)
+{
+    if (CBOR_PARSER_READER_CONTROL >= 0) {
+        if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
+#ifdef CBOR_PARSER_TRANSFER_STRING_FUNCTION
+            return CBOR_PARSER_TRANSFER_STRING_FUNCTION(it->source.token, ptr, offset, len);
+#else
+            return it->parser->source.ops->transfer_string(it->source.token, ptr, offset, len);
+#endif
+        }
+    }
+
+    it->source.ptr += offset;
+    if (can_read_bytes(it, len)) {
+        *CONST_CAST(const void **, ptr) = it->source.ptr;
+        it->source.ptr += len;
+        return CborNoError;
+    }
+    return CborErrorUnexpectedEOF;
 }
 
 static inline void *read_bytes_unchecked(const CborValue *it, void *dst, size_t offset, size_t n)
 {
-    return memcpy(dst, it->ptr + offset, n);
+    if (CBOR_PARSER_READER_CONTROL >= 0) {
+        if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) {
+#ifdef CBOR_PARSER_READ_BYTES_FUNCTION
+            return CBOR_PARSER_READ_BYTES_FUNCTION(it->source.token, dst, offset, n);
+#else
+            return it->parser->source.ops->read_bytes(it->source.token, dst, offset, n);
+#endif
+        }
+    }
+
+    return memcpy(dst, it->source.ptr + offset, n);
 }
 
 #ifdef __GNUC__

src/cborparser.c

@@ -343,15 +343,26 @@ uint64_t _cbor_value_decode_int64_internal(const CborValue *value)
 CborError cbor_parser_init(const uint8_t *buffer, size_t size, uint32_t flags, CborParser *parser, CborValue *it)
 {
     memset(parser, 0, sizeof(*parser));
-    parser->end = buffer + size;
-    parser->flags = flags;
+    parser->source.end = buffer + size;
+    parser->flags = (enum CborParserGlobalFlags)flags;
     it->parser = parser;
-    it->ptr = buffer;
+    it->source.ptr = buffer;
     it->remaining = 1;      /* there's one type altogether, usually an array or map */
     it->flags = 0;
     return preparse_value(it);
 }
 
+CborError cbor_parser_init_reader(const struct CborParserOperations *ops, CborParser *parser, CborValue *it, void *token)
+{
+    memset(parser, 0, sizeof(*parser));
+    parser->source.ops = ops;
+    parser->flags = CborParserFlag_ExternalSource;
+    it->parser = parser;
+    it->source.token = token;
+    it->remaining = 1;
+    return preparse_value(it);
+}
+
 /**
  * \fn bool cbor_value_at_end(const CborValue *it)
  *
@@ -382,6 +393,11 @@ CborError cbor_parser_init(const uint8_t *buffer, size_t size, uint32_t flags, C
  * Note that the error recovery is not precise and the pointer may not indicate
  * the exact byte containing bad data.
  *
+ * This function makes sense only when using a linear buffer (that is, when the
+ * parser is initialize by cbor_parser_init()). If using an external source,
+ * this function may return garbage; instead, consult the external source itself
+ * to find out more details about the presence of more data.
+ *
  * \sa cbor_value_at_end()
  */
 
@@ -1029,17 +1045,12 @@ last_chunk:
             ++bytesNeeded;
         }
 
-        *bufferptr = it->ptr + bytesNeeded;
         if (*len != (size_t)*len)
             return CborErrorDataTooLarge;
 
-        if (add_check_overflow(bytesNeeded, *len, &bytesNeeded))
-            return CborErrorUnexpectedEOF;
-
-        if (!can_read_bytes(it, bytesNeeded))
-            return CborErrorUnexpectedEOF;
-
-        advance_bytes(it, bytesNeeded);
+        CborError err = transfer_string(it, bufferptr, bytesNeeded, *len);
+        if (err)
+            return err;
     } else {
         return CborErrorIllegalType;
     }

src/cborvalidation.c

@@ -471,6 +471,8 @@ static CborError validate_container(CborValue *it, int containerType, uint32_t f
             continue;
 
         if (flags & CborValidateMapIsSorted) {
+            if (it->parser->flags & CborParserFlag_ExternalSource)
+                return CborErrorUnimplementedValidation;
             if (previous) {
                 size_t bytelen1 = (size_t)(previous_end - previous);
                 size_t bytelen2 = (size_t)(cbor_value_get_next_byte(it) - current);

tests/parser/tst_parser.cpp

@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2019 Intel Corporation
+** Copyright (C) 2021 Intel Corporation
 **
 ** Permission is hereby granted, free of charge, to any person obtaining a copy
 ** of this software and associated documentation files (the "Software"), to deal
@@ -85,6 +85,9 @@ private slots:
     void mapsAndArrays_data() { arrays_data(); }
     void mapsAndArrays();
 
+    void readerApi_data() { arrays_data(); }
+    void readerApi();
+
     // chunked string API
     void chunkedString_data();
     void chunkedString();
@@ -1038,6 +1041,54 @@ void tst_Parser::mapsAndArrays()
                    "{_ 1: [_ " + expected + "], \"Hello\": {_ " + expected + ": (_ )}}");
 }
 
+void tst_Parser::readerApi()
+{
+    QFETCH(QByteArray, data);
+    QFETCH(QString, expected);
+
+    struct Input {
+        QByteArray data;
+        int consumed;
+    } input = { data, 0 };
+
+    CborParserOperations ops;
+    ops.can_read_bytes = [](void *token, size_t len) {
+        auto input = static_cast<Input *>(token);
+        return input->data.size() - input->consumed >= int(len);
+    };
+    ops.read_bytes = [](void *token, void *dst, size_t offset, size_t len) {
+        auto input = static_cast<Input *>(token);
+        return memcpy(dst, input->data.constData() + input->consumed + offset, len);
+    };
+    ops.advance_bytes = [](void *token, size_t len) {
+        auto input = static_cast<Input *>(token);
+        input->consumed += int(len);
+    };
+    ops.transfer_string = [](void *token, const void **userptr, size_t offset, size_t len) {
+        // ###
+        auto input = static_cast<Input *>(token);
+        if (input->data.size() - input->consumed < int(len + offset))
+            return CborErrorUnexpectedEOF;
+        input->consumed += int(offset);
+        *userptr = input->data.constData() + input->consumed;
+        input->consumed += int(len);
+        return CborNoError;
+    };
+
+    CborParser parser;
+    CborValue first;
+    CborError err = cbor_parser_init_reader(&ops, &parser, &first, &input);
+    QCOMPARE(err, CborNoError);
+
+    QString decoded;
+    err = parseOne(&first, &decoded);
+    QCOMPARE(err, CborNoError);
+    QCOMPARE(decoded, expected);
+
+    // check we consumed everything
+    QCOMPARE(input.consumed, data.size());
+}
+
 void tst_Parser::chunkedString_data()
 {
     QTest::addColumn<QByteArray>("data");