Examples with RepeatInputStream com.amazon.ion.util.RepeatInputStream used on opensource projects

Example 6 with RepeatInputStream

use of com.amazon.ion.util.RepeatInputStream in project ion-java by amzn.

the class IonReaderBinaryRawLargeStreamTest method readLargeAnnotatedContainer.

@Test
public void readLargeAnnotatedContainer() throws Exception {
    final Timestamp timestamp = Timestamp.forDay(2000, 1, 1);
    byte[] raw = testData(timestamp);
    ByteArrayOutputStream dataBuilder = new ByteArrayOutputStream();
    // Field name. Conveniently use SID 5 ("version"), which is in the system symbol table.
    dataBuilder.write(0x85);
    // List with length subfield.
    dataBuilder.write(0xBE);
    IonBinary.writeVarUInt(dataBuilder, raw.length);
    dataBuilder.write(raw);
    byte[] data = dataBuilder.toByteArray();
    // 100000 makes the value exceed Integer.MAX_VALUE by an arbitrary amount.
    final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 100000;
    ByteArrayOutputStream header = new ByteArrayOutputStream();
    header.write(BINARY_VERSION_MARKER_1_0);
    long containerLength = (long) data.length * totalNumberOfBatches;
    // Annotation wrapper with length subfield.
    header.write(0xEE);
    IonBinary.writeVarUInt(header, containerLength + 3 + IonBinary.lenVarUInt(containerLength));
    // One byte of annotations.
    header.write(0x81);
    // Conveniently use SID 4 ("name"), which is in the system symbol table.
    header.write(0x84);
    // Struct with length subfield.
    header.write(0xDE);
    // Length
    IonBinary.writeVarUInt(header, containerLength);
    InputStream inputStream = new SequenceInputStream(new ByteArrayInputStream(header.toByteArray()), // This will provide the data 'totalNumberOfBatches' times
    new RepeatInputStream(data, totalNumberOfBatches - 1));
    IonReader reader = IonReaderBuilder.standard().build(inputStream);
    assertEquals(IonType.STRUCT, reader.next());
    String[] annotations = reader.getTypeAnnotations();
    assertEquals(1, annotations.length);
    assertEquals("name", annotations[0]);
    reader.stepIn();
    int batchesRead = 0;
    while (reader.next() != null) {
        assertEquals(IonType.LIST, reader.getType());
        // Materializing on every iteration makes the tests take too long. Do it on every 100.
        boolean materializeValues = (batchesRead % 100) == 0;
        if (materializeValues) {
            assertEquals("version", reader.getFieldName());
            reader.stepIn();
            assertEquals(IonType.STRING, reader.next());
            assertEquals("foo", reader.stringValue());
            assertEquals(IonType.DECIMAL, reader.next());
            assertEquals(BigDecimal.TEN, reader.decimalValue());
            assertEquals(IonType.TIMESTAMP, reader.next());
            assertEquals(timestamp, reader.timestampValue());
            assertNull(reader.next());
            reader.stepOut();
        }
        batchesRead++;
    }
    assertNull(reader.next());
    reader.stepOut();
    assertNull(reader.next());
    assertEquals(totalNumberOfBatches, batchesRead);
}

Example 7 with RepeatInputStream

use of com.amazon.ion.util.RepeatInputStream in project ion-java by amzn.

the class IonReaderBinaryRawLargeStreamTest method readLargeScalarStream.

public void readLargeScalarStream(IonReaderBuilder readerBuilder) throws Exception {
    final Timestamp timestamp = Timestamp.forDay(2000, 1, 1);
    byte[] data = testData(timestamp);
    // The binary reader uses Integer.MIN_VALUE to mean NO_LIMIT for its _local_remaining value, which keeps track
    // of the remaining number of bytes in the current value. Between values at the top level, this should always be
    // NO_LIMIT. No arithmetic should ever be performed on the value when it is set to NO_LIMIT. If bugs exist that
    // violate this, then between top level values _local_remaining will never again be NO_LIMIT, meaning that
    // arithmetic will continue to be performed on it. Eventually, due to integer overflow, the value will roll over
    // into a small enough positive value that the reader will erroneously determine that there are fewer bytes
    // remaining than are needed to complete the current value. The reader will then finish early before reading the
    // entire stream. The bug that prompted this test to be written involved an unconditional subtraction of the
    // current value's length as declared in its header from the current value of _local_remaining within
    // stringValue(), decimalValue(), and timestampValue(). This caused _local_remaining to overflow to a very
    // large value immediately. For every top level value subsequently read, the length of that value would be
    // subtracted from _local_remaining until eventually _local_remaining prematurely reached 0 around the time
    // the stream reached Integer.MAX_VALUE in length.
    // Repeat the batch a sufficient number of times to exceed a total stream length of Integer.MAX_VALUE, plus
    // a few more to make sure batches continue to be read correctly.
    // 7 makes the value exceed Integer.MAX_VALUE by an arbitrary amount.
    final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 7;
    InputStream inputStream = new SequenceInputStream(new ByteArrayInputStream(BINARY_VERSION_MARKER_1_0), // This will provide the data 'totalNumberOfBatches' times
    new RepeatInputStream(data, totalNumberOfBatches - 1));
    IonReader reader = readerBuilder.build(inputStream);
    reader.next();
    assertEquals("foo", reader.stringValue());
    reader.next();
    assertEquals(BigDecimal.TEN, reader.decimalValue());
    reader.next();
    assertEquals(timestamp, reader.timestampValue());
    int batchesRead = 1;
    while (reader.next() != null) {
        assertEquals(IonType.STRING, reader.getType());
        assertEquals(IonType.DECIMAL, reader.next());
        assertEquals(IonType.TIMESTAMP, reader.next());
        batchesRead++;
    }
    assertEquals(totalNumberOfBatches, batchesRead);
}

Example 8 with RepeatInputStream

use of com.amazon.ion.util.RepeatInputStream in project ion-java by amzn.

the class IonReaderBinaryRawLargeStreamTest method skipLargeNestedContainer.

@Test
public void skipLargeNestedContainer() throws Exception {
    final Timestamp timestamp = Timestamp.forDay(2000, 1, 1);
    byte[] data = testData(timestamp);
    // 512 makes the value exceed Integer.MAX_VALUE by an arbitrary amount.
    final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 512;
    final long nestedDataLength = (long) data.length * totalNumberOfBatches;
    ByteArrayOutputStream header = new ByteArrayOutputStream();
    header.write(BINARY_VERSION_MARKER_1_0);
    // S-exp with length subfield.
    header.write(0xCE);
    // Length
    IonBinary.writeVarUInt(header, 1 + IonBinary.lenVarUInt(nestedDataLength) + nestedDataLength);
    // List with length subfield.
    header.write(0xBE);
    IonBinary.writeVarUInt(header, nestedDataLength);
    InputStream inputStream = new SequenceInputStream(new ByteArrayInputStream(header.toByteArray()), new SequenceInputStream(// This will provide the data 'totalNumberOfBatches' times
    new RepeatInputStream(data, totalNumberOfBatches - 1), // The string "bar"
    new ByteArrayInputStream(new byte[] { (byte) 0x83, 'b', 'a', 'r' })));
    IonReader reader = IonReaderBuilder.standard().build(inputStream);
    assertEquals(IonType.SEXP, reader.next());
    reader.stepIn();
    assertEquals(IonType.LIST, reader.next());
    assertNull(reader.next());
    reader.stepOut();
    assertEquals(IonType.STRING, reader.next());
    assertEquals("bar", reader.stringValue());
    assertNull(reader.next());
}

Example 9 with RepeatInputStream

use of com.amazon.ion.util.RepeatInputStream in project ion-java by amzn.

the class IonReaderBinaryRawLargeStreamTest method skipLargeAnnotatedContainer.

@Test
public void skipLargeAnnotatedContainer() throws Exception {
    final Timestamp timestamp = Timestamp.forDay(2000, 1, 1);
    byte[] raw = testData(timestamp);
    ByteArrayOutputStream dataBuilder = new ByteArrayOutputStream();
    // Annotation wrapper with length subfield.
    dataBuilder.write(0xEE);
    IonBinary.writeVarUInt(dataBuilder, 3 + IonBinary.lenVarUInt(raw.length) + raw.length);
    // One byte of annotations.
    dataBuilder.write(0x81);
    // Annotation. Conveniently use SID 5 ("version"), which is in the system symbol table.
    dataBuilder.write(0x85);
    // S-exp with length subfield.
    dataBuilder.write(0xCE);
    IonBinary.writeVarUInt(dataBuilder, raw.length);
    dataBuilder.write(raw);
    byte[] data = dataBuilder.toByteArray();
    // 1000000 makes the value exceed Integer.MAX_VALUE by an arbitrary amount.
    final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 1000000;
    ByteArrayOutputStream header = new ByteArrayOutputStream();
    header.write(BINARY_VERSION_MARKER_1_0);
    long containerLength = (long) data.length * totalNumberOfBatches;
    // Annotation wrapper with length subfield.
    header.write(0xEE);
    IonBinary.writeVarUInt(header, containerLength + 3 + IonBinary.lenVarUInt(containerLength));
    // One byte of annotations.
    header.write(0x81);
    // Conveniently use SID 4 ("name"), which is in the system symbol table.
    header.write(0x84);
    // List with length subfield.
    header.write(0xBE);
    // Length
    IonBinary.writeVarUInt(header, containerLength);
    InputStream inputStream = new SequenceInputStream(new ByteArrayInputStream(header.toByteArray()), // This will provide the data 'totalNumberOfBatches' times
    new RepeatInputStream(data, totalNumberOfBatches - 1));
    IonReader reader = IonReaderBuilder.standard().build(inputStream);
    assertEquals(IonType.LIST, reader.next());
    String[] annotations = reader.getTypeAnnotations();
    assertEquals(1, annotations.length);
    assertEquals("name", annotations[0]);
    assertNull(reader.next());
}