Examples with ByteArraySegment io.pravega.common.util.ByteArraySegment used on opensource projects

Example 36 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DataFrameTests method appendRecords.

private int appendRecords(List<ByteArraySegment> allRecords, DataFrame dataFrame) {
    int fullRecordsAppended = 0;
    boolean filledUpFrame = false;
    for (ByteArraySegment record : allRecords) {
        // Append the first half of the record as one DataFrame Entry.
        // true - this is the first entry for the record.
        dataFrame.startNewEntry(true);
        int firstHalfLength = record.getLength() / 2;
        int bytesAppended = dataFrame.append(record.slice(0, firstHalfLength).getBufferViewReader());
        // false - we did not finish the record.
        dataFrame.endEntry(false);
        if (bytesAppended < firstHalfLength) {
            // We filled out the frame.
            filledUpFrame = true;
            break;
        }
        // Append the second half of the record as one DataFrame Entry.
        // false - this is not the first entry for the record.
        dataFrame.startNewEntry(false);
        int secondHalfLength = record.getLength() - firstHalfLength;
        bytesAppended = dataFrame.append(record.slice(firstHalfLength, secondHalfLength).getBufferViewReader());
        fullRecordsAppended += bytesAppended;
        if (bytesAppended < secondHalfLength) {
            // We filled out the frame.
            // false - we did not finish the record.
            dataFrame.endEntry(false);
            filledUpFrame = true;
            break;
        }
        // true - we finished the record.
        dataFrame.endEntry(true);
        fullRecordsAppended++;
    }
    Assert.assertTrue("We did not fill up the DataFrame. This test may not exercise all of the features of DataFrame.", filledUpFrame);
    return fullRecordsAppended;
}

Example 37 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class StreamSegmentAppendOperationTests method testSerializationV0V1.

/**
 * Tests the following scenarios:
 * - Serialization and deserialization using V1.
 * - Serialization using V1 and deserialization using V0.
 * - Serialization using V0 and deserialization using V1.
 * <p>
 * NOTE: {@link #testSerialization()} validates that {@link StreamSegmentAppendOperation.Serializer} works well
 * with default settings (for normal scenarios). This method validates backwards compatibility.
 */
@Test
public void testSerializationV0V1() throws IOException {
    // Writes V0, reads both V0, V1.
    val defaultSerializer = new StreamSegmentAppendOperation.Serializer();
    // Writes and reads V0 only.
    val serializerV0 = new StreamSegmentAppendOperation.SerializerV0();
    // Writes V1, reads both V0, V1.
    val serializerV1 = new SerializerV1();
    val random = RandomFactory.create();
    val operations = Arrays.asList(createOperation(random), new StreamSegmentAppendOperation(1, new ByteArraySegment(new byte[1]), null));
    for (val baseOp : operations) {
        val baseOpUUIDOnly = new StreamSegmentAppendOperation(baseOp.getStreamSegmentId(), baseOp.getData(), baseOp.getAttributeUpdates() == null ? null : AttributeUpdateCollection.from(baseOp.getAttributeUpdates().getUUIDAttributeUpdates()));
        baseOp.setSequenceNumber(MathHelpers.abs(random.nextLong()));
        baseOpUUIDOnly.setSequenceNumber(baseOp.getSequenceNumber());
        configurePreSerialization(baseOp, random);
        baseOpUUIDOnly.setStreamSegmentOffset(baseOp.getStreamSegmentOffset());
        // 1. Deserialize the V0 serialization using ALL serializers. This simulates an upgrade scenario from old code.
        // This can only do UUIDs.
        val v0Serialized = serializerV0.serialize(baseOpUUIDOnly);
        val v0v0Deserialized = serializerV0.deserialize(v0Serialized);
        OperationComparer.DEFAULT.assertEquals(baseOpUUIDOnly, v0v0Deserialized);
        val v0v1Deserialized = serializerV1.deserialize(v0Serialized);
        OperationComparer.DEFAULT.assertEquals(baseOpUUIDOnly, v0v1Deserialized);
        val v0vdDeserialized = defaultSerializer.deserialize(v0Serialized);
        OperationComparer.DEFAULT.assertEquals(baseOpUUIDOnly, v0vdDeserialized);
        // 2. Deserialize the V1 serializer using V1 and Default serializer. This simulates normal operations.
        val v1Serialized = serializerV1.serialize(baseOp);
        val v1v1Deserialized = serializerV1.deserialize(v1Serialized);
        OperationComparer.DEFAULT.assertEquals(baseOp, v1v1Deserialized);
        val v1vdDeserialized = defaultSerializer.deserialize(v1Serialized);
        OperationComparer.DEFAULT.assertEquals(baseOp, v1vdDeserialized);
        AssertExtensions.assertThrows("V0 should not be able to deserialize V1", () -> serializerV0.deserialize(v1Serialized), ex -> ex instanceof SerializationException);
        // 3. Deserialize the Default serializer using ALL serializers. This simulates normal operations and "rollback".
        val dSerialized = defaultSerializer.serialize(baseOp);
        val vdv0Deserialized = serializerV0.deserialize(dSerialized);
        // V0 doesn't know about Variable IDs.
        OperationComparer.DEFAULT.assertEquals(baseOpUUIDOnly, vdv0Deserialized);
        val vdv1Deserialized = serializerV1.deserialize(dSerialized);
        OperationComparer.DEFAULT.assertEquals(baseOp, vdv1Deserialized);
        val vdvdDeserialized = defaultSerializer.deserialize(dSerialized);
        OperationComparer.DEFAULT.assertEquals(baseOp, vdvdDeserialized);
    }
}

Example 38 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BucketUpdateTests method testBuilderCopiedEntries.

/**
 * Tests the Builder in the presence of copied entries.
 */
@Test
public void testBuilderCopiedEntries() {
    int count = 20;
    val bucket = new TableBucket(UUID.randomUUID(), 0L);
    val builder = BucketUpdate.forBucket(bucket);
    val expectedUpdate = new HashSet<Integer>();
    for (int i = 0; i < count; i++) {
        boolean hasExistingKey = i % 3 == 0;
        boolean isCopy = i % 5 == 0;
        boolean copyWithSmallerVersion = i % 10 == 0;
        if (hasExistingKey) {
            builder.withExistingKey(new BucketUpdate.KeyInfo(new ByteArraySegment(new byte[] { (byte) i }), i + 1, i + 1));
        }
        long updateVersion = isCopy ? (copyWithSmallerVersion ? i : i + 1) : // No copy
        (i + 1) * 10;
        builder.withKeyUpdate(new BucketUpdate.KeyUpdate(new ByteArraySegment(new byte[] { (byte) i }), (i + 1) * 10, updateVersion, false));
        if (hasExistingKey && !copyWithSmallerVersion || (!hasExistingKey && !isCopy)) {
            expectedUpdate.add(i);
        }
    }
    val b = builder.build();
    for (int i = 0; i < count; i++) {
        boolean expectedIsUpdated = expectedUpdate.contains(i);
        boolean isUpdated = b.isKeyUpdated(new ByteArraySegment(new byte[] { (byte) i }));
        Assert.assertEquals("Unexpected update status for key " + i, expectedIsUpdated, isUpdated);
    }
}

Example 39 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BucketUpdateTests method testSuperseding.

/**
 * Tests the {@link BucketUpdate.KeyInfo#supersedes} method and {@link BucketUpdate.Builder#build} with superseded
 * Key updates.
 */
@Test
public void testSuperseding() {
    val bucket = new TableBucket(UUID.randomUUID(), 0L);
    val existingKey = new BucketUpdate.KeyInfo(new ByteArraySegment(new byte[] { (byte) 1 }), 10, 9);
    // This one has the same version but higher offset, so it should supersede it.
    val sameVersionHigherOffsetUpdate = new BucketUpdate.KeyUpdate(new ByteArraySegment(new byte[] { (byte) 1 }), 11, 9, false);
    Assert.assertTrue(sameVersionHigherOffsetUpdate.supersedes(existingKey));
    Assert.assertFalse(existingKey.supersedes(sameVersionHigherOffsetUpdate));
    val bu1 = BucketUpdate.forBucket(bucket).withExistingKey(existingKey).withKeyUpdate(sameVersionHigherOffsetUpdate).build();
    Assert.assertTrue("Expected superseded key to be reported as updated", bu1.isKeyUpdated(existingKey.getKey()));
    Assert.assertEquals("Unexpected updated key that supersedes existing key.", sameVersionHigherOffsetUpdate, bu1.getKeyUpdates().stream().findFirst().orElse(null));
    Assert.assertEquals("Unexpected Bucket Offset for superseding key.", sameVersionHigherOffsetUpdate.getOffset(), bu1.getBucketOffset());
    // This one has a lower version but higher offset.
    val lowerVersionUpdate = new BucketUpdate.KeyUpdate(new ByteArraySegment(new byte[] { (byte) 1 }), 12, 8, false);
    Assert.assertTrue(existingKey.supersedes(lowerVersionUpdate));
    Assert.assertFalse(lowerVersionUpdate.supersedes(existingKey));
    val bu2 = BucketUpdate.forBucket(bucket).withExistingKey(existingKey).withKeyUpdate(lowerVersionUpdate).build();
    Assert.assertFalse("Expected non-superseded key to not be reported as updated", bu2.isKeyUpdated(existingKey.getKey()));
    Assert.assertTrue("Not expecting any updates for non-superseded key.", bu2.getKeyUpdates().isEmpty());
    Assert.assertEquals("Unexpected Bucket Offset for non-superseding key.", existingKey.getOffset(), bu2.getBucketOffset());
}

Example 40 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class BucketUpdateTests method testBuilder.

/**
 * Tests the Builder in the absence of copied entries.
 */
@Test
public void testBuilder() throws Exception {
    int count = 5;
    val bucket = new TableBucket(UUID.randomUUID(), 0L);
    val builder = BucketUpdate.forBucket(bucket);
    val b1 = builder.build();
    Assert.assertEquals("Unexpected bucket.", bucket, b1.getBucket());
    Assert.assertFalse("Not expecting any updates at this time.", b1.hasUpdates());
    for (int i = 0; i < count; i++) {
        builder.withExistingKey(new BucketUpdate.KeyInfo(new ByteArraySegment(new byte[] { (byte) i }), i, i));
        builder.withKeyUpdate(new BucketUpdate.KeyUpdate(new ByteArraySegment(new byte[] { (byte) -i }), i, i, i % 2 == 0));
    }
    val b2 = builder.build();
    Assert.assertTrue("Unexpected result from isKeyUpdated for updated key.", b2.isKeyUpdated(new ByteArraySegment(new byte[] { (byte) -1 })));
    Assert.assertFalse("Unexpected result from isKeyUpdated for non-updated key.", b2.isKeyUpdated(new ByteArraySegment(new byte[] { (byte) -count })));
    Assert.assertEquals("Unexpected existing keys count.", count, b2.getExistingKeys().size());
    Assert.assertEquals("Unexpected updated keys count.", count, b2.getKeyUpdates().size());
    val existingIterator = b2.getExistingKeys().stream().sorted(Comparator.comparingLong(BucketUpdate.KeyInfo::getOffset)).iterator();
    val updatesIterator = b2.getKeyUpdates().stream().sorted(Comparator.comparingLong(BucketUpdate.KeyInfo::getOffset)).iterator();
    for (int i = 0; i < count; i++) {
        val e = existingIterator.next();
        val u = updatesIterator.next();
        Assert.assertEquals("Unexpected key for existing " + i, (byte) i, (byte) e.getKey().getReader().read());
        Assert.assertEquals("Unexpected offset for existing " + i, i, e.getOffset());
        Assert.assertEquals("Unexpected key for update " + i, (byte) -i, (byte) u.getKey().getReader().read());
        Assert.assertEquals("Unexpected offset for update " + i, i, u.getOffset());
        Assert.assertEquals("Unexpected value for isDeleted " + i, i % 2 == 0, u.isDeleted());
    }
}