Examples with Slice com.datatorrent.netlet.util.Slice used on opensource projects

Example 61 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class AbstractBlockReaderTest method testCountersTransfer.

@Test
public void testCountersTransfer() throws Exception {
    TestUtils.MockBatchedOperatorStats readerStats = new TestUtils.MockBatchedOperatorStats(2);
    readerStats.operatorStats = Lists.newArrayList();
    readerStats.operatorStats.add(new ReaderStats(10, 1, 100, 1));
    TestReader sliceReader = new TestReader();
    sliceReader.processStats(readerStats);
    List<Partitioner.Partition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>>> partitions = Lists.newArrayList();
    DefaultPartition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>> apartition = new DefaultPartition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>>(sliceReader);
    TestUtils.MockPartition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>> pseudoParttion = new TestUtils.MockPartition<>(apartition, readerStats);
    partitions.add(pseudoParttion);
    Collection<Partitioner.Partition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>>> newPartitions = sliceReader.definePartitions(partitions, null);
    List<Partitioner.Partition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>>> newMocks = Lists.newArrayList();
    for (Partitioner.Partition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>> partition : newPartitions) {
        partition.getPartitionedInstance().counters.setCounter(AbstractBlockReader.ReaderCounterKeys.BLOCKS, new MutableLong(1));
        newMocks.add(new TestUtils.MockPartition<>((DefaultPartition<AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream>>) partition, readerStats));
    }
    sliceReader.partitionCount = 1;
    newPartitions = sliceReader.definePartitions(newMocks, null);
    Assert.assertEquals(1, newPartitions.size());
    AbstractBlockReader<Slice, BlockMetadata.FileBlockMetadata, FSDataInputStream> last = newPartitions.iterator().next().getPartitionedInstance();
    Assert.assertEquals("num blocks", 8, last.counters.getCounter(AbstractBlockReader.ReaderCounterKeys.BLOCKS).longValue());
}

Example 62 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class JavaSerializationStreamCodecTest method testFinalFieldSerialization.

@Test
public void testFinalFieldSerialization() throws Exception {
    TestTuple t1 = new TestTuple(5);
    JavaSerializationStreamCodec<Serializable> c = new JavaSerializationStreamCodec<Serializable>();
    Slice dsp = c.toByteArray(t1);
    TestTuple t2 = (TestTuple) c.fromByteArray(dsp);
    Assert.assertEquals("", t1.finalField, t2.finalField);
}

Example 63 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class KryoStreamCodecTest method testFinalFieldSerialization.

@Test
public void testFinalFieldSerialization() throws Exception {
    TestTuple t1 = new TestTuple(5);
    TestKryoStreamCodec codec = new TestKryoStreamCodec();
    Slice dsp = codec.toByteArray(t1);
    TestTuple t2 = (TestTuple) codec.fromByteArray(dsp);
    Assert.assertEquals("", t1.field, t2.field);
}

Example 64 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class BufferSlice method equals.

/**
 * let this class equals with com.datatorrent.netlet.util.Slice
 */
@Override
public boolean equals(Object obj) {
    if (obj == null) {
        return false;
    }
    if (!Slice.class.isAssignableFrom(obj.getClass())) {
        return false;
    }
    final Slice other = (Slice) obj;
    if (this.length != other.length) {
        return false;
    }
    final int offset1 = this.offset;
    final byte[] buffer1 = this.buffer;
    int i = offset1 + this.length;
    final byte[] buffer2 = other.buffer;
    int j = other.offset + other.length;
    while (i-- > offset1) {
        if (buffer1[i] != buffer2[--j]) {
            return false;
        }
    }
    return true;
}

Example 65 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class FSWindowDataManager method save.

/**
 * Save writes 2 entries to the wal: <br/>
 * <ol>
 *   <li>window id</li>
 *   <li>artifact</li>
 * </ol>
 * Note: The wal is being used in batch mode so the part file will never be rotated between the 2 entries.<br/>
 * The wal part file may be rotated after both the entries, when
 * {@link FileSystemWAL.FileSystemWALWriter#rotateIfNecessary()} is triggered.
 *
 * @param object    state
 * @param windowId  window id
 * @throws IOException
 */
@Override
public void save(Object object, long windowId) throws IOException {
    closeReaders();
    FileSystemWAL.FileSystemWALWriter writer = wal.getWriter();
    byte[] windowIdBytes = Longs.toByteArray(windowId);
    writer.append(new Slice(windowIdBytes));
    /**
     * writer.append() will copy the data to the file output stream.
     * So the data in the buffer is not needed any more, and it is safe to reset the serializationBuffer.
     *
     * And as the data in stream memory can be cleaned all at once. So don't need to separate data by different windows,
     * so beginWindow() and endWindow() don't need to be called
     */
    writer.append(toSlice(object));
    serializationBuffer.reset();
    wal.beforeCheckpoint(windowId);
    wal.windowWalParts.put(windowId, writer.getCurrentPointer().getPartNum());
    writer.rotateIfNecessary();
}