Examples with ByteBufferInputStream com.github.ambry.utils.ByteBufferInputStream used on opensource projects

Example 76 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class LogTest method appendErrorCasesTest.

/**
 * Tests cases where bad arguments are provided to the append operations.
 * @throws IOException
 * @throws StoreException
 */
@Test
public void appendErrorCasesTest() throws IOException, StoreException {
    Log log = new Log(tempDir.getAbsolutePath(), LOG_CAPACITY, StoreTestUtils.DEFAULT_DISK_SPACE_ALLOCATOR, createStoreConfig(SEGMENT_CAPACITY, setFilePermissionEnabled), metrics, null);
    try {
        // write exceeds size of a single segment.
        ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes((int) (SEGMENT_CAPACITY + 1 - LogSegment.HEADER_SIZE)));
        try {
            log.appendFrom(buffer);
            fail("Cannot append a write of size greater than log segment size");
        } catch (IllegalArgumentException e) {
            assertEquals("Position of buffer has changed", 0, buffer.position());
        }
        try {
            log.appendFrom(Channels.newChannel(new ByteBufferInputStream(buffer)), buffer.remaining());
            fail("Cannot append a write of size greater than log segment size");
        } catch (IllegalArgumentException e) {
            assertEquals("Position of buffer has changed", 0, buffer.position());
        }
    } finally {
        log.close(false);
        cleanDirectory(tempDir);
    }
}

Example 77 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class StoreFindTokenTest method doSerDeTest.

// serDeTest() helpers
/**
 * Serializes {@code token} in all formats and ensures that the {@link StoreFindToken} obtained from the
 * deserialization matches the original.
 * @param token the {@link StoreFindToken} that has to be serialized/deserialized.
 * @param versions {@link List} of valid versions that the token to be tested for
 * @throws IOException
 */
private void doSerDeTest(StoreFindToken token, Short... versions) throws IOException {
    for (Short version : versions) {
        DataInputStream stream = getSerializedStream(token, version);
        StoreFindToken deSerToken = StoreFindToken.fromBytes(stream, STORE_KEY_FACTORY);
        assertEquals("Stream should have ended ", 0, stream.available());
        assertEquals("Version mismatch for token ", version.shortValue(), deSerToken.getVersion());
        compareTokens(token, deSerToken);
        assertEquals("SessionId does not match", token.getSessionId(), deSerToken.getSessionId());
        if (version >= VERSION_2) {
            assertEquals("IncarnationId mismatch ", token.getIncarnationId(), deSerToken.getIncarnationId());
        }
        if (version == VERSION_3) {
            assertEquals("Reset key mismatch ", token.getResetKey(), deSerToken.getResetKey());
            assertEquals("Reset key type mismatch", token.getResetKeyType(), deSerToken.getResetKeyType());
            assertEquals("Reset key life version mismatch", token.getResetKeyVersion(), deSerToken.getResetKeyVersion());
        }
        // use StoreFindToken's actual serialize method to verify that token is serialized in the expected
        // version
        stream = new DataInputStream(new ByteBufferInputStream(ByteBuffer.wrap(deSerToken.toBytes())));
        deSerToken = StoreFindToken.fromBytes(stream, STORE_KEY_FACTORY);
        assertEquals("Stream should have ended ", 0, stream.available());
        assertEquals("Version mismatch for token ", version.shortValue(), deSerToken.getVersion());
        compareTokens(token, deSerToken);
        assertEquals("SessionId does not match", token.getSessionId(), deSerToken.getSessionId());
        if (version >= VERSION_2) {
            assertEquals("IncarnationId mismatch ", token.getIncarnationId(), deSerToken.getIncarnationId());
        }
    }
}

Example 78 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class StoreMessageReadSetTest method storeMessageReadSetTest.

/**
 * Primarily tests {@link StoreMessageReadSet} and its APIs but also checks the the {@link Comparable} APIs of
 * {@link BlobReadOptions}.
 * @throws IOException
 */
@Test
public void storeMessageReadSetTest() throws IOException, StoreException {
    int logCapacity = 2000;
    int segCapacity = 1000;
    Log log = new Log(tempDir.getAbsolutePath(), logCapacity, StoreTestUtils.DEFAULT_DISK_SPACE_ALLOCATOR, createStoreConfig(segCapacity, setFilePermissionEnabled), metrics, null);
    try {
        LogSegment firstSegment = log.getFirstSegment();
        int availableSegCapacity = (int) (segCapacity - firstSegment.getStartOffset());
        byte[] srcOfTruth = TestUtils.getRandomBytes(2 * availableSegCapacity);
        ReadableByteChannel dataChannel = Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(srcOfTruth)));
        log.appendFrom(dataChannel, availableSegCapacity);
        log.appendFrom(dataChannel, availableSegCapacity);
        LogSegment secondSegment = log.getNextSegment(firstSegment);
        Offset firstSegOffset1 = new Offset(firstSegment.getName(), firstSegment.getStartOffset());
        Offset firstSegOffset2 = new Offset(firstSegment.getName(), firstSegment.getStartOffset() + availableSegCapacity / 2);
        Offset secondSegOffset1 = new Offset(secondSegment.getName(), secondSegment.getStartOffset());
        Offset secondSegOffset2 = new Offset(secondSegment.getName(), secondSegment.getStartOffset() + availableSegCapacity / 2);
        List<MockId> mockIdList = new ArrayList<>();
        MockId mockId = new MockId("id1");
        mockIdList.add(mockId);
        BlobReadOptions ro1 = new BlobReadOptions(log, firstSegOffset2, new MessageInfo(mockId, availableSegCapacity / 3, 1, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), System.currentTimeMillis() + TestUtils.RANDOM.nextInt(10000)));
        mockId = new MockId("id2");
        mockIdList.add(mockId);
        BlobReadOptions ro2 = new BlobReadOptions(log, secondSegOffset1, new MessageInfo(mockId, availableSegCapacity / 4, 1, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), System.currentTimeMillis() + TestUtils.RANDOM.nextInt(10000)));
        mockId = new MockId("id3");
        mockIdList.add(mockId);
        BlobReadOptions ro3 = new BlobReadOptions(log, secondSegOffset2, new MessageInfo(mockId, availableSegCapacity / 2, 1, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), System.currentTimeMillis() + TestUtils.RANDOM.nextInt(10000)));
        mockId = new MockId("id4");
        mockIdList.add(mockId);
        BlobReadOptions ro4 = new BlobReadOptions(log, firstSegOffset1, new MessageInfo(mockId, availableSegCapacity / 5, 1, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), System.currentTimeMillis() + TestUtils.RANDOM.nextInt(10000)));
        // to test equality in the compareTo() of BlobReadOptions
        mockId = new MockId("id5");
        mockIdList.add(mockId);
        BlobReadOptions ro5 = new BlobReadOptions(log, firstSegOffset2, new MessageInfo(mockId, availableSegCapacity / 6, 1, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), System.currentTimeMillis() + TestUtils.RANDOM.nextInt(10000)));
        List<BlobReadOptions> options = new ArrayList<>(Arrays.asList(ro1, ro2, ro3, ro4, ro5));
        MessageReadSet readSet = new StoreMessageReadSet(options);
        assertEquals(readSet.count(), options.size());
        // options should get sorted by offsets in the constructor
        assertEquals(readSet.getKeyAt(0), mockIdList.get(3));
        assertEquals(readSet.sizeInBytes(0), availableSegCapacity / 5);
        assertEquals(readSet.getKeyAt(1), mockIdList.get(0));
        assertEquals(readSet.sizeInBytes(1), availableSegCapacity / 3);
        assertEquals(readSet.getKeyAt(2), mockIdList.get(4));
        assertEquals(readSet.sizeInBytes(2), availableSegCapacity / 6);
        assertEquals(readSet.getKeyAt(3), mockIdList.get(1));
        assertEquals(readSet.sizeInBytes(3), availableSegCapacity / 4);
        assertEquals(readSet.getKeyAt(4), mockIdList.get(2));
        assertEquals(readSet.sizeInBytes(4), availableSegCapacity / 2);
        ByteBuffer readBuf = ByteBuffer.allocate(availableSegCapacity / 5);
        ByteBufferOutputStream stream = new ByteBufferOutputStream(readBuf);
        // read the first one all at once
        if (doDataPrefetch) {
            readSet.doPrefetch(0, 0, Long.MAX_VALUE);
        }
        long written = readSet.writeTo(0, Channels.newChannel(stream), 0, Long.MAX_VALUE);
        assertEquals("Return value from writeTo() is incorrect", availableSegCapacity / 5, written);
        assertArrayEquals(readBuf.array(), Arrays.copyOfRange(srcOfTruth, 0, availableSegCapacity / 5));
        // read the second one byte by byte
        readBuf = ByteBuffer.allocate(availableSegCapacity / 3);
        stream = new ByteBufferOutputStream(readBuf);
        WritableByteChannel channel = Channels.newChannel(stream);
        long currentReadOffset = 0;
        if (doDataPrefetch) {
            readSet.doPrefetch(1, currentReadOffset, availableSegCapacity / 3);
        }
        while (currentReadOffset < availableSegCapacity / 3) {
            written = readSet.writeTo(1, channel, currentReadOffset, 1);
            assertEquals("Return value from writeTo() is incorrect", 1, written);
            currentReadOffset++;
        }
        long startOffset = availableSegCapacity / 2;
        long endOffset = availableSegCapacity / 2 + availableSegCapacity / 3;
        assertArrayEquals(readBuf.array(), Arrays.copyOfRange(srcOfTruth, (int) startOffset, (int) endOffset));
        // read the last one in multiple stages
        readBuf = ByteBuffer.allocate(availableSegCapacity / 2);
        stream = new ByteBufferOutputStream(readBuf);
        channel = Channels.newChannel(stream);
        currentReadOffset = 0;
        if (doDataPrefetch) {
            readSet.doPrefetch(4, currentReadOffset, availableSegCapacity / 2);
        }
        while (currentReadOffset < availableSegCapacity / 2) {
            written = readSet.writeTo(4, channel, currentReadOffset, availableSegCapacity / 6);
            long expectedWritten = Math.min(availableSegCapacity / 2 - currentReadOffset, availableSegCapacity / 6);
            assertEquals("Return value from writeTo() is incorrect", expectedWritten, written);
            currentReadOffset += availableSegCapacity / 6;
        }
        startOffset = availableSegCapacity + availableSegCapacity / 2;
        endOffset = startOffset + availableSegCapacity / 2;
        assertArrayEquals(readBuf.array(), Arrays.copyOfRange(srcOfTruth, (int) startOffset, (int) endOffset));
        // should not write anything if relative offset is at the size
        readBuf = ByteBuffer.allocate(1);
        stream = new ByteBufferOutputStream(readBuf);
        channel = Channels.newChannel(stream);
        written = readSet.writeTo(0, channel, readSet.sizeInBytes(0), 1);
        assertEquals("No data should have been written", 0, written);
        try {
            readSet.sizeInBytes(options.size());
            fail("Reading should have failed because index is out of bounds");
        } catch (IndexOutOfBoundsException e) {
        // expected. Nothing to do.
        }
        try {
            readSet.writeTo(options.size(), channel, 100, 100);
            fail("Reading should have failed because index is out of bounds");
        } catch (IndexOutOfBoundsException e) {
        // expected. Nothing to do.
        }
        try {
            readSet.getKeyAt(options.size());
            fail("Getting key should have failed because index is out of bounds");
        } catch (IndexOutOfBoundsException e) {
        // expected. Nothing to do.
        }
    } finally {
        log.close(false);
    }
}

Example 79 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class DirectoryUploader method writeToAmbryReplica.

/**
 * Adding a blob to a particular server.
 * @param stream stream containing the actual blob
 * @param blobId blob id generated by ambry
 * @param replicaId replica to which the blob has to be added
 * @param correlationId coorelation id to uniquely identify the call
 * @return true if write succeeds, else false
 * @throws Exception
 */
private boolean writeToAmbryReplica(BlobProperties blobProperties, ByteBuffer userMetaData, InputStream stream, BlobId blobId, ReplicaId replicaId, AtomicInteger correlationId, boolean enableVerboseLogging) throws Exception {
    ArrayList<BlobId> blobIds = new ArrayList<BlobId>();
    blobIds.add(blobId);
    ConnectedChannel blockingChannel = null;
    try {
        blockingChannel = connectionPool.checkOutConnection(replicaId.getDataNodeId().getHostname(), new Port(replicaId.getDataNodeId().getPort(), PortType.PLAINTEXT), 100000);
        int size = (int) blobProperties.getBlobSize();
        ByteBufferInputStream blobStream = new ByteBufferInputStream(stream, size);
        PutRequest putRequest = new PutRequest(correlationId.incrementAndGet(), "consumerThread", blobId, blobProperties, userMetaData, Unpooled.wrappedBuffer(blobStream.getByteBuffer()), size, BlobType.DataBlob, null);
        if (enableVerboseLogging) {
            System.out.println("Put Request to a replica : " + putRequest + " for blobId " + blobId);
        }
        PutResponse putResponse = getPutResponseFromStream(blockingChannel, putRequest, connectionPool);
        if (putResponse == null) {
            System.out.println("PutResponse to a replica " + replicaId + " failed with null Response for blobId " + blobId);
            blockingChannel = null;
            return false;
        } else {
            if (putResponse.getError() != ServerErrorCode.No_Error && putResponse.getError() != ServerErrorCode.Blob_Already_Exists) {
                System.out.println("PutResponse to a replica " + replicaId + " failed with Error code  " + putResponse.getError() + " for blobId " + blobId);
                return false;
            }
            return true;
        }
    } finally {
        if (stream != null) {
            stream.close();
        }
        if (blockingChannel != null) {
            connectionPool.checkInConnection(blockingChannel);
        }
    }
}