Examples with BlobData com.github.ambry.messageformat.BlobData used on opensource projects

Example 6 with BlobData

use of com.github.ambry.messageformat.BlobData in project ambry by linkedin.

the class ServerTestUtil method endToEndReplicationWithMultiNodeMultiPartitionTest.

static void endToEndReplicationWithMultiNodeMultiPartitionTest(int interestedDataNodePortNumber, Port dataNode1Port, Port dataNode2Port, Port dataNode3Port, MockCluster cluster, SSLConfig clientSSLConfig1, SSLConfig clientSSLConfig2, SSLConfig clientSSLConfig3, SSLSocketFactory clientSSLSocketFactory1, SSLSocketFactory clientSSLSocketFactory2, SSLSocketFactory clientSSLSocketFactory3, MockNotificationSystem notificationSystem, boolean testEncryption) throws Exception {
    // interestedDataNodePortNumber is used to locate the datanode and hence has to be PlainTextPort
    MockClusterMap clusterMap = cluster.getClusterMap();
    BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
    List<AmbryServer> serverList = cluster.getServers();
    byte[] usermetadata = new byte[100];
    byte[] data = new byte[100];
    byte[] encryptionKey = null;
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties properties = new BlobProperties(100, "serviceid1", null, null, false, TestUtils.TTL_SECS, cluster.time.milliseconds(), accountId, containerId, false, null, null, null);
    long expectedExpiryTimeMs = getExpiryTimeMs(properties);
    TestUtils.RANDOM.nextBytes(usermetadata);
    TestUtils.RANDOM.nextBytes(data);
    if (testEncryption) {
        encryptionKey = new byte[100];
        TestUtils.RANDOM.nextBytes(encryptionKey);
    }
    // connect to all the servers
    ConnectedChannel channel1 = getBlockingChannelBasedOnPortType(dataNode1Port, "localhost", clientSSLSocketFactory1, clientSSLConfig1);
    ConnectedChannel channel2 = getBlockingChannelBasedOnPortType(dataNode2Port, "localhost", clientSSLSocketFactory2, clientSSLConfig2);
    ConnectedChannel channel3 = getBlockingChannelBasedOnPortType(dataNode3Port, "localhost", clientSSLSocketFactory3, clientSSLConfig3);
    // put all the blobs to random servers
    channel1.connect();
    channel2.connect();
    channel3.connect();
    int noOfParallelThreads = 3;
    int totalBlobsToPut = 50;
    CountDownLatch latch = new CountDownLatch(noOfParallelThreads);
    List<DirectSender> runnables = new ArrayList<DirectSender>(noOfParallelThreads);
    ConnectedChannel channel = null;
    for (int i = 0; i < noOfParallelThreads; i++) {
        if (i % noOfParallelThreads == 0) {
            channel = channel1;
        } else if (i % noOfParallelThreads == 1) {
            channel = channel2;
        } else if (i % noOfParallelThreads == 2) {
            channel = channel3;
        }
        DirectSender runnable = new DirectSender(cluster, channel, totalBlobsToPut, data, usermetadata, properties, encryptionKey, latch);
        runnables.add(runnable);
        Thread threadToRun = new Thread(runnable);
        threadToRun.start();
    }
    assertTrue("Did not put all blobs in 2 minutes", latch.await(2, TimeUnit.MINUTES));
    // wait till replication can complete
    List<BlobId> blobIds = new ArrayList<BlobId>();
    for (int i = 0; i < runnables.size(); i++) {
        blobIds.addAll(runnables.get(i).getBlobIds());
    }
    for (BlobId blobId : blobIds) {
        notificationSystem.awaitBlobCreations(blobId.getID());
    }
    // Now that the blob is created and replicated, test the cases where a put request arrives for the same blob id
    // later than replication.
    testLatePutRequest(blobIds.get(0), properties, usermetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.No_Error);
    // Test the case where a put arrives with the same id as one in the server, but the blob is not identical.
    BlobProperties differentProperties = new BlobProperties(properties.getBlobSize(), properties.getServiceId(), accountId, containerId, testEncryption, cluster.time.milliseconds());
    testLatePutRequest(blobIds.get(0), differentProperties, usermetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
    byte[] differentUserMetadata = Arrays.copyOf(usermetadata, usermetadata.length);
    differentUserMetadata[0] = (byte) ~differentUserMetadata[0];
    testLatePutRequest(blobIds.get(0), properties, differentUserMetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
    byte[] differentData = Arrays.copyOf(data, data.length);
    differentData[0] = (byte) ~differentData[0];
    testLatePutRequest(blobIds.get(0), properties, usermetadata, differentData, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
    // verify blob properties, metadata and blob across all nodes
    for (int i = 0; i < 3; i++) {
        channel = null;
        if (i == 0) {
            channel = channel1;
        } else if (i == 1) {
            channel = channel2;
        } else if (i == 2) {
            channel = channel3;
        }
        ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        for (int j = 0; j < blobIds.size(); j++) {
            ArrayList<BlobId> ids = new ArrayList<BlobId>();
            ids.add(blobIds.get(j));
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
            DataInputStream stream = channel.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            try {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                assertEquals(100, propertyOutput.getBlobSize());
                assertEquals("serviceid1", propertyOutput.getServiceId());
                assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
                assertEquals("Expiration time mismatch (props)", expectedExpiryTimeMs, getExpiryTimeMs(propertyOutput));
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                fail();
            }
            // get user metadata
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
            stream = channel.sendAndReceive(getRequest).getInputStream();
            resp = GetResponse.readFrom(stream, clusterMap);
            try {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                assertArrayEquals(usermetadata, userMetadataOutput.array());
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                e.printStackTrace();
                fail();
            }
            releaseNettyBufUnderneathStream(stream);
            // get blob
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
            stream = channel.sendAndReceive(getRequest).getInputStream();
            resp = GetResponse.readFrom(stream, clusterMap);
            try {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                byte[] blobout = getBlobDataAndRelease(blobData);
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                e.printStackTrace();
                fail();
            }
            releaseNettyBufUnderneathStream(stream);
            // get blob all
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
            stream = channel.sendAndReceive(getRequest).getInputStream();
            resp = GetResponse.readFrom(stream, clusterMap);
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                assertEquals("Expiration time mismatch (props)", expectedExpiryTimeMs, getExpiryTimeMs(blobAll.getBlobInfo().getBlobProperties()));
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                e.printStackTrace();
                fail();
            }
        }
    }
    // ttl update all blobs and wait for replication
    Map<ConnectedChannel, List<BlobId>> channelToBlobIds = new HashMap<>();
    for (int i = 0; i < blobIds.size(); i++) {
        final BlobId blobId = blobIds.get(i);
        if (i % 3 == 0) {
            channelToBlobIds.computeIfAbsent(channel1, updateChannel -> new ArrayList<>()).add(blobId);
        } else if (i % 3 == 1) {
            channelToBlobIds.computeIfAbsent(channel2, updateChannel -> new ArrayList<>()).add(blobId);
        } else {
            channelToBlobIds.computeIfAbsent(channel3, updateChannel -> new ArrayList<>()).add(blobId);
        }
    }
    channelToBlobIds.entrySet().stream().map(entry -> CompletableFuture.supplyAsync(() -> {
        try {
            for (BlobId blobId : entry.getValue()) {
                updateBlobTtl(entry.getKey(), blobId, cluster.time.milliseconds());
            }
            return null;
        } catch (Throwable e) {
            throw new RuntimeException("Exception updating ttl for: " + entry, e);
        }
    })).forEach(CompletableFuture::join);
    // check that the TTL update has propagated
    blobIds.forEach(blobId -> notificationSystem.awaitBlobUpdates(blobId.getID(), UpdateType.TTL_UPDATE));
    // check all servers
    for (ConnectedChannel channelToUse : new ConnectedChannel[] { channel1, channel2, channel3 }) {
        for (BlobId blobId : blobIds) {
            checkTtlUpdateStatus(channelToUse, clusterMap, blobIdFactory, blobId, data, true, Utils.Infinite_Time);
        }
    }
    // delete random blobs, wait for replication and ensure it is deleted in all nodes
    Set<BlobId> blobsDeleted = new HashSet<BlobId>();
    Set<BlobId> blobsChecked = new HashSet<BlobId>();
    for (int i = 0; i < blobIds.size(); i++) {
        int j = new Random().nextInt(3);
        if (j == 0) {
            j = new Random().nextInt(3);
            if (j == 0) {
                channel = channel1;
            } else if (j == 1) {
                channel = channel2;
            } else if (j == 2) {
                channel = channel3;
            }
            DeleteRequest deleteRequest = new DeleteRequest(1, "reptest", blobIds.get(i), System.currentTimeMillis());
            DataInputStream deleteResponseStream = channel.sendAndReceive(deleteRequest).getInputStream();
            DeleteResponse deleteResponse = DeleteResponse.readFrom(deleteResponseStream);
            releaseNettyBufUnderneathStream(deleteResponseStream);
            assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
            blobsDeleted.add(blobIds.get(i));
        }
    }
    Iterator<BlobId> iterator = blobsDeleted.iterator();
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    while (iterator.hasNext()) {
        BlobId deletedId = iterator.next();
        notificationSystem.awaitBlobDeletions(deletedId.getID());
        for (int j = 0; j < 3; j++) {
            if (j == 0) {
                channel = channel1;
            } else if (j == 1) {
                channel = channel2;
            } else if (j == 2) {
                channel = channel3;
            }
            ArrayList<BlobId> ids = new ArrayList<BlobId>();
            ids.add(deletedId);
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(deletedId.getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
            DataInputStream stream = channel.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            assertEquals(ServerErrorCode.Blob_Deleted, resp.getPartitionResponseInfoList().get(0).getErrorCode());
            releaseNettyBufUnderneathStream(stream);
        }
    }
    // take a server down, clean up a mount path, start and ensure replication fixes it
    serverList.get(0).shutdown();
    serverList.get(0).awaitShutdown();
    MockDataNodeId dataNode = (MockDataNodeId) clusterMap.getDataNodeId("localhost", interestedDataNodePortNumber);
    System.out.println("Cleaning mount path " + dataNode.getMountPaths().get(0));
    for (ReplicaId replicaId : clusterMap.getReplicaIds(dataNode)) {
        if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(0)) == 0) {
            System.out.println("Cleaning partition " + replicaId.getPartitionId());
        }
    }
    deleteFolderContent(new File(dataNode.getMountPaths().get(0)), false);
    for (int i = 0; i < blobIds.size(); i++) {
        for (ReplicaId replicaId : blobIds.get(i).getPartition().getReplicaIds()) {
            if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(0)) == 0) {
                if (blobsDeleted.contains(blobIds.get(i))) {
                    notificationSystem.decrementDeletedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
                } else {
                    notificationSystem.decrementCreatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
                    notificationSystem.decrementUpdatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort(), UpdateType.TTL_UPDATE);
                }
            }
        }
    }
    cluster.reinitServer(0);
    channel1.disconnect();
    channel1.connect();
    for (int j = 0; j < blobIds.size(); j++) {
        if (blobsDeleted.contains(blobIds.get(j))) {
            notificationSystem.awaitBlobDeletions(blobIds.get(j).getID());
        } else {
            notificationSystem.awaitBlobCreations(blobIds.get(j).getID());
            notificationSystem.awaitBlobUpdates(blobIds.get(j).getID(), UpdateType.TTL_UPDATE);
        }
        ArrayList<BlobId> ids = new ArrayList<BlobId>();
        ids.add(blobIds.get(j));
        partitionRequestInfoList.clear();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
        GetResponse resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
        } else {
            try {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                assertEquals(100, propertyOutput.getBlobSize());
                assertEquals("serviceid1", propertyOutput.getServiceId());
                assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get user metadata
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
        } else {
            try {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                assertArrayEquals(usermetadata, userMetadataOutput.array());
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
        } else {
            try {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                byte[] blobout = getBlobDataAndRelease(blobData);
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob all
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
            blobsDeleted.remove(blobIds.get(j));
            blobsChecked.add(blobIds.get(j));
        } else {
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
    }
    assertEquals(0, blobsDeleted.size());
    // take a server down, clean all contents, start and ensure replication fixes it
    serverList.get(0).shutdown();
    serverList.get(0).awaitShutdown();
    dataNode = (MockDataNodeId) clusterMap.getDataNodeId("localhost", interestedDataNodePortNumber);
    for (int i = 0; i < dataNode.getMountPaths().size(); i++) {
        System.out.println("Cleaning mount path " + dataNode.getMountPaths().get(i));
        for (ReplicaId replicaId : clusterMap.getReplicaIds(dataNode)) {
            if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(i)) == 0) {
                System.out.println("Cleaning partition " + replicaId.getPartitionId());
            }
        }
        deleteFolderContent(new File(dataNode.getMountPaths().get(i)), false);
    }
    for (int i = 0; i < blobIds.size(); i++) {
        if (blobsChecked.contains(blobIds.get(i))) {
            notificationSystem.decrementDeletedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
        } else {
            notificationSystem.decrementCreatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
        }
    }
    cluster.reinitServer(0);
    channel1.disconnect();
    channel1.connect();
    for (int j = 0; j < blobIds.size(); j++) {
        if (blobsChecked.contains(blobIds.get(j))) {
            notificationSystem.awaitBlobDeletions(blobIds.get(j).getID());
        } else {
            notificationSystem.awaitBlobCreations(blobIds.get(j).getID());
        }
        ArrayList<BlobId> ids = new ArrayList<BlobId>();
        ids.add(blobIds.get(j));
        partitionRequestInfoList.clear();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
        GetResponse resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
        } else {
            try {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                assertEquals(100, propertyOutput.getBlobSize());
                assertEquals("serviceid1", propertyOutput.getServiceId());
                assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get user metadata
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
        } else {
            try {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                assertArrayEquals(usermetadata, userMetadataOutput.array());
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
        } else {
            try {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                byte[] blobout = getBlobDataAndRelease(blobData);
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob all
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
            blobsChecked.remove(blobIds.get(j));
            blobsDeleted.add(blobIds.get(j));
        } else {
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
    }
    assertEquals(0, blobsChecked.size());
    short expectedLifeVersion = 1;
    for (int i = 0; i < 2; i++) {
        expectedLifeVersion += i;
        // First undelete all deleted blobs
        for (BlobId deletedId : blobsDeleted) {
            UndeleteRequest undeleteRequest = new UndeleteRequest(2, "reptest", deletedId, System.currentTimeMillis());
            DataInputStream undeleteResponseStream = channel3.sendAndReceive(undeleteRequest).getInputStream();
            UndeleteResponse undeleteResponse = UndeleteResponse.readFrom(undeleteResponseStream);
            releaseNettyBufUnderneathStream(undeleteResponseStream);
            assertEquals(ServerErrorCode.No_Error, undeleteResponse.getError());
            assertEquals(expectedLifeVersion, undeleteResponse.getLifeVersion());
        }
        Thread.sleep(5000);
        // Then use get request to get all the data back and make sure the lifeVersion is correct
        for (BlobId id : blobsDeleted) {
            // We don't need to wait for blob undeletes, since one of the hosts has Put Record deleted
            // from disk, so undelete this blob would end up replicating Put Record instead of undelete.
            // notificationSystem.awaitBlobUndeletes(id.toString());
            ArrayList<BlobId> ids = new ArrayList<>();
            ids.add(id);
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(id.getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            // get blob all
            GetRequest getRequest = new GetRequest(1, "clientid20", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
            DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            assertEquals(ServerErrorCode.No_Error, resp.getError());
            assertEquals(1, resp.getPartitionResponseInfoList().size());
            assertEquals(ServerErrorCode.No_Error, resp.getPartitionResponseInfoList().get(0).getErrorCode());
            assertEquals(1, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().size());
            MessageInfo info = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
            assertEquals(expectedLifeVersion, info.getLifeVersion());
            assertFalse(info.isDeleted());
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                fail();
            }
        }
        for (BlobId id : blobsDeleted) {
            DeleteRequest deleteRequest = new DeleteRequest(1, "reptest", id, System.currentTimeMillis());
            DataInputStream deleteResponseStream = channel.sendAndReceive(deleteRequest).getInputStream();
            DeleteResponse deleteResponse = DeleteResponse.readFrom(deleteResponseStream);
            releaseNettyBufUnderneathStream(deleteResponseStream);
            assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        }
        Thread.sleep(1000);
        for (BlobId id : blobsDeleted) {
            ArrayList<BlobId> ids = new ArrayList<>();
            ids.add(id);
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(id.getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            // get blob all
            GetRequest getRequest = new GetRequest(1, "clientid200", MessageFormatFlags.All, partitionRequestInfoList, GetOption.Include_All);
            DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            assertEquals(ServerErrorCode.No_Error, resp.getError());
            assertEquals(1, resp.getPartitionResponseInfoList().size());
            assertEquals(ServerErrorCode.No_Error, resp.getPartitionResponseInfoList().get(0).getErrorCode());
            assertEquals(1, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().size());
            MessageInfo info = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
            assertEquals(expectedLifeVersion, info.getLifeVersion());
            assertTrue(info.isDeleted());
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                fail();
            }
        }
    }
    channel1.disconnect();
    channel2.disconnect();
    channel3.disconnect();
}

Example 7 with BlobData

use of com.github.ambry.messageformat.BlobData in project ambry by linkedin.

the class Verifier method run.

@Override
public void run() {
    try {
        List<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        while (requestsVerified.get() != totalRequests.get() && !cancelTest.get()) {
            Payload payload = payloadQueue.poll(1000, TimeUnit.MILLISECONDS);
            if (payload != null) {
                notificationSystem.awaitBlobCreations(payload.blobId);
                for (MockDataNodeId dataNodeId : clusterMap.getDataNodes()) {
                    ConnectedChannel channel1 = null;
                    try {
                        BlobId blobId = new BlobId(payload.blobId, clusterMap);
                        Port port = new Port(portType == PortType.PLAINTEXT ? dataNodeId.getPort() : dataNodeId.getSSLPort(), portType);
                        channel1 = connectionPool.checkOutConnection("localhost", port, 10000);
                        ArrayList<BlobId> ids = new ArrayList<BlobId>();
                        ids.add(blobId);
                        partitionRequestInfoList.clear();
                        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
                        partitionRequestInfoList.add(partitionRequestInfo);
                        GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        DataInputStream stream = channel1.receive().getInputStream();
                        GetResponse resp = GetResponse.readFrom(stream, clusterMap);
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println(dataNodeId.getHostname() + " " + dataNodeId.getPort() + " " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                                if (propertyOutput.getBlobSize() != payload.blobProperties.getBlobSize()) {
                                    String exceptionMsg = "blob size not matching " + " expected " + payload.blobProperties.getBlobSize() + " actual " + propertyOutput.getBlobSize();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (!propertyOutput.getServiceId().equals(payload.blobProperties.getServiceId())) {
                                    String exceptionMsg = "service id not matching " + " expected " + payload.blobProperties.getServiceId() + " actual " + propertyOutput.getBlobSize();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (propertyOutput.getAccountId() != payload.blobProperties.getAccountId()) {
                                    String exceptionMsg = "accountid not matching " + " expected " + payload.blobProperties.getAccountId() + " actual " + propertyOutput.getAccountId();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (propertyOutput.getContainerId() != payload.blobProperties.getContainerId()) {
                                    String exceptionMsg = "containerId not matching " + " expected " + payload.blobProperties.getContainerId() + " actual " + propertyOutput.getContainerId();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (propertyOutput.isEncrypted() != payload.blobProperties.isEncrypted()) {
                                    String exceptionMsg = "IsEncrypted not matching " + " expected " + payload.blobProperties.isEncrypted() + " actual " + propertyOutput.isEncrypted();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in blobproperty");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException(e);
                            }
                        }
                        // get user metadata
                        ids.clear();
                        ids.add(blobId);
                        partitionRequestInfoList.clear();
                        partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
                        partitionRequestInfoList.add(partitionRequestInfo);
                        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        stream = channel1.receive().getInputStream();
                        resp = GetResponse.readFrom(stream, clusterMap);
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println("Error after get user metadata " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                                if (userMetadataOutput.compareTo(ByteBuffer.wrap(payload.metadata)) != 0) {
                                    throw new IllegalStateException();
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in usermetadatga");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException();
                            }
                        }
                        // get blob
                        ids.clear();
                        ids.add(blobId);
                        partitionRequestInfoList.clear();
                        partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
                        partitionRequestInfoList.add(partitionRequestInfo);
                        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        stream = channel1.receive().getInputStream();
                        resp = GetResponse.readFrom(stream, clusterMap);
                        // System.out.println("response from get " + resp.getError());
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println("Error after get blob " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                                byte[] blobout = new byte[(int) blobData.getSize()];
                                ByteBuf buffer = blobData.content();
                                try {
                                    buffer.readBytes(blobout);
                                } finally {
                                    buffer.release();
                                }
                                if (ByteBuffer.wrap(blobout).compareTo(ByteBuffer.wrap(payload.blob)) != 0) {
                                    throw new IllegalStateException();
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in blobdata");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException();
                            }
                        }
                        // get blob all
                        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        stream = channel1.receive().getInputStream();
                        resp = GetResponse.readFrom(stream, clusterMap);
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println("Error after get blob " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), new BlobIdFactory(clusterMap));
                                byte[] blobout = new byte[(int) blobAll.getBlobData().getSize()];
                                ByteBuf buffer = blobAll.getBlobData().content();
                                try {
                                    buffer.readBytes(blobout);
                                } finally {
                                    buffer.release();
                                }
                                if (ByteBuffer.wrap(blobout).compareTo(ByteBuffer.wrap(payload.blob)) != 0) {
                                    throw new IllegalStateException();
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in bloball");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException();
                            }
                        }
                        if (payload.blobProperties.getTimeToLiveInSeconds() != Utils.Infinite_Time) {
                            // ttl update, check and wait for replication
                            ServerTestUtil.updateBlobTtl(channel1, new BlobId(payload.blobId, clusterMap), time.milliseconds());
                            ServerTestUtil.checkTtlUpdateStatus(channel1, clusterMap, new BlobIdFactory(clusterMap), blobId, payload.blob, true, Utils.Infinite_Time);
                            notificationSystem.awaitBlobUpdates(payload.blobId, UpdateType.TTL_UPDATE);
                            BlobProperties old = payload.blobProperties;
                            payload.blobProperties = new BlobProperties(old.getBlobSize(), old.getServiceId(), old.getOwnerId(), old.getContentType(), old.isEncrypted(), Utils.Infinite_Time, old.getCreationTimeInMs(), old.getAccountId(), old.getContainerId(), old.isEncrypted(), old.getExternalAssetTag(), old.getContentEncoding(), old.getFilename());
                        }
                    } catch (Exception e) {
                        if (channel1 != null) {
                            connectionPool.destroyConnection(channel1);
                            channel1 = null;
                        }
                    } finally {
                        if (channel1 != null) {
                            connectionPool.checkInConnection(channel1);
                            channel1 = null;
                        }
                    }
                }
                requestsVerified.incrementAndGet();
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        cancelTest.set(true);
    } finally {
        completedLatch.countDown();
    }
}

Example 8 with BlobData

use of com.github.ambry.messageformat.BlobData in project ambry by linkedin.

the class ServerHardDeleteTest method getAndVerify.

/**
 * Fetches the Blob(for all MessageFormatFlags) and verifies the content
 * @param channel the {@link BlockingChannel} to use to send and receive data
 * @param blobsCount the total number of blobs that needs to be verified against
 * @throws Exception
 */
void getAndVerify(ConnectedChannel channel, int blobsCount) throws Exception {
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<>();
    ArrayList<BlobId> ids = new ArrayList<>();
    for (int i = 0; i < blobsCount; i++) {
        ids.add(blobIdList.get(i));
    }
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIdList.get(0).getPartition(), ids);
    partitionRequestInfoList.add(partitionRequestInfo);
    ArrayList<MessageFormatFlags> flags = new ArrayList<>();
    flags.add(MessageFormatFlags.BlobProperties);
    flags.add(MessageFormatFlags.BlobUserMetadata);
    flags.add(MessageFormatFlags.Blob);
    for (MessageFormatFlags flag : flags) {
        GetRequest getRequest = new GetRequest(1, "clientid2", flag, partitionRequestInfoList, GetOption.Include_All);
        GetResponse resp = GetResponse.readFrom(channel.sendAndReceive(getRequest).getInputStream(), mockClusterMap);
        if (flag == MessageFormatFlags.BlobProperties) {
            for (int i = 0; i < blobsCount; i++) {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                Assert.assertEquals(properties.get(i).getBlobSize(), propertyOutput.getBlobSize());
                Assert.assertEquals("serviceid1", propertyOutput.getServiceId());
                Assert.assertEquals("AccountId mismatch", properties.get(i).getAccountId(), propertyOutput.getAccountId());
                Assert.assertEquals("ContainerId mismatch", properties.get(i).getContainerId(), propertyOutput.getContainerId());
            }
        } else if (flag == MessageFormatFlags.BlobUserMetadata) {
            for (int i = 0; i < blobsCount; i++) {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                Assert.assertArrayEquals(userMetadataOutput.array(), usermetadata.get(i));
            }
        } else if (flag == MessageFormatFlags.Blob) {
            for (int i = 0; i < blobsCount; i++) {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                Assert.assertEquals(properties.get(i).getBlobSize(), blobData.getSize());
                byte[] dataOutput = new byte[(int) blobData.getSize()];
                ByteBuf buffer = blobData.content();
                try {
                    buffer.readBytes(dataOutput);
                } finally {
                    buffer.release();
                }
                Assert.assertArrayEquals(dataOutput, data.get(i));
            }
        } else {
            throw new IllegalArgumentException("Unrecognized message format flags " + flags);
        }
    }
}

Example 9 with BlobData

use of com.github.ambry.messageformat.BlobData in project ambry by linkedin.

the class ServerTestUtil method checkBlobContent.

private static void checkBlobContent(MockClusterMap clusterMap, BlobId blobId, ConnectedChannel channel, byte[] dataToCheck, byte[] encryptionKey) throws IOException, MessageFormatException {
    ArrayList<BlobId> listIds = new ArrayList<BlobId>();
    listIds.add(blobId);
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    partitionRequestInfoList.clear();
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobId.getPartition(), listIds);
    partitionRequestInfoList.add(partitionRequestInfo);
    GetRequest getRequest3 = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
    DataInputStream stream = channel.sendAndReceive(getRequest3).getInputStream();
    GetResponse resp = GetResponse.readFrom(stream, clusterMap);
    assertEquals(ServerErrorCode.No_Error, resp.getError());
    BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
    byte[] blobout = getBlobDataAndRelease(blobData);
    assertArrayEquals(dataToCheck, blobout);
    if (encryptionKey != null) {
        assertNotNull("EncryptionKey should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
        assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
    } else {
        assertNull("EncryptionKey should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
    }
    releaseNettyBufUnderneathStream(stream);
}

Example 10 with BlobData

use of com.github.ambry.messageformat.BlobData in project ambry by linkedin.

the class DumpDataHelper method readSingleRecordFromLog.

/**
 * Fetches one blob record from the log
 * @param randomAccessFile {@link RandomAccessFile} referring to the log file
 * @param currentOffset the offset at which to read the record from
 * @param clusterMap the {@link ClusterMap} object to use to generate BlobId
 * @param currentTimeInMs current time in ms to determine expiration
 * @param metrics {@link StoreToolsMetrics} instance
 * @return the {@link LogBlobRecordInfo} containing the blob record info
 * @throws IOException
 * @throws MessageFormatException
 */
static LogBlobRecordInfo readSingleRecordFromLog(RandomAccessFile randomAccessFile, long currentOffset, ClusterMap clusterMap, long currentTimeInMs, StoreToolsMetrics metrics) throws IOException, MessageFormatException {
    String messageheader = null;
    BlobId blobId = null;
    String encryptionKey = null;
    String blobProperty = null;
    String usermetadata = null;
    String blobDataOutput = null;
    String deleteMsg = null;
    boolean isDeleted = false;
    boolean isExpired = false;
    long expiresAtMs = -1;
    int totalRecordSize = 0;
    final Timer.Context context = metrics.readSingleBlobRecordFromLogTimeMs.time();
    try {
        randomAccessFile.seek(currentOffset);
        short version = randomAccessFile.readShort();
        MessageFormatRecord.MessageHeader_Format header = null;
        if (version == MessageFormatRecord.Message_Header_Version_V1) {
            ByteBuffer buffer = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize());
            buffer.putShort(version);
            randomAccessFile.read(buffer.array(), 2, buffer.capacity() - 2);
            buffer.clear();
            header = new MessageFormatRecord.MessageHeader_Format_V1(buffer);
            messageheader = " Header - version " + header.getVersion() + " messagesize " + header.getMessageSize() + " currentOffset " + currentOffset + " blobPropertiesRelativeOffset " + header.getBlobPropertiesRecordRelativeOffset() + " userMetadataRelativeOffset " + header.getUserMetadataRecordRelativeOffset() + " dataRelativeOffset " + header.getBlobRecordRelativeOffset() + " crc " + header.getCrc();
            totalRecordSize += header.getMessageSize() + buffer.capacity();
        } else if (version == MessageFormatRecord.Message_Header_Version_V2) {
            ByteBuffer buffer = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize());
            buffer.putShort(version);
            randomAccessFile.read(buffer.array(), 2, buffer.capacity() - 2);
            buffer.clear();
            header = new MessageFormatRecord.MessageHeader_Format_V2(buffer);
            messageheader = " Header - version " + header.getVersion() + " messagesize " + header.getMessageSize() + " currentOffset " + currentOffset + " blobEncryptionKeyRelativeOffset " + header.getBlobEncryptionKeyRecordRelativeOffset() + " blobPropertiesRelativeOffset " + header.getBlobPropertiesRecordRelativeOffset() + " userMetadataRelativeOffset " + header.getUserMetadataRecordRelativeOffset() + " dataRelativeOffset " + header.getBlobRecordRelativeOffset() + " crc " + header.getCrc();
            totalRecordSize += header.getMessageSize() + buffer.capacity();
        } else if (version == MessageFormatRecord.Message_Header_Version_V3) {
            ByteBuffer buffer = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V3.getHeaderSize());
            buffer.putShort(version);
            randomAccessFile.read(buffer.array(), 2, buffer.capacity() - 2);
            buffer.clear();
            header = new MessageFormatRecord.MessageHeader_Format_V3(buffer);
            messageheader = " Header - version " + header.getVersion() + " messagesize " + header.getMessageSize() + " currentOffset " + currentOffset + " blobEncryptionKeyRelativeOffset " + header.getBlobEncryptionKeyRecordRelativeOffset() + " blobPropertiesRelativeOffset " + header.getBlobPropertiesRecordRelativeOffset() + " userMetadataRelativeOffset " + header.getUserMetadataRecordRelativeOffset() + " dataRelativeOffset " + header.getBlobRecordRelativeOffset() + " crc " + header.getCrc();
            totalRecordSize += header.getMessageSize() + buffer.capacity();
        } else {
            throw new MessageFormatException("Header version not supported " + version, MessageFormatErrorCodes.IO_Error);
        }
        // read blob id
        InputStream streamlog = Channels.newInputStream(randomAccessFile.getChannel());
        blobId = new BlobId(new DataInputStream(streamlog), clusterMap);
        totalRecordSize += blobId.sizeInBytes();
        if (header.getBlobPropertiesRecordRelativeOffset() != MessageFormatRecord.Message_Header_Invalid_Relative_Offset) {
            ByteBuffer blobEncryptionKey = null;
            if (header.hasEncryptionKeyRecord()) {
                blobEncryptionKey = MessageFormatRecord.deserializeBlobEncryptionKey(streamlog);
                encryptionKey = "EncryptionKey found which is of size " + blobEncryptionKey.remaining();
            }
            BlobProperties props = MessageFormatRecord.deserializeBlobProperties(streamlog);
            expiresAtMs = Utils.addSecondsToEpochTime(props.getCreationTimeInMs(), props.getTimeToLiveInSeconds());
            isExpired = isExpired(expiresAtMs, currentTimeInMs);
            blobProperty = " Blob properties - blobSize  " + props.getBlobSize() + " serviceId " + props.getServiceId() + ", isExpired " + isExpired + " accountId " + props.getAccountId() + " containerId " + props.getContainerId();
            ByteBuffer metadata = MessageFormatRecord.deserializeUserMetadata(streamlog);
            usermetadata = " Metadata - size " + metadata.capacity();
            BlobData blobData = MessageFormatRecord.deserializeBlob(streamlog);
            blobDataOutput = "Blob - size " + blobData.getSize();
        } else {
            UpdateRecord updateRecord = MessageFormatRecord.deserializeUpdateRecord(streamlog);
            switch(updateRecord.getType()) {
                case DELETE:
                    isDeleted = true;
                    deleteMsg = "delete change : AccountId:" + updateRecord.getAccountId() + ", ContainerId:" + updateRecord.getContainerId() + ", DeletionTimeInSecs:" + updateRecord.getUpdateTimeInMs();
                    break;
                default:
                    // TODO (TTL update): handle TTL update
                    throw new IllegalStateException("Unrecognized update record type: " + updateRecord.getType());
            }
        }
        return new LogBlobRecordInfo(messageheader, blobId, encryptionKey, blobProperty, usermetadata, blobDataOutput, deleteMsg, isDeleted, isExpired, expiresAtMs, totalRecordSize);
    } finally {
        context.stop();
    }
}