Search in sources :

Example 11 with Http2ClientMetrics

use of com.github.ambry.network.http2.Http2ClientMetrics in project ambry by linkedin.

the class Http2NetworkClientTest method putGetTest.

@Test
public void putGetTest() throws Exception {
    MockClusterMap clusterMap = http2Cluster.getClusterMap();
    DataNodeId dataNodeId = http2Cluster.getGeneralDataNode();
    BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
    SSLFactory sslFactory = new NettySslHttp2Factory(clientSSLConfig);
    Http2NetworkClient networkClient = new Http2NetworkClient(new Http2ClientMetrics(new MetricRegistry()), new Http2ClientConfig(new VerifiableProperties(new Properties())), sslFactory, eventLoopGroup);
    // Put a blob
    int blobSize = 1024 * 1024;
    byte[] usermetadata = new byte[1000];
    byte[] data = new byte[blobSize];
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties properties = new BlobProperties(blobSize, "serviceid1", accountId, containerId, false);
    TestUtils.RANDOM.nextBytes(usermetadata);
    TestUtils.RANDOM.nextBytes(data);
    List<? extends PartitionId> partitionIds = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
    short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
    BlobId blobId1 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
    // put blob 1
    PutRequest putRequest = new PutRequest(1, "client1", blobId1, properties, ByteBuffer.wrap(usermetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, null);
    RequestInfo request = new RequestInfo(dataNodeId.getHostname(), new Port(dataNodeId.getHttp2Port(), PortType.HTTP2), putRequest, clusterMap.getReplicaIds(dataNodeId).get(0), null);
    List<ResponseInfo> responseInfos = networkClient.sendAndPoll(Collections.singletonList(request), new HashSet<>(), 300);
    long startTime = SystemTime.getInstance().milliseconds();
    while (responseInfos.size() == 0) {
        responseInfos = networkClient.sendAndPoll(Collections.EMPTY_LIST, new HashSet<>(), 300);
        if (SystemTime.getInstance().milliseconds() - startTime >= 6000) {
            fail("Network Client no reponse and timeout.");
        }
        Thread.sleep(30);
    }
    assertEquals("Should be only one response", 1, responseInfos.size());
    DataInputStream dis = new NettyByteBufDataInputStream(responseInfos.get(0).content());
    PutResponse putResponse = PutResponse.readFrom(dis);
    assertEquals("No error expected.", ServerErrorCode.No_Error, putResponse.getError());
    // Get the blob
    // get blob properties
    ArrayList<BlobId> ids = new ArrayList<BlobId>();
    MockPartitionId partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
    ids.add(blobId1);
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(partition, ids);
    partitionRequestInfoList.add(partitionRequestInfo);
    GetRequest getRequest = new GetRequest(1, "http2-clientid", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
    request = new RequestInfo(dataNodeId.getHostname(), new Port(dataNodeId.getHttp2Port(), PortType.HTTP2), getRequest, clusterMap.getReplicaIds(dataNodeId).get(0), null);
    responseInfos = networkClient.sendAndPoll(Collections.singletonList(request), new HashSet<>(), 300);
    startTime = SystemTime.getInstance().milliseconds();
    while (responseInfos.size() == 0) {
        responseInfos = networkClient.sendAndPoll(Collections.EMPTY_LIST, new HashSet<>(), 300);
        if (SystemTime.getInstance().milliseconds() - startTime >= 3000) {
            fail("Network Client no response and timeout.");
        }
        Thread.sleep(30);
    }
    assertEquals("Should be only one response", 1, responseInfos.size());
    dis = new NettyByteBufDataInputStream(responseInfos.get(0).content());
    GetResponse resp = GetResponse.readFrom(dis, clusterMap);
    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
    // verify BlobProperties
    BlobProperties propertyOutput = blobAll.getBlobInfo().getBlobProperties();
    assertEquals(blobSize, propertyOutput.getBlobSize());
    assertEquals("serviceid1", propertyOutput.getServiceId());
    assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
    assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
    // verify UserMetadata
    byte[] userMetadataOutput = blobAll.getBlobInfo().getUserMetadata();
    assertArrayEquals(usermetadata, userMetadataOutput);
    // verify content
    byte[] actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
    assertArrayEquals("Content mismatch.", data, actualBlobData);
}
Also used : SSLFactory(com.github.ambry.commons.SSLFactory) Http2ClientMetrics(com.github.ambry.network.http2.Http2ClientMetrics) Port(com.github.ambry.network.Port) ArrayList(java.util.ArrayList) NettySslHttp2Factory(com.github.ambry.commons.NettySslHttp2Factory) BlobProperties(com.github.ambry.messageformat.BlobProperties) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) PartitionRequestInfo(com.github.ambry.protocol.PartitionRequestInfo) RequestInfo(com.github.ambry.network.RequestInfo) PutResponse(com.github.ambry.protocol.PutResponse) BlobAll(com.github.ambry.messageformat.BlobAll) GetRequest(com.github.ambry.protocol.GetRequest) HashSet(java.util.HashSet) ResponseInfo(com.github.ambry.network.ResponseInfo) NettyByteBufDataInputStream(com.github.ambry.utils.NettyByteBufDataInputStream) VerifiableProperties(com.github.ambry.config.VerifiableProperties) MockPartitionId(com.github.ambry.clustermap.MockPartitionId) MetricRegistry(com.codahale.metrics.MetricRegistry) PutRequest(com.github.ambry.protocol.PutRequest) Http2ClientConfig(com.github.ambry.config.Http2ClientConfig) DataInputStream(java.io.DataInputStream) NettyByteBufDataInputStream(com.github.ambry.utils.NettyByteBufDataInputStream) PartitionRequestInfo(com.github.ambry.protocol.PartitionRequestInfo) GetResponse(com.github.ambry.protocol.GetResponse) BlobIdFactory(com.github.ambry.commons.BlobIdFactory) BlobProperties(com.github.ambry.messageformat.BlobProperties) Http2NetworkClient(com.github.ambry.network.http2.Http2NetworkClient) DataNodeId(com.github.ambry.clustermap.DataNodeId) MockDataNodeId(com.github.ambry.clustermap.MockDataNodeId) BlobId(com.github.ambry.commons.BlobId) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) Test(org.junit.Test)

Example 12 with Http2ClientMetrics

use of com.github.ambry.network.http2.Http2ClientMetrics in project ambry by linkedin.

the class Http2NetworkClientTest method warmUpConnections.

@Test
public void warmUpConnections() throws Exception {
    int connectionPerPort = 4;
    SSLFactory sslFactory = new NettySslHttp2Factory(clientSSLConfig);
    Properties properties = new Properties();
    properties.setProperty(Http2ClientConfig.HTTP2_MIN_CONNECTION_PER_PORT, Integer.toString(connectionPerPort));
    Http2ClientConfig http2ClientConfig = new Http2ClientConfig(new VerifiableProperties(properties));
    Http2NetworkClient networkClient = new Http2NetworkClient(new Http2ClientMetrics(new MetricRegistry()), http2ClientConfig, sslFactory, eventLoopGroup);
    MockClusterMap clusterMap = http2Cluster.getClusterMap();
    assertEquals("Connection count is not expected", 0, networkClient.warmUpConnections(clusterMap.getDataNodeIds(), 0, 1000, new ArrayList<>()));
    assertEquals("Connection count is not expected", clusterMap.getDataNodeIds().size() * connectionPerPort / 2, networkClient.warmUpConnections(clusterMap.getDataNodeIds(), 50, 1000, new ArrayList<>()));
    assertEquals("Connection count is not expected", clusterMap.getDataNodeIds().size() * connectionPerPort, networkClient.warmUpConnections(clusterMap.getDataNodeIds(), 100, 1000, new ArrayList<>()));
    // All connection should be failed. Connection refused exceptions will be in test log.
    List<Port> ports = new ArrayList<>();
    ports.add(new Port(79, PortType.HTTP2));
    ports.add(new Port(78, PortType.PLAINTEXT));
    assertEquals("Connection count is not expected", 0, networkClient.warmUpConnections(Collections.singletonList(new MockDataNodeId(ports, null, "DC1")), 100, 1000, new ArrayList<>()));
}
Also used : SSLFactory(com.github.ambry.commons.SSLFactory) Http2ClientMetrics(com.github.ambry.network.http2.Http2ClientMetrics) VerifiableProperties(com.github.ambry.config.VerifiableProperties) MetricRegistry(com.codahale.metrics.MetricRegistry) Port(com.github.ambry.network.Port) ArrayList(java.util.ArrayList) Http2ClientConfig(com.github.ambry.config.Http2ClientConfig) NettySslHttp2Factory(com.github.ambry.commons.NettySslHttp2Factory) BlobProperties(com.github.ambry.messageformat.BlobProperties) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) MockDataNodeId(com.github.ambry.clustermap.MockDataNodeId) Http2NetworkClient(com.github.ambry.network.http2.Http2NetworkClient) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) Test(org.junit.Test)

Example 13 with Http2ClientMetrics

use of com.github.ambry.network.http2.Http2ClientMetrics in project ambry by linkedin.

the class ServerTestUtil method getBlockingChannelBasedOnPortType.

/**
 * Returns BlockingChannel, SSLBlockingChannel or Http2BlockingChannel depending on whether the port type is PlainText,
 * SSL or HTTP2 port for the given targetPort
 * @param portType The type of port to connect to
 * @param dataNodeId To which {@link MockDataNodeId} to connect
 * @param sslSocketFactory the {@link SSLSocketFactory}.
 * @param sslConfig the {@link SSLConfig}.
 * @return ConnectedChannel
 */
private static ConnectedChannel getBlockingChannelBasedOnPortType(PortType portType, DataNodeId dataNodeId, SSLSocketFactory sslSocketFactory, SSLConfig sslConfig) {
    ConnectedChannel channel = null;
    String hostName = dataNodeId.getHostname();
    if (portType == PortType.PLAINTEXT) {
        channel = new BlockingChannel(hostName, dataNodeId.getPort(), 10000, 10000, 10000, 2000);
    } else if (portType == PortType.SSL) {
        channel = new SSLBlockingChannel(hostName, dataNodeId.getSSLPort(), new MetricRegistry(), 10000, 10000, 10000, 4000, sslSocketFactory, sslConfig);
    } else if (portType == PortType.HTTP2) {
        channel = new Http2BlockingChannel(hostName, dataNodeId.getHttp2Port(), sslConfig, new Http2ClientConfig(new VerifiableProperties(new Properties())), new Http2ClientMetrics(new MetricRegistry()));
    }
    return channel;
}
Also used : Http2ClientMetrics(com.github.ambry.network.http2.Http2ClientMetrics) Http2BlockingChannel(com.github.ambry.network.http2.Http2BlockingChannel) VerifiableProperties(com.github.ambry.config.VerifiableProperties) MetricRegistry(com.codahale.metrics.MetricRegistry) ConnectedChannel(com.github.ambry.network.ConnectedChannel) Http2ClientConfig(com.github.ambry.config.Http2ClientConfig) BlobProperties(com.github.ambry.messageformat.BlobProperties) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) BlockingChannel(com.github.ambry.network.BlockingChannel) Http2BlockingChannel(com.github.ambry.network.http2.Http2BlockingChannel) SSLBlockingChannel(com.github.ambry.network.SSLBlockingChannel) SSLBlockingChannel(com.github.ambry.network.SSLBlockingChannel)

Aggregations

MetricRegistry (com.codahale.metrics.MetricRegistry)10 Test (org.junit.Test)9 MultiplexedChannelRecordTest (com.github.ambry.network.http2.MultiplexedChannelRecordTest)7 ChannelPool (io.netty.channel.pool.ChannelPool)7 SocketChannel (io.netty.channel.socket.SocketChannel)7 NioSocketChannel (io.netty.channel.socket.nio.NioSocketChannel)7 Http2ClientConfig (com.github.ambry.config.Http2ClientConfig)5 Http2ClientMetrics (com.github.ambry.network.http2.Http2ClientMetrics)5 Channel (io.netty.channel.Channel)5 Http2StreamChannel (io.netty.handler.codec.http2.Http2StreamChannel)5 DefaultPromise (io.netty.util.concurrent.DefaultPromise)5 ArrayList (java.util.ArrayList)5 NettySslHttp2Factory (com.github.ambry.commons.NettySslHttp2Factory)4 SSLFactory (com.github.ambry.commons.SSLFactory)4 Port (com.github.ambry.network.Port)4 IOException (java.io.IOException)4 DataNodeId (com.github.ambry.clustermap.DataNodeId)3 VerifiableProperties (com.github.ambry.config.VerifiableProperties)3 BlobProperties (com.github.ambry.messageformat.BlobProperties)3 Properties (java.util.Properties)3