Search in sources :

Example 1 with NettyChannelBuilder

use of org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder in project ozone by apache.

the class FollowerAppendLogEntryGenerator method call.

@Override
public Void call() throws Exception {
    inFlightMessages = new LinkedBlockingQueue<>(inflightLimit);
    timer = getMetrics().timer("append-entry");
    byte[] data = RandomStringUtils.randomAscii(chunkSize).getBytes(StandardCharsets.UTF_8);
    dataToWrite = ByteString.copyFrom(data);
    OzoneConfiguration conf = createOzoneConfiguration();
    setServerIdFromFile(conf);
    Preconditions.assertTrue(getThreadNo() == 1, "This test should be executed from one thread");
    // the raft identifier which is used by the freon
    requestor = RaftPeerProto.newBuilder().setId(RaftPeerId.valueOf(FAKE_LEADER_ID).toByteString()).setAddress(FAKE_LEADER_ADDDRESS).build();
    NettyChannelBuilder channelBuilder = NettyChannelBuilder.forTarget(serverAddress);
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
    ManagedChannel build = channelBuilder.build();
    stub = RaftServerProtocolServiceGrpc.newStub(build);
    if (rateLimit != 0) {
        rateLimiter = new TimedSemaphore(1, TimeUnit.SECONDS, rateLimit);
    }
    init();
    sender = stub.appendEntries(this);
    if (nextIndex == 0) {
        // first: configure a new ratis group (one follower, one fake leader
        // (freon))
        configureGroup();
        RequestVoteReplyProto vote = requestVote().get(1000, TimeUnit.SECONDS);
        LOG.info("Datanode answered to the vote request: {}", vote);
        if (!vote.getServerReply().getSuccess()) {
            throw new RuntimeException("Datanode didn't vote to the fake freon leader.");
        }
        // send the first appendEntry. This one is special as it initialized the
        // log.
        long callId = callIdRandom.nextLong();
        inFlightMessages.put(callId);
        sender.onNext(createInitialLogEntry(callId));
        nextIndex = 1L;
    }
    // We can generate as mach entry as we need.
    runTests(this::sendAppendLogEntryRequest);
    if (rateLimiter != null) {
        rateLimiter.shutdown();
    }
    return null;
}
Also used : OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) NettyChannelBuilder(org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder) ManagedChannel(org.apache.ratis.thirdparty.io.grpc.ManagedChannel) RequestVoteReplyProto(org.apache.ratis.proto.RaftProtos.RequestVoteReplyProto) TimedSemaphore(org.apache.commons.lang3.concurrent.TimedSemaphore)

Example 2 with NettyChannelBuilder

use of org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder in project ozone by apache.

the class LeaderAppendLogEntryGenerator method call.

@Override
public Void call() throws Exception {
    inFlightMessages = new LinkedBlockingQueue<>(inflightLimit);
    OzoneConfiguration conf = createOzoneConfiguration();
    byte[] data = RandomStringUtils.randomAscii(chunkSize).getBytes(StandardCharsets.UTF_8);
    dataToWrite = ByteString.copyFrom(data);
    setServerIdFromFile(conf);
    requestor = RaftPeerProto.newBuilder().setId(RaftPeerId.valueOf(FAKE_FOLLOWER_ID1).toByteString()).setAddress(FAKE_LEADER_ADDDRESS1).build();
    NettyChannelBuilder channelBuilder = NettyChannelBuilder.forTarget(serverAddress);
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
    ManagedChannel build = channelBuilder.build();
    stub = RaftServerProtocolServiceGrpc.newStub(build);
    init();
    if (nextIndex == 0) {
        configureGroup();
    }
    Thread.sleep(3000L);
    XceiverClientRatis client = createXceiverClient(conf);
    client.connect();
    long containerId = 1L;
    System.out.println(client.sendCommand(createContainerRequest(containerId)));
    timer = getMetrics().timer("append-entry");
    runTests(step -> timer.time(() -> {
        inFlightMessages.put(step);
        XceiverClientReply xceiverClientReply = client.sendCommandAsync(createChunkWriteRequest(containerId, step));
        xceiverClientReply.getResponse().thenApply(response -> inFlightMessages.remove(step));
        return null;
    }));
    return null;
}
Also used : XceiverClientReply(org.apache.hadoop.hdds.scm.XceiverClientReply) Name(org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name) RaftGroup(org.apache.ratis.protocol.RaftGroup) CreateContainerRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.CreateContainerRequestProto) LoggerFactory(org.slf4j.LoggerFactory) Random(java.util.Random) ByteString(org.apache.ratis.thirdparty.com.google.protobuf.ByteString) HddsVersionProvider(org.apache.hadoop.hdds.cli.HddsVersionProvider) RatisReplicationConfig(org.apache.hadoop.hdds.client.RatisReplicationConfig) RaftPeer(org.apache.ratis.protocol.RaftPeer) ReplicationFactor(org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor) ChunkInfo(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo) NegotiationType(org.apache.ratis.thirdparty.io.grpc.netty.NegotiationType) UUID(java.util.UUID) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) StandardCharsets(java.nio.charset.StandardCharsets) XceiverClientReply(org.apache.hadoop.hdds.scm.XceiverClientReply) RaftServerProtocolServiceGrpc(org.apache.ratis.proto.grpc.RaftServerProtocolServiceGrpc) ContainerType(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType) List(java.util.List) Option(picocli.CommandLine.Option) ClientId(org.apache.ratis.protocol.ClientId) RaftClientReply(org.apache.ratis.protocol.RaftClientReply) DatanodeBlockID(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.DatanodeBlockID) RaftProperties(org.apache.ratis.conf.RaftProperties) ManagedChannel(org.apache.ratis.thirdparty.io.grpc.ManagedChannel) Timer(com.codahale.metrics.Timer) RaftServerProtocolServiceStub(org.apache.ratis.proto.grpc.RaftServerProtocolServiceGrpc.RaftServerProtocolServiceStub) RandomStringUtils(org.apache.commons.lang3.RandomStringUtils) ContainerCommandRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto) RaftPeerProto(org.apache.ratis.proto.RaftProtos.RaftPeerProto) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) PipelineState(org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState) Callable(java.util.concurrent.Callable) ArrayList(java.util.ArrayList) RaftGroupId(org.apache.ratis.protocol.RaftGroupId) XceiverClientRatis(org.apache.hadoop.hdds.scm.XceiverClientRatis) Type(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type) ChecksumType(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType) WriteChunkRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto) NettyChannelBuilder(org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder) Command(picocli.CommandLine.Command) ChecksumData(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumData) Logger(org.slf4j.Logger) RaftPeerId(org.apache.ratis.protocol.RaftPeerId) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) IOException(java.io.IOException) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) PipelineID(org.apache.hadoop.hdds.scm.pipeline.PipelineID) RaftClient(org.apache.ratis.client.RaftClient) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) NettyChannelBuilder(org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder) ManagedChannel(org.apache.ratis.thirdparty.io.grpc.ManagedChannel) XceiverClientRatis(org.apache.hadoop.hdds.scm.XceiverClientRatis)

Example 3 with NettyChannelBuilder

use of org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder in project ozone by apache.

the class XceiverClientGrpc method connectToDatanode.

private synchronized void connectToDatanode(DatanodeDetails dn) throws IOException {
    if (isConnected(dn)) {
        return;
    }
    // read port from the data node, on failure use default configured
    // port.
    int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
    if (port == 0) {
        port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
    }
    // Add credential context to the client call
    if (LOG.isDebugEnabled()) {
        LOG.debug("Nodes in pipeline : {}", pipeline.getNodes());
        LOG.debug("Connecting to server : {}", dn.getIpAddress());
    }
    NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext().maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE).intercept(new GrpcClientInterceptor());
    if (secConfig.isGrpcTlsEnabled()) {
        SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
        if (caCerts != null) {
            sslContextBuilder.trustManager(caCerts);
        }
        if (secConfig.useTestCert()) {
            channelBuilder.overrideAuthority("localhost");
        }
        channelBuilder.useTransportSecurity().sslContext(sslContextBuilder.build());
    } else {
        channelBuilder.usePlaintext();
    }
    ManagedChannel channel = channelBuilder.build();
    XceiverClientProtocolServiceStub asyncStub = XceiverClientProtocolServiceGrpc.newStub(channel);
    asyncStubs.put(dn.getUuid(), asyncStub);
    channels.put(dn.getUuid(), channel);
}
Also used : XceiverClientProtocolServiceStub(org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc.XceiverClientProtocolServiceStub) SslContextBuilder(org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder) NettyChannelBuilder(org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder) GrpcClientInterceptor(org.apache.hadoop.hdds.tracing.GrpcClientInterceptor) ManagedChannel(org.apache.ratis.thirdparty.io.grpc.ManagedChannel)

Example 4 with NettyChannelBuilder

use of org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder in project ozone by apache.

the class SCMUpdateServiceGrpcClient method createChannel.

public void createChannel() {
    NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(host, port).usePlaintext().maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
    channel = channelBuilder.build();
    updateClient = SCMUpdateServiceGrpc.newStub(channel);
    subscribeClient = SCMUpdateServiceGrpc.newBlockingStub(channel);
}
Also used : NettyChannelBuilder(org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder)

Aggregations

NettyChannelBuilder (org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder)4 ManagedChannel (org.apache.ratis.thirdparty.io.grpc.ManagedChannel)3 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)2 Timer (com.codahale.metrics.Timer)1 IOException (java.io.IOException)1 StandardCharsets (java.nio.charset.StandardCharsets)1 ArrayList (java.util.ArrayList)1 List (java.util.List)1 Random (java.util.Random)1 UUID (java.util.UUID)1 Callable (java.util.concurrent.Callable)1 LinkedBlockingQueue (java.util.concurrent.LinkedBlockingQueue)1 RandomStringUtils (org.apache.commons.lang3.RandomStringUtils)1 TimedSemaphore (org.apache.commons.lang3.concurrent.TimedSemaphore)1 HddsVersionProvider (org.apache.hadoop.hdds.cli.HddsVersionProvider)1 RatisReplicationConfig (org.apache.hadoop.hdds.client.RatisReplicationConfig)1 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)1 Name (org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name)1 ChecksumData (org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumData)1 ChecksumType (org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType)1