Examples with DatanodeRatisServerConfig org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig used on opensource projects

Example 1 with DatanodeRatisServerConfig

use of org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig in project ozone by apache.

the class TestFreonWithPipelineDestroy method init.

/**
 * Create a MiniDFSCluster for testing.
 * <p>
 * Ozone is made active by setting OZONE_ENABLED = true
 */
@BeforeClass
public static void init() throws Exception {
    conf = new OzoneConfiguration();
    conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 1, TimeUnit.SECONDS);
    conf.setTimeDuration(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, 1, TimeUnit.SECONDS);
    DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class);
    ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
    ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3));
    conf.setFromObject(ratisServerConfig);
    RatisClientConfig.RaftConfig raftClientConfig = conf.getObject(RatisClientConfig.RaftConfig.class);
    raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3));
    raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(3));
    conf.setFromObject(raftClientConfig);
    cluster = MiniOzoneCluster.newBuilder(conf).setHbProcessorInterval(1000).setHbInterval(1000).setNumDatanodes(3).setTotalPipelineNumLimit(8).build();
    cluster.waitForClusterToBeReady();
}

Example 2 with DatanodeRatisServerConfig

use of org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig in project ozone by apache.

the class TestNodeFailure method init.

/**
 * Create a MiniDFSCluster for testing.
 *
 * @throws IOException
 */
@BeforeClass
public static void init() throws Exception {
    final OzoneConfiguration conf = new OzoneConfiguration();
    DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class);
    ratisServerConfig.setFollowerSlownessTimeout(Duration.ofSeconds(10));
    ratisServerConfig.setNoLeaderTimeout(Duration.ofMinutes(5));
    conf.setFromObject(ratisServerConfig);
    conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 1);
    conf.set(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, "2s");
    cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(6).setHbInterval(1000).setHbProcessorInterval(1000).build();
    cluster.waitForClusterToBeReady();
    final StorageContainerManager scm = cluster.getStorageContainerManager();
    pipelineManager = scm.getPipelineManager();
    ratisPipelines = pipelineManager.getPipelines(RatisReplicationConfig.getInstance(ReplicationFactor.THREE));
    timeForFailure = (int) ratisServerConfig.getFollowerSlownessTimeout();
}

Example 3 with DatanodeRatisServerConfig

use of org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig in project ozone by apache.

the class RootedOzoneContract method createCluster.

public static void createCluster() throws IOException {
    OzoneConfiguration conf = new OzoneConfiguration();
    DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class);
    ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
    ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10));
    conf.setFromObject(ratisServerConfig);
    RatisClientConfig.RaftConfig raftClientConfig = conf.getObject(RatisClientConfig.RaftConfig.class);
    raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3));
    raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10));
    conf.setFromObject(raftClientConfig);
    conf.addResource(CONTRACT_XML);
    cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
    try {
        cluster.waitForClusterToBeReady();
        cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.THREE, 180000);
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 4 with DatanodeRatisServerConfig

use of org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig in project ozone by apache.

the class Test2WayCommitInRatis method startCluster.

/**
 * Create a MiniDFSCluster for testing.
 * <p>
 * Ozone is made active by setting OZONE_ENABLED = true
 *
 * @throws IOException
 */
private void startCluster(OzoneConfiguration conf) throws Exception {
    chunkSize = 100;
    flushSize = 2 * chunkSize;
    maxFlushSize = 2 * flushSize;
    blockSize = 2 * maxFlushSize;
    // Make sure the pipeline does not get destroyed quickly
    conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 60, TimeUnit.SECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 60000, TimeUnit.SECONDS);
    DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class);
    ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
    ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10));
    conf.setFromObject(ratisServerConfig);
    RatisClientConfig.RaftConfig raftClientConfig = conf.getObject(RatisClientConfig.RaftConfig.class);
    raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3));
    raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10));
    conf.setFromObject(raftClientConfig);
    conf.setQuietMode(false);
    cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).setBlockSize(blockSize).setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize).setStreamBufferMaxSize(maxFlushSize).setStreamBufferSizeUnit(StorageUnit.BYTES).build();
    cluster.waitForClusterToBeReady();
    // the easiest way to create an open container is creating a key
    client = OzoneClientFactory.getRpcClient(conf);
    objectStore = client.getObjectStore();
    volumeName = "watchforcommithandlingtest";
    bucketName = volumeName;
    objectStore.createVolume(volumeName);
    objectStore.getVolume(volumeName).createBucket(bucketName);
    storageContainerLocationClient = cluster.getStorageContainerLocationClient();
}

Example 5 with DatanodeRatisServerConfig

use of org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig in project ozone by apache.

the class TestContainerReplicationEndToEnd method init.

/**
 * Create a MiniDFSCluster for testing.
 *
 * @throws IOException
 */
@BeforeClass
public static void init() throws Exception {
    conf = new OzoneConfiguration();
    path = GenericTestUtils.getTempPath(TestContainerStateMachineFailures.class.getSimpleName());
    File baseDir = new File(path);
    baseDir.mkdirs();
    containerReportInterval = 2000;
    conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, containerReportInterval, TimeUnit.MILLISECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 5 * containerReportInterval, TimeUnit.MILLISECONDS);
    conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL, 10 * containerReportInterval, TimeUnit.MILLISECONDS);
    conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 1000, TimeUnit.SECONDS);
    DatanodeRatisServerConfig ratisServerConfig = conf.getObject(DatanodeRatisServerConfig.class);
    ratisServerConfig.setFollowerSlownessTimeout(Duration.ofSeconds(1000));
    ratisServerConfig.setNoLeaderTimeout(Duration.ofSeconds(1000));
    conf.setFromObject(ratisServerConfig);
    ReplicationManagerConfiguration replicationConf = conf.getObject(ReplicationManagerConfiguration.class);
    replicationConf.setInterval(Duration.ofMillis(containerReportInterval));
    conf.setFromObject(replicationConf);
    conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 2);
    conf.setQuietMode(false);
    cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(4).setTotalPipelineNumLimit(6).setHbInterval(200).build();
    cluster.waitForClusterToBeReady();
    cluster.getStorageContainerManager().getReplicationManager().start();
    // the easiest way to create an open container is creating a key
    client = OzoneClientFactory.getRpcClient(conf);
    objectStore = client.getObjectStore();
    xceiverClientManager = new XceiverClientManager(conf);
    volumeName = "testcontainerstatemachinefailures";
    bucketName = volumeName;
    objectStore.createVolume(volumeName);
    objectStore.getVolume(volumeName).createBucket(bucketName);
}