Examples with RoundRobinVolumeChoosingPolicy org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy used on opensource projects

Example 1 with RoundRobinVolumeChoosingPolicy

use of org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy in project ozone by apache.

the class ChunkManagerDiskWrite method call.

@Override
public Void call() throws Exception {
    try {
        init();
        OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
        VolumeSet volumeSet = new MutableVolumeSet("dnid", "clusterid", ozoneConfiguration, null, StorageVolume.VolumeType.DATA_VOLUME, null);
        Random random = new Random();
        VolumeChoosingPolicy volumeChoicePolicy = new RoundRobinVolumeChoosingPolicy();
        final int threadCount = getThreadNo();
        // create a dedicated (NEW) container for each thread
        for (int i = 1; i <= threadCount; i++) {
            // use a non-negative container id
            long containerId = random.nextLong() & 0x0F_FF_FF_FF_FF_FF_FF_FFL;
            KeyValueContainerData keyValueContainerData = new KeyValueContainerData(containerId, containerLayout, 1_000_000L, getPrefix(), "nodeid");
            KeyValueContainer keyValueContainer = new KeyValueContainer(keyValueContainerData, ozoneConfiguration);
            keyValueContainer.create(volumeSet, volumeChoicePolicy, "scmid");
            containersPerThread.put(i, keyValueContainer);
        }
        blockSize = chunkSize * chunksPerBlock;
        data = randomAscii(chunkSize).getBytes(UTF_8);
        chunkManager = ChunkManagerFactory.createChunkManager(ozoneConfiguration, null, null);
        timer = getMetrics().timer("chunk-write");
        LOG.info("Running chunk write test: threads={} chunkSize={} " + "chunksPerBlock={} layout={}", threadCount, chunkSize, chunksPerBlock, containerLayout);
        runTests(this::writeChunk);
    } finally {
        if (chunkManager != null) {
            chunkManager.shutdown();
        }
    }
    return null;
}

Example 2 with RoundRobinVolumeChoosingPolicy

use of org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy in project ozone by apache.

the class GeneratorDatanode method call.

@Override
public Void call() throws Exception {
    init();
    config = createOzoneConfiguration();
    BlockManager blockManager = new BlockManagerImpl(config);
    chunkManager = ChunkManagerFactory.createChunkManager(config, blockManager, null);
    final Collection<String> storageDirs = HddsServerUtil.getDatanodeStorageDirs(config);
    String firstStorageDir = StorageLocation.parse(storageDirs.iterator().next()).getUri().getPath();
    final Path hddsDir = Paths.get(firstStorageDir, "hdds");
    if (!Files.exists(hddsDir)) {
        throw new NoSuchFileException(hddsDir + " doesn't exist. Please start a real cluster to initialize the " + "VERSION descriptors, and re-start this generator after the files" + " are created (but after cluster is stopped).");
    }
    scmId = getScmIdFromStoragePath(hddsDir);
    final File versionFile = new File(firstStorageDir, "hdds/VERSION");
    Properties props = DatanodeVersionFile.readFrom(versionFile);
    if (props.isEmpty()) {
        throw new InconsistentStorageStateException("Version file " + versionFile + " is missing");
    }
    String clusterId = HddsVolumeUtil.getProperty(props, OzoneConsts.CLUSTER_ID, versionFile);
    datanodeId = HddsVolumeUtil.getProperty(props, OzoneConsts.DATANODE_UUID, versionFile);
    volumeSet = new MutableVolumeSet(datanodeId, clusterId, config, null, StorageVolume.VolumeType.DATA_VOLUME, null);
    volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
    final OzoneClientConfig ozoneClientConfig = config.getObject(OzoneClientConfig.class);
    checksum = new Checksum(ozoneClientConfig.getChecksumType(), ozoneClientConfig.getBytesPerChecksum());
    timer = getMetrics().timer("datanode-generator");
    runTests(this::generateData);
    return null;
}

Example 3 with RoundRobinVolumeChoosingPolicy

use of org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy in project ozone by apache.

the class TestContainerPersistence method init.

@BeforeClass
public static void init() {
    conf = new OzoneConfiguration();
    hddsPath = GenericTestUtils.getTempPath(TestContainerPersistence.class.getSimpleName());
    conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, hddsPath);
    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, hddsPath);
    volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
}

Example 4 with RoundRobinVolumeChoosingPolicy

use of org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy in project ozone by apache.

the class TestHddsDispatcher method testContainerCloseActionWhenFull.

@Test
public void testContainerCloseActionWhenFull() throws IOException {
    String testDir = GenericTestUtils.getTempPath(TestHddsDispatcher.class.getSimpleName());
    OzoneConfiguration conf = new OzoneConfiguration();
    conf.set(HDDS_DATANODE_DIR_KEY, testDir);
    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir);
    DatanodeDetails dd = randomDatanodeDetails();
    MutableVolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, null, StorageVolume.VolumeType.DATA_VOLUME, null);
    try {
        UUID scmId = UUID.randomUUID();
        ContainerSet containerSet = new ContainerSet();
        DatanodeStateMachine stateMachine = Mockito.mock(DatanodeStateMachine.class);
        StateContext context = Mockito.mock(StateContext.class);
        Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(dd);
        Mockito.when(context.getParent()).thenReturn(stateMachine);
        KeyValueContainerData containerData = new KeyValueContainerData(1L, layout, (long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(), dd.getUuidString());
        Container container = new KeyValueContainer(containerData, conf);
        container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), scmId.toString());
        containerSet.addContainer(container);
        ContainerMetrics metrics = ContainerMetrics.create(conf);
        Map<ContainerType, Handler> handlers = Maps.newHashMap();
        for (ContainerType containerType : ContainerType.values()) {
            handlers.put(containerType, Handler.getHandlerForContainerType(containerType, conf, context.getParent().getDatanodeDetails().getUuidString(), containerSet, volumeSet, metrics, NO_OP_ICR_SENDER));
        }
        HddsDispatcher hddsDispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers, context, metrics, null);
        hddsDispatcher.setClusterId(scmId.toString());
        ContainerCommandResponseProto responseOne = hddsDispatcher.dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 1L), null);
        Assert.assertEquals(ContainerProtos.Result.SUCCESS, responseOne.getResult());
        verify(context, times(0)).addContainerActionIfAbsent(Mockito.any(ContainerAction.class));
        containerData.setBytesUsed(Double.valueOf(StorageUnit.MB.toBytes(950)).longValue());
        ContainerCommandResponseProto responseTwo = hddsDispatcher.dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 2L), null);
        Assert.assertEquals(ContainerProtos.Result.SUCCESS, responseTwo.getResult());
        verify(context, times(1)).addContainerActionIfAbsent(Mockito.any(ContainerAction.class));
    } finally {
        volumeSet.shutdown();
        ContainerMetrics.remove();
        FileUtils.deleteDirectory(new File(testDir));
    }
}

Example 5 with RoundRobinVolumeChoosingPolicy

use of org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy in project ozone by apache.

the class TestOzoneContainer method setUp.

@Before
public void setUp() throws Exception {
    conf = new OzoneConfiguration();
    conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, folder.getRoot().getAbsolutePath());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().getAbsolutePath());
    commitSpaceMap = new HashMap<String, Long>();
    volumeSet = new MutableVolumeSet(datanodeDetails.getUuidString(), conf, null, StorageVolume.VolumeType.DATA_VOLUME, null);
    volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
}