Examples with KeyValueContainer org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer used on opensource projects

Example 1 with KeyValueContainer

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer in project ozone by apache.

the class ChunkManagerDiskWrite method call.

@Override
public Void call() throws Exception {
    try {
        init();
        OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
        VolumeSet volumeSet = new MutableVolumeSet("dnid", "clusterid", ozoneConfiguration, null, StorageVolume.VolumeType.DATA_VOLUME, null);
        Random random = new Random();
        VolumeChoosingPolicy volumeChoicePolicy = new RoundRobinVolumeChoosingPolicy();
        final int threadCount = getThreadNo();
        // create a dedicated (NEW) container for each thread
        for (int i = 1; i <= threadCount; i++) {
            // use a non-negative container id
            long containerId = random.nextLong() & 0x0F_FF_FF_FF_FF_FF_FF_FFL;
            KeyValueContainerData keyValueContainerData = new KeyValueContainerData(containerId, containerLayout, 1_000_000L, getPrefix(), "nodeid");
            KeyValueContainer keyValueContainer = new KeyValueContainer(keyValueContainerData, ozoneConfiguration);
            keyValueContainer.create(volumeSet, volumeChoicePolicy, "scmid");
            containersPerThread.put(i, keyValueContainer);
        }
        blockSize = chunkSize * chunksPerBlock;
        data = randomAscii(chunkSize).getBytes(UTF_8);
        chunkManager = ChunkManagerFactory.createChunkManager(ozoneConfiguration, null, null);
        timer = getMetrics().timer("chunk-write");
        LOG.info("Running chunk write test: threads={} chunkSize={} " + "chunksPerBlock={} layout={}", threadCount, chunkSize, chunksPerBlock, containerLayout);
        runTests(this::writeChunk);
    } finally {
        if (chunkManager != null) {
            chunkManager.shutdown();
        }
    }
    return null;
}

Example 2 with KeyValueContainer

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer in project ozone by apache.

the class FilePerChunkStrategy method writeChunk.

/**
 * writes a given chunk.
 *
 * @param container - Container for the chunk
 * @param blockID - ID of the block
 * @param info - ChunkInfo
 * @param data - data of the chunk
 * @param dispatcherContext - dispatcherContextInfo
 * @throws StorageContainerException
 */
@Override
public void writeChunk(Container container, BlockID blockID, ChunkInfo info, ChunkBuffer data, DispatcherContext dispatcherContext) throws StorageContainerException {
    checkLayoutVersion(container);
    Preconditions.checkNotNull(dispatcherContext);
    DispatcherContext.WriteChunkStage stage = dispatcherContext.getStage();
    try {
        KeyValueContainer kvContainer = (KeyValueContainer) container;
        KeyValueContainerData containerData = kvContainer.getContainerData();
        HddsVolume volume = containerData.getVolume();
        File chunkFile = getChunkFile(kvContainer, blockID, info);
        boolean isOverwrite = ChunkUtils.validateChunkForOverwrite(chunkFile, info);
        File tmpChunkFile = getTmpChunkFile(chunkFile, dispatcherContext);
        if (LOG.isDebugEnabled()) {
            LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file:{}", info.getChunkName(), stage, chunkFile, tmpChunkFile);
        }
        long len = info.getLen();
        // ignore offset in chunk info
        long offset = 0;
        switch(stage) {
            case WRITE_DATA:
                if (isOverwrite) {
                    // if the actual chunk file already exists here while writing the temp
                    // chunk file, then it means the same ozone client request has
                    // generated two raft log entries. This can happen either because
                    // retryCache expired in Ratis (or log index mismatch/corruption in
                    // Ratis). This can be solved by two approaches as of now:
                    // 1. Read the complete data in the actual chunk file ,
                    // verify the data integrity and in case it mismatches , either
                    // 2. Delete the chunk File and write the chunk again. For now,
                    // let's rewrite the chunk file
                    // TODO: once the checksum support for write chunks gets plugged in,
                    // the checksum needs to be verified for the actual chunk file and
                    // the data to be written here which should be efficient and
                    // it matches we can safely return without rewriting.
                    LOG.warn("ChunkFile already exists {}. Deleting it.", chunkFile);
                    FileUtil.fullyDelete(chunkFile);
                }
                if (tmpChunkFile.exists()) {
                    // If the tmp chunk file already exists it means the raft log got
                    // appended, but later on the log entry got truncated in Ratis leaving
                    // behind garbage.
                    // TODO: once the checksum support for data chunks gets plugged in,
                    // instead of rewriting the chunk here, let's compare the checkSums
                    LOG.warn("tmpChunkFile already exists {}. Overwriting it.", tmpChunkFile);
                }
                // Initially writes to temporary chunk file.
                ChunkUtils.writeData(tmpChunkFile, data, offset, len, volume, doSyncWrite);
                // committed here.
                break;
            case COMMIT_DATA:
                // to actual chunk file.
                if (isOverwrite) {
                    // if the actual chunk file already exists , it implies the write
                    // chunk transaction in the containerStateMachine is getting
                    // reapplied. This can happen when a node restarts.
                    // TODO: verify the checkSums for the existing chunkFile and the
                    // chunkInfo to be committed here
                    LOG.warn("ChunkFile already exists {}", chunkFile);
                    return;
                }
                // While committing a chunk , just rename the tmp chunk file which has
                // the same term and log index appended as the current transaction
                commitChunk(tmpChunkFile, chunkFile);
                // Increment container stats here, as we commit the data.
                containerData.updateWriteStats(len, isOverwrite);
                break;
            case COMBINED:
                // directly write to the chunk file
                ChunkUtils.writeData(chunkFile, data, offset, len, volume, doSyncWrite);
                containerData.updateWriteStats(len, isOverwrite);
                break;
            default:
                throw new IOException("Can not identify write operation.");
        }
    } catch (StorageContainerException ex) {
        throw ex;
    } catch (IOException ex) {
        throw new StorageContainerException("Internal error: ", ex, IO_EXCEPTION);
    }
}

Example 3 with KeyValueContainer

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer in project ozone by apache.

the class TestFailureHandlingByClient method testBlockCountOnFailures.

/**
 * Test whether blockData and Container metadata (block count and used
 * bytes) is updated correctly when there is a write failure.
 * We can combine this test with {@link #testBlockWritesWithDnFailures()}
 * as that test also simulates a write failure and client writes failed
 * chunk writes to a new block.
 */
private void testBlockCountOnFailures(OmKeyInfo omKeyInfo) throws Exception {
    // testBlockWritesWithDnFailures writes chunkSize*2.5 size of data into
    // KeyOutputStream. But before closing the outputStream, 2 of the DNs in
    // the pipeline being written to are closed. So the key will be written
    // to 2 blocks as atleast the last 0.5 chunk would not be committed to the
    // first block before the stream is closed.
    /**
     * There are 3 possible scenarios:
     * 1. Block1 has 2 chunks and OMKeyInfo also has 2 chunks against this block
     *    => Block2 should have 1 chunk
     *    (2 chunks were written to Block1, committed and acknowledged by
     *    CommitWatcher)
     * 2. Block1 has 1 chunk and OMKeyInfo has 1 chunk against this block
     *    => Block2 should have 2 chunks
     *    (Possibly 2 chunks were written but only 1 was committed to the
     *    block)
     * 3. Block1 has 2 chunks but OMKeyInfo has only 1 chunk against this block
     *    => Block2 should have 2 chunks
     *    (This happens when the 2nd chunk has been committed to Block1 but
     *    not acknowledged by CommitWatcher before pipeline shutdown)
     */
    // Get information about the first and second block (in different pipelines)
    List<OmKeyLocationInfo> locationList = omKeyInfo.getLatestVersionLocations().getLocationList();
    long containerId1 = locationList.get(0).getContainerID();
    List<DatanodeDetails> block1DNs = locationList.get(0).getPipeline().getNodes();
    long containerId2 = locationList.get(1).getContainerID();
    List<DatanodeDetails> block2DNs = locationList.get(1).getPipeline().getNodes();
    int block2ExpectedChunkCount;
    if (locationList.get(0).getLength() == 2 * chunkSize) {
        // Scenario 1
        block2ExpectedChunkCount = 1;
    } else {
        // Scenario 2
        block2ExpectedChunkCount = 2;
    }
    // For the first block, first 2 DNs in the pipeline are shutdown (to
    // simulate a failure). It should have 1 or 2 chunks (depending on
    // whether the DN CommitWatcher successfully acknowledged the 2nd chunk
    // write or not). The 3rd chunk would not exist on the first pipeline as
    // the pipeline would be closed before the last 0.5 chunk was committed
    // to the block.
    KeyValueContainerData containerData1 = ((KeyValueContainer) cluster.getHddsDatanode(block1DNs.get(2)).getDatanodeStateMachine().getContainer().getContainerSet().getContainer(containerId1)).getContainerData();
    try (ReferenceCountedDB containerDb1 = BlockUtils.getDB(containerData1, conf)) {
        BlockData blockData1 = containerDb1.getStore().getBlockDataTable().get(Long.toString(locationList.get(0).getBlockID().getLocalID()));
        // The first Block could have 1 or 2 chunkSize of data
        int block1NumChunks = blockData1.getChunks().size();
        Assert.assertTrue(block1NumChunks >= 1);
        Assert.assertEquals(chunkSize * block1NumChunks, blockData1.getSize());
        Assert.assertEquals(1, containerData1.getBlockCount());
        Assert.assertEquals(chunkSize * block1NumChunks, containerData1.getBytesUsed());
    }
    // Verify that the second block has the remaining 0.5*chunkSize of data
    KeyValueContainerData containerData2 = ((KeyValueContainer) cluster.getHddsDatanode(block2DNs.get(0)).getDatanodeStateMachine().getContainer().getContainerSet().getContainer(containerId2)).getContainerData();
    try (ReferenceCountedDB containerDb2 = BlockUtils.getDB(containerData2, conf)) {
        BlockData blockData2 = containerDb2.getStore().getBlockDataTable().get(Long.toString(locationList.get(1).getBlockID().getLocalID()));
        // The second Block should have 0.5 chunkSize of data
        Assert.assertEquals(block2ExpectedChunkCount, blockData2.getChunks().size());
        Assert.assertEquals(1, containerData2.getBlockCount());
        int expectedBlockSize;
        if (block2ExpectedChunkCount == 1) {
            expectedBlockSize = chunkSize / 2;
        } else {
            expectedBlockSize = chunkSize + chunkSize / 2;
        }
        Assert.assertEquals(expectedBlockSize, blockData2.getSize());
        Assert.assertEquals(expectedBlockSize, containerData2.getBytesUsed());
    }
}

Example 4 with KeyValueContainer

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer in project ozone by apache.

the class TestContainerPersistence method addContainer.

private KeyValueContainer addContainer(ContainerSet cSet, long cID) throws IOException {
    long commitBytesBefore = 0;
    long commitBytesAfter = 0;
    long commitIncrement = 0;
    KeyValueContainerData data = new KeyValueContainerData(cID, layout, ContainerTestHelper.CONTAINER_MAX_SIZE, UUID.randomUUID().toString(), UUID.randomUUID().toString());
    data.addMetadata("VOLUME", "shire");
    data.addMetadata("owner)", "bilbo");
    KeyValueContainer container = new KeyValueContainer(data, conf);
    container.create(volumeSet, volumeChoosingPolicy, SCM_ID);
    commitBytesBefore = container.getContainerData().getVolume().getCommittedBytes();
    cSet.addContainer(container);
    commitBytesAfter = container.getContainerData().getVolume().getCommittedBytes();
    commitIncrement = commitBytesAfter - commitBytesBefore;
    // did we commit space for the new container?
    Assert.assertTrue(commitIncrement == ContainerTestHelper.CONTAINER_MAX_SIZE);
    return container;
}

Example 5 with KeyValueContainer

use of org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer in project ozone by apache.

the class TestContainerPersistence method testUpdateContainer.

/**
 * Tries to update an existing and non-existing container. Verifies container
 * map and persistent data both updated.
 *
 * @throws IOException
 */
@Test
public void testUpdateContainer() throws IOException {
    long testContainerID = ContainerTestHelper.getTestContainerID();
    KeyValueContainer container = (KeyValueContainer) addContainer(containerSet, testContainerID);
    File orgContainerFile = container.getContainerFile();
    Assert.assertTrue(orgContainerFile.exists());
    Map<String, String> newMetadata = Maps.newHashMap();
    newMetadata.put("VOLUME", "shire_new");
    newMetadata.put("owner", "bilbo_new");
    container.update(newMetadata, false);
    Assert.assertEquals(1, containerSet.getContainerMapCopy().size());
    Assert.assertTrue(containerSet.getContainerMapCopy().containsKey(testContainerID));
    // Verify in-memory map
    KeyValueContainerData actualNewData = (KeyValueContainerData) containerSet.getContainer(testContainerID).getContainerData();
    Assert.assertEquals("shire_new", actualNewData.getMetadata().get("VOLUME"));
    Assert.assertEquals("bilbo_new", actualNewData.getMetadata().get("owner"));
    // Verify container data on disk
    File containerBaseDir = new File(actualNewData.getMetadataPath()).getParentFile();
    File newContainerFile = ContainerUtils.getContainerFile(containerBaseDir);
    Assert.assertTrue("Container file should exist.", newContainerFile.exists());
    Assert.assertEquals("Container file should be in same location.", orgContainerFile.getAbsolutePath(), newContainerFile.getAbsolutePath());
    ContainerData actualContainerData = ContainerDataYaml.readContainerFile(newContainerFile);
    Assert.assertEquals("shire_new", actualContainerData.getMetadata().get("VOLUME"));
    Assert.assertEquals("bilbo_new", actualContainerData.getMetadata().get("owner"));
    // Test force update flag.
    // Close the container and then try to update without force update flag.
    container.close();
    try {
        container.update(newMetadata, false);
    } catch (StorageContainerException ex) {
        Assert.assertEquals("Updating a closed container without " + "force option is not allowed. ContainerID: " + testContainerID, ex.getMessage());
    }
    // Update with force flag, it should be success.
    newMetadata.put("VOLUME", "shire_new_1");
    newMetadata.put("owner", "bilbo_new_1");
    container.update(newMetadata, true);
    // Verify in-memory map
    actualNewData = (KeyValueContainerData) containerSet.getContainer(testContainerID).getContainerData();
    Assert.assertEquals("shire_new_1", actualNewData.getMetadata().get("VOLUME"));
    Assert.assertEquals("bilbo_new_1", actualNewData.getMetadata().get("owner"));
}