Examples with ErasureCoderOptions org.apache.hadoop.io.erasurecode.ErasureCoderOptions used on opensource projects

Example 1 with ErasureCoderOptions

use of org.apache.hadoop.io.erasurecode.ErasureCoderOptions in project hadoop by apache.

the class StripedReconstructor method initDecoderIfNecessary.

// Initialize decoder
protected void initDecoderIfNecessary() {
    if (decoder == null) {
        ErasureCoderOptions coderOptions = new ErasureCoderOptions(ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits());
        decoder = CodecUtil.createRawDecoder(conf, ecPolicy.getCodecName(), coderOptions);
    }
}

Example 2 with ErasureCoderOptions

use of org.apache.hadoop.io.erasurecode.ErasureCoderOptions in project hadoop by apache.

the class StripedFileTestUtil method verifyParityBlocks.

static void verifyParityBlocks(Configuration conf, final long size, final int cellSize, byte[][] dataBytes, byte[][] parityBytes, Set<Integer> checkSet, String codecName) {
    // verify the parity blocks
    int parityBlkSize = (int) StripedBlockUtil.getInternalBlockLength(size, cellSize, dataBytes.length, dataBytes.length);
    final byte[][] expectedParityBytes = new byte[parityBytes.length][];
    for (int i = 0; i < parityBytes.length; i++) {
        expectedParityBytes[i] = new byte[parityBlkSize];
    }
    for (int i = 0; i < dataBytes.length; i++) {
        if (dataBytes[i] == null) {
            dataBytes[i] = new byte[dataBytes[0].length];
        } else if (dataBytes[i].length < dataBytes[0].length) {
            final byte[] tmp = dataBytes[i];
            dataBytes[i] = new byte[dataBytes[0].length];
            System.arraycopy(tmp, 0, dataBytes[i], 0, tmp.length);
        }
    }
    ErasureCoderOptions coderOptions = new ErasureCoderOptions(dataBytes.length, parityBytes.length);
    final RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf, codecName, coderOptions);
    encoder.encode(dataBytes, expectedParityBytes);
    for (int i = 0; i < parityBytes.length; i++) {
        if (checkSet.contains(i + dataBytes.length)) {
            Assert.assertArrayEquals("i=" + i, expectedParityBytes[i], parityBytes[i]);
        }
    }
}

Example 3 with ErasureCoderOptions

use of org.apache.hadoop.io.erasurecode.ErasureCoderOptions in project hadoop by apache.

the class TestErasureCoderBase method createEncoder.

/**
   * Create the raw erasure encoder to test
   * @return
   */
protected ErasureCoder createEncoder() {
    ErasureCoder encoder;
    try {
        ErasureCoderOptions options = new ErasureCoderOptions(numDataUnits, numParityUnits, allowChangeInputs, allowDump);
        Constructor<? extends ErasureCoder> constructor = (Constructor<? extends ErasureCoder>) encoderClass.getConstructor(ErasureCoderOptions.class);
        encoder = constructor.newInstance(options);
    } catch (Exception e) {
        throw new RuntimeException("Failed to create encoder", e);
    }
    encoder.setConf(getConf());
    return encoder;
}

Example 4 with ErasureCoderOptions

use of org.apache.hadoop.io.erasurecode.ErasureCoderOptions in project hadoop by apache.

the class TestErasureCoderBase method createDecoder.

/**
   * create the raw erasure decoder to test
   * @return
   */
protected ErasureCoder createDecoder() {
    ErasureCoder decoder;
    try {
        ErasureCoderOptions options = new ErasureCoderOptions(numDataUnits, numParityUnits, allowChangeInputs, allowDump);
        Constructor<? extends ErasureCoder> constructor = (Constructor<? extends ErasureCoder>) decoderClass.getConstructor(ErasureCoderOptions.class);
        decoder = constructor.newInstance(options);
    } catch (Exception e) {
        throw new RuntimeException("Failed to create decoder", e);
    }
    decoder.setConf(getConf());
    return decoder;
}

Example 5 with ErasureCoderOptions

use of org.apache.hadoop.io.erasurecode.ErasureCoderOptions in project hadoop by apache.

the class TestDFSStripedInputStream method testPreadWithDNFailure.

@Test
public void testPreadWithDNFailure() throws Exception {
    final int numBlocks = 4;
    final int failedDNIdx = dataBlocks - 1;
    DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks, stripesPerBlock, false, ecPolicy);
    LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(filePath.toString(), 0, blockGroupSize);
    assert lbs.get(0) instanceof LocatedStripedBlock;
    LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
    for (int i = 0; i < dataBlocks + parityBlocks; i++) {
        Block blk = new Block(bg.getBlock().getBlockId() + i, stripesPerBlock * cellSize, bg.getBlock().getGenerationStamp());
        blk.setGenerationStamp(bg.getBlock().getGenerationStamp());
        cluster.injectBlocks(i, Arrays.asList(blk), bg.getBlock().getBlockPoolId());
    }
    DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(), filePath.toString(), false, ecPolicy, null);
    int readSize = blockGroupSize;
    byte[] readBuffer = new byte[readSize];
    byte[] expected = new byte[readSize];
    /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
    for (int i = 0; i < stripesPerBlock; i++) {
        for (int j = 0; j < dataBlocks; j++) {
            for (int k = 0; k < cellSize; k++) {
                int posInBlk = i * cellSize + k;
                int posInFile = i * cellSize * dataBlocks + j * cellSize + k;
                expected[posInFile] = SimulatedFSDataset.simulatedByte(new Block(bg.getBlock().getBlockId() + j), posInBlk);
            }
        }
    }
    ErasureCoderOptions coderOptions = new ErasureCoderOptions(dataBlocks, parityBlocks);
    RawErasureDecoder rawDecoder = CodecUtil.createRawDecoder(conf, ecPolicy.getCodecName(), coderOptions);
    // Update the expected content for decoded data
    int[] missingBlkIdx = new int[parityBlocks];
    for (int i = 0; i < missingBlkIdx.length; i++) {
        if (i == 0) {
            missingBlkIdx[i] = failedDNIdx;
        } else {
            missingBlkIdx[i] = dataBlocks + i;
        }
    }
    cluster.stopDataNode(failedDNIdx);
    for (int i = 0; i < stripesPerBlock; i++) {
        byte[][] decodeInputs = new byte[dataBlocks + parityBlocks][cellSize];
        byte[][] decodeOutputs = new byte[missingBlkIdx.length][cellSize];
        for (int j = 0; j < dataBlocks; j++) {
            int posInBuf = i * cellSize * dataBlocks + j * cellSize;
            if (j != failedDNIdx) {
                System.arraycopy(expected, posInBuf, decodeInputs[j], 0, cellSize);
            }
        }
        for (int j = dataBlocks; j < dataBlocks + parityBlocks; j++) {
            for (int k = 0; k < cellSize; k++) {
                int posInBlk = i * cellSize + k;
                decodeInputs[j][k] = SimulatedFSDataset.simulatedByte(new Block(bg.getBlock().getBlockId() + j), posInBlk);
            }
        }
        for (int m : missingBlkIdx) {
            decodeInputs[m] = null;
        }
        rawDecoder.decode(decodeInputs, missingBlkIdx, decodeOutputs);
        int posInBuf = i * cellSize * dataBlocks + failedDNIdx * cellSize;
        System.arraycopy(decodeOutputs[0], 0, expected, posInBuf, cellSize);
    }
    int delta = 10;
    int done = 0;
    // read a small delta, shouldn't trigger decode
    // |cell_0 |
    // |10     |
    done += in.read(0, readBuffer, 0, delta);
    assertEquals(delta, done);
    assertArrayEquals(Arrays.copyOf(expected, done), Arrays.copyOf(readBuffer, done));
    // both head and trail cells are partial
    // |c_0      |c_1    |c_2 |c_3 |c_4      |c_5         |
    // |256K - 10|missing|256K|256K|256K - 10|not in range|
    done += in.read(delta, readBuffer, delta, cellSize * (dataBlocks - 1) - 2 * delta);
    assertEquals(cellSize * (dataBlocks - 1) - delta, done);
    assertArrayEquals(Arrays.copyOf(expected, done), Arrays.copyOf(readBuffer, done));
    // read the rest
    done += in.read(done, readBuffer, done, readSize - done);
    assertEquals(readSize, done);
    assertArrayEquals(expected, readBuffer);
}