Examples with RawErasureEncoder org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder used on opensource projects

Example 1 with RawErasureEncoder

use of org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder in project hadoop by apache.

the class StripedFileTestUtil method verifyParityBlocks.

static void verifyParityBlocks(Configuration conf, final long size, final int cellSize, byte[][] dataBytes, byte[][] parityBytes, Set<Integer> checkSet, String codecName) {
    // verify the parity blocks
    int parityBlkSize = (int) StripedBlockUtil.getInternalBlockLength(size, cellSize, dataBytes.length, dataBytes.length);
    final byte[][] expectedParityBytes = new byte[parityBytes.length][];
    for (int i = 0; i < parityBytes.length; i++) {
        expectedParityBytes[i] = new byte[parityBlkSize];
    }
    for (int i = 0; i < dataBytes.length; i++) {
        if (dataBytes[i] == null) {
            dataBytes[i] = new byte[dataBytes[0].length];
        } else if (dataBytes[i].length < dataBytes[0].length) {
            final byte[] tmp = dataBytes[i];
            dataBytes[i] = new byte[dataBytes[0].length];
            System.arraycopy(tmp, 0, dataBytes[i], 0, tmp.length);
        }
    }
    ErasureCoderOptions coderOptions = new ErasureCoderOptions(dataBytes.length, parityBytes.length);
    final RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf, codecName, coderOptions);
    encoder.encode(dataBytes, expectedParityBytes);
    for (int i = 0; i < parityBytes.length; i++) {
        if (checkSet.contains(i + dataBytes.length)) {
            Assert.assertArrayEquals("i=" + i, expectedParityBytes[i], parityBytes[i]);
        }
    }
}

Example 2 with RawErasureEncoder

use of org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder in project hadoop by apache.

the class TestCodecRawCoderMapping method testRSDefaultRawCoder.

@Test
public void testRSDefaultRawCoder() {
    ErasureCoderOptions coderOptions = new ErasureCoderOptions(numDataUnit, numParityUnit);
    // should return default raw coder of rs codec
    RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
    Assert.assertTrue(encoder instanceof RSRawEncoder);
    RawErasureDecoder decoder = CodecUtil.createRawDecoder(conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
    Assert.assertTrue(decoder instanceof RSRawDecoder);
    // should return default raw coder of rs-legacy codec
    encoder = CodecUtil.createRawEncoder(conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
    Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
    decoder = CodecUtil.createRawDecoder(conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
    Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
}

Example 3 with RawErasureEncoder

use of org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder in project hadoop by apache.

the class DummyErasureEncoder method prepareEncodingStep.

@Override
protected ErasureCodingStep prepareEncodingStep(ECBlockGroup blockGroup) {
    RawErasureEncoder rawEncoder = new DummyRawEncoder(getOptions());
    ECBlock[] inputBlocks = getInputBlocks(blockGroup);
    return new ErasureEncodingStep(inputBlocks, getOutputBlocks(blockGroup), rawEncoder);
}

Example 4 with RawErasureEncoder

use of org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder in project hadoop by apache.

the class HHXORErasureDecoder method prepareDecodingStep.

@Override
protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {
    RawErasureDecoder rawDecoder;
    RawErasureEncoder rawEncoder;
    ECBlock[] inputBlocks = getInputBlocks(blockGroup);
    ECBlock[] outputBlocks = getOutputBlocks(blockGroup);
    rawDecoder = checkCreateRSRawDecoder();
    rawEncoder = checkCreateXorRawEncoder();
    return new HHXORErasureDecodingStep(inputBlocks, getErasedIndexes(inputBlocks), outputBlocks, rawDecoder, rawEncoder);
}

Example 5 with RawErasureEncoder

use of org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder in project hadoop by apache.

the class HHXORErasureEncoder method prepareEncodingStep.

@Override
protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {
    RawErasureEncoder rsRawEncoderTmp = checkCreateRSRawEncoder();
    RawErasureEncoder xorRawEncoderTmp = checkCreateXorRawEncoder();
    ECBlock[] inputBlocks = getInputBlocks(blockGroup);
    return new HHXORErasureEncodingStep(inputBlocks, getOutputBlocks(blockGroup), rsRawEncoderTmp, xorRawEncoderTmp);
}