Examples with RandomDatum org.apache.hadoop.io.RandomDatum used on opensource projects

Example 1 with RandomDatum

use of org.apache.hadoop.io.RandomDatum in project hadoop by apache.

the class CryptoStreamsTestBase method setUp.

@Before
public void setUp() throws IOException {
    // Generate data
    final int seed = new Random().nextInt();
    final DataOutputBuffer dataBuf = new DataOutputBuffer();
    final RandomDatum.Generator generator = new RandomDatum.Generator(seed);
    for (int i = 0; i < count; ++i) {
        generator.next();
        final RandomDatum key = generator.getKey();
        final RandomDatum value = generator.getValue();
        key.write(dataBuf);
        value.write(dataBuf);
    }
    LOG.info("Generated " + count + " records");
    data = dataBuf.getData();
    dataLen = dataBuf.getLength();
}

Example 2 with RandomDatum

use of org.apache.hadoop.io.RandomDatum in project hadoop by apache.

the class TestCryptoCodec method cryptoCodecTest.

private void cryptoCodecTest(Configuration conf, int seed, int count, String encCodecClass, String decCodecClass, byte[] iv) throws IOException, GeneralSecurityException {
    CryptoCodec encCodec = null;
    try {
        encCodec = (CryptoCodec) ReflectionUtils.newInstance(conf.getClassByName(encCodecClass), conf);
    } catch (ClassNotFoundException cnfe) {
        throw new IOException("Illegal crypto codec!");
    }
    LOG.info("Created a Codec object of type: " + encCodecClass);
    // Generate data
    DataOutputBuffer data = new DataOutputBuffer();
    RandomDatum.Generator generator = new RandomDatum.Generator(seed);
    for (int i = 0; i < count; ++i) {
        generator.next();
        RandomDatum key = generator.getKey();
        RandomDatum value = generator.getValue();
        key.write(data);
        value.write(data);
    }
    LOG.info("Generated " + count + " records");
    // Encrypt data
    DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer();
    CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer, encCodec, bufferSize, key, iv);
    out.write(data.getData(), 0, data.getLength());
    out.flush();
    out.close();
    LOG.info("Finished encrypting data");
    CryptoCodec decCodec = null;
    try {
        decCodec = (CryptoCodec) ReflectionUtils.newInstance(conf.getClassByName(decCodecClass), conf);
    } catch (ClassNotFoundException cnfe) {
        throw new IOException("Illegal crypto codec!");
    }
    LOG.info("Created a Codec object of type: " + decCodecClass);
    // Decrypt data
    DataInputBuffer decryptedDataBuffer = new DataInputBuffer();
    decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, encryptedDataBuffer.getLength());
    CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize, key, iv);
    DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in));
    // Check
    DataInputBuffer originalData = new DataInputBuffer();
    originalData.reset(data.getData(), 0, data.getLength());
    DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData));
    for (int i = 0; i < count; ++i) {
        RandomDatum k1 = new RandomDatum();
        RandomDatum v1 = new RandomDatum();
        k1.readFields(originalIn);
        v1.readFields(originalIn);
        RandomDatum k2 = new RandomDatum();
        RandomDatum v2 = new RandomDatum();
        k2.readFields(dataIn);
        v2.readFields(dataIn);
        assertTrue("original and encrypted-then-decrypted-output not equal", k1.equals(k2) && v1.equals(v2));
        // original and encrypted-then-decrypted-output have the same hashCode
        Map<RandomDatum, String> m = new HashMap<RandomDatum, String>();
        m.put(k1, k1.toString());
        m.put(v1, v1.toString());
        String result = m.get(k2);
        assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
        result = m.get(v2);
        assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
    }
    // Decrypt data byte-at-a-time
    originalData.reset(data.getData(), 0, data.getLength());
    decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, encryptedDataBuffer.getLength());
    in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize, key, iv);
    // Check
    originalIn = new DataInputStream(new BufferedInputStream(originalData));
    int expected;
    do {
        expected = originalIn.read();
        assertEquals("Decrypted stream read by byte does not match", expected, in.read());
    } while (expected != -1);
    // Seek to a certain position and decrypt
    originalData.reset(data.getData(), 0, data.getLength());
    decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, encryptedDataBuffer.getLength());
    in = new CryptoInputStream(new TestCryptoStreams.FakeInputStream(decryptedDataBuffer), decCodec, bufferSize, key, iv);
    int seekPos = data.getLength() / 3;
    in.seek(seekPos);
    // Check
    TestCryptoStreams.FakeInputStream originalInput = new TestCryptoStreams.FakeInputStream(originalData);
    originalInput.seek(seekPos);
    do {
        expected = originalInput.read();
        assertEquals("Decrypted stream read by byte does not match", expected, in.read());
    } while (expected != -1);
    LOG.info("SUCCESS! Completed checking " + count + " records");
    // Check secure random generator
    testSecureRandom(encCodec);
}

Example 3 with RandomDatum

use of org.apache.hadoop.io.RandomDatum in project SSM by Intel-bigdata.

the class TestArrayFile method testArrayFile.

@Test
public void testArrayFile() throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.getLocal(conf);
    RandomDatum[] data = generate(10000);
    writeTest(fs, data, TEST_FILE);
    readTest(fs, data, TEST_FILE, conf);
}

Example 4 with RandomDatum

use of org.apache.hadoop.io.RandomDatum in project SSM by Intel-bigdata.

the class TestArrayFile method readTest.

private static void readTest(FileSystem fs, RandomDatum[] data, String file, Configuration conf) throws IOException {
    RandomDatum v = new RandomDatum();
    if (LOG.isDebugEnabled()) {
        LOG.debug("reading " + data.length + " debug");
    }
    ArrayFile.Reader reader = new ArrayFile.Reader(fs, file, conf);
    try {
        for (int i = 0; i < data.length; i++) {
            // try forwards
            reader.get(i, v);
            if (!v.equals(data[i])) {
                throw new RuntimeException("wrong value at " + i);
            }
        }
        for (int i = data.length - 1; i >= 0; i--) {
            // then backwards
            reader.get(i, v);
            if (!v.equals(data[i])) {
                throw new RuntimeException("wrong value at " + i);
            }
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("done reading " + data.length + " debug");
        }
    } finally {
        reader.close();
    }
}

Example 5 with RandomDatum

use of org.apache.hadoop.io.RandomDatum in project SSM by Intel-bigdata.

the class TestArrayFile method testEmptyFile.

@Test
public void testEmptyFile() throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.getLocal(conf);
    writeTest(fs, new RandomDatum[0], TEST_FILE);
    ArrayFile.Reader reader = new ArrayFile.Reader(fs, TEST_FILE, conf);
    assertNull(reader.get(0, new RandomDatum()));
    reader.close();
}