Examples with DataInputStream java.io.DataInputStream used on opensource projects

Example 16 with DataInputStream

use of java.io.DataInputStream in project pinot by linkedin.

the class DataTableImplV2 method deserializeDictionaryMap.

private Map<String, Map<Integer, String>> deserializeDictionaryMap(byte[] bytes) throws IOException {
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
    DataInputStream dataInputStream = new DataInputStream(byteArrayInputStream);
    int numDictionaries = dataInputStream.readInt();
    Map<String, Map<Integer, String>> dictionaryMap = new HashMap<>(numDictionaries);
    int readLength;
    for (int i = 0; i < numDictionaries; i++) {
        int columnNameLength = dataInputStream.readInt();
        byte[] columnNameBytes = new byte[columnNameLength];
        readLength = dataInputStream.read(columnNameBytes);
        assert readLength == columnNameLength;
        Map<Integer, String> dictionary = new HashMap<>();
        dictionaryMap.put(new String(columnNameBytes, UTF_8), dictionary);
        int dictionarySize = dataInputStream.readInt();
        for (int j = 0; j < dictionarySize; j++) {
            int key = dataInputStream.readInt();
            int valueLength = dataInputStream.readInt();
            byte[] valueBytes = new byte[valueLength];
            readLength = dataInputStream.read(valueBytes);
            assert readLength == valueLength;
            dictionary.put(key, new String(valueBytes, UTF_8));
        }
    }
    return dictionaryMap;
}

Example 17 with DataInputStream

use of java.io.DataInputStream in project pinot by linkedin.

the class DataTableImplV2 method deserializeMetadata.

private Map<String, String> deserializeMetadata(byte[] bytes) throws IOException {
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
    DataInputStream dataInputStream = new DataInputStream(byteArrayInputStream);
    int numEntries = dataInputStream.readInt();
    Map<String, String> metadata = new HashMap<>(numEntries);
    int readLength;
    for (int i = 0; i < numEntries; i++) {
        int keyLength = dataInputStream.readInt();
        byte[] keyBytes = new byte[keyLength];
        readLength = dataInputStream.read(keyBytes);
        assert readLength == keyLength;
        int valueLength = dataInputStream.readInt();
        byte[] valueBytes = new byte[valueLength];
        readLength = dataInputStream.read(valueBytes);
        assert readLength == valueLength;
        metadata.put(new String(keyBytes, UTF_8), new String(valueBytes, UTF_8));
    }
    return metadata;
}

Example 18 with DataInputStream

use of java.io.DataInputStream in project pinot by linkedin.

the class FixedByteSkipListSCMVWriterTest method testSingleColMultiValue.

@Test
public void testSingleColMultiValue() throws Exception {
    File file = new File("test_single_col_multi_value_writer.dat");
    file.delete();
    int rows = 100;
    int[][] data = new int[rows][];
    Random r = new Random();
    int totalNumValues = 0;
    for (int i = 0; i < rows; i++) {
        int numValues = r.nextInt(100) + 1;
        data[i] = new int[numValues];
        for (int j = 0; j < numValues; j++) {
            data[i][j] = r.nextInt();
        }
        totalNumValues += numValues;
    }
    FixedByteSkipListMultiValueWriter writer = new FixedByteSkipListMultiValueWriter(file, rows, totalNumValues, 4);
    int numChunks = writer.getNumChunks();
    int[] chunkOffsets = new int[numChunks];
    int chunkId = 0;
    int offset = 0;
    for (int i = 0; i < rows; i++) {
        writer.setIntArray(i, data[i]);
        if (i % writer.getDocsPerChunk() == 0) {
            chunkOffsets[chunkId] = offset;
            chunkId = chunkId + 1;
        }
        offset += data[i].length;
    }
    writer.close();
    DataInputStream dis = new DataInputStream(new FileInputStream(file));
    for (int i = 0; i < numChunks; i++) {
        Assert.assertEquals(dis.readInt(), chunkOffsets[i]);
    }
    int numBytesForBitmap = (totalNumValues + 7) / 8;
    byte[] bitsetBytes = new byte[numBytesForBitmap];
    dis.read(bitsetBytes);
    CustomBitSet customBit = CustomBitSet.withByteBuffer(numBytesForBitmap, ByteBuffer.wrap(bitsetBytes));
    offset = 0;
    //    System.out.println("getChunkOffsetHeaderSize:" + writer.getChunkOffsetHeaderSize());
    for (int i = 0; i < rows; i++) {
        Assert.assertTrue(customBit.isBitSet(offset));
        for (int j = 0; j < data[i].length; j++) {
            Assert.assertEquals(dis.readInt(), data[i][j]);
        }
        offset += data[i].length;
    }
    dis.close();
    file.delete();
    //    raf.close();
    customBit.close();
}

Example 19 with DataInputStream

use of java.io.DataInputStream in project head by mifos.

the class MpesaImportTest method checkIfOutputMatchesExpected.

private void checkIfOutputMatchesExpected(String path) throws FileNotFoundException, IOException, URISyntaxException {
    DataInputStream in = new DataInputStream(new FileInputStream(new File(new URI(path + ".expected.txt"))));
    BufferedReader br = new BufferedReader(new InputStreamReader(in));
    String line = null;
    while (true) {
        line = br.readLine();
        if (line == null) {
            break;
        }
        if (!selenium.isTextPresent(line.trim())) {
            Assert.fail("No text <" + line.trim() + "> present on the page");
        }
    }
}

Example 20 with DataInputStream

use of java.io.DataInputStream in project mavuno by metzlerd.

the class Tuple method readFields.

/**
	 * Deserializes the Tuple.
	 * 
	 * @param in
	 *            source for raw byte representation
	 */
public void readFields(DataInput in) throws IOException {
    int numFields = in.readInt();
    mObjects = new Object[numFields];
    mSymbols = new String[numFields];
    mFields = new String[numFields];
    mTypes = new Class[numFields];
    for (int i = 0; i < numFields; i++) {
        mFields[i] = in.readUTF();
    }
    for (int i = 0; i < numFields; i++) {
        byte type = in.readByte();
        if (type == SYMBOL) {
            String className = in.readUTF();
            try {
                mTypes[i] = Class.forName(className);
            } catch (Exception e) {
                e.printStackTrace();
            }
            mObjects[i] = null;
            mSymbols[i] = in.readUTF();
        } else if (type == INT) {
            mTypes[i] = Integer.class;
            mObjects[i] = in.readInt();
        } else if (type == BOOLEAN) {
            mTypes[i] = Boolean.class;
            mObjects[i] = in.readBoolean();
        } else if (type == LONG) {
            mTypes[i] = Long.class;
            mObjects[i] = in.readLong();
        } else if (type == FLOAT) {
            mTypes[i] = Float.class;
            mObjects[i] = in.readFloat();
        } else if (type == DOUBLE) {
            mTypes[i] = Double.class;
            mObjects[i] = in.readDouble();
        } else if (type == STRING) {
            mTypes[i] = String.class;
            mObjects[i] = in.readUTF();
        } else {
            try {
                String className = in.readUTF();
                mTypes[i] = Class.forName(className);
                int sz = in.readInt();
                byte[] bytes = new byte[sz];
                in.readFully(bytes);
                Writable obj = (Writable) mTypes[i].newInstance();
                obj.readFields(new DataInputStream(new ByteArrayInputStream(bytes)));
                mObjects[i] = obj;
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}