Examples with ByteArrayCursor herddb.utils.ByteArrayCursor used on opensource projects

Example 6 with ByteArrayCursor

use of herddb.utils.ByteArrayCursor in project herddb by diennea.

the class FileDataStorageManager method readIndexPage.

private <X> X readIndexPage(DataReader<X> reader, Path pageFile, InputStream stream) throws IOException, DataStorageManagerException {
    int size = (int) Files.size(pageFile);
    byte[] dataPage = new byte[size];
    int read = stream.read(dataPage);
    if (read != size) {
        throw new IOException("short read, read " + read + " instead of " + size + " bytes from " + pageFile);
    }
    try (ByteArrayCursor dataIn = ByteArrayCursor.wrap(dataPage)) {
        /*
             * When writing with O_DIRECT this stream will be zero padded at the end. It isn't a problem: reader
             * must already handle his stop condition without reading file till the end because it contains an
             * hash after data end that must not be read by the reader.
             */
        // version
        long version = dataIn.readVLong();
        // flags for future implementations
        long flags = dataIn.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted data file " + pageFile.toAbsolutePath());
        }
        X result = reader.read(dataIn);
        int pos = dataIn.getPosition();
        long hashFromFile = dataIn.readLong();
        if (hashChecksEnabled && hashFromFile != NO_HASH_PRESENT) {
            // after the hash we will have zeroes or garbage
            // the hash is not at the end of file, but after data
            long hashFromDigest = XXHash64Utils.hash(dataPage, 0, pos);
            if (hashFromDigest != hashFromFile) {
                throw new DataStorageManagerException("Corrupted datafile " + pageFile + ". Bad hash " + hashFromFile + " <> " + hashFromDigest);
            }
        }
        return result;
    }
}

Example 7 with ByteArrayCursor

use of herddb.utils.ByteArrayCursor in project herddb by diennea.

the class BookKeeperDataStorageManager method rawReadDataPage.

private static List<Record> rawReadDataPage(byte[] dataPage) throws IOException, DataStorageManagerException {
    try (ByteArrayCursor dataIn = ByteArrayCursor.wrap(dataPage)) {
        // version
        long version = dataIn.readVLong();
        // flags for future implementations
        long flags = dataIn.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted data");
        }
        int numRecords = dataIn.readInt();
        List<Record> result = new ArrayList<>(numRecords);
        for (int i = 0; i < numRecords; i++) {
            Bytes key = dataIn.readBytesNoCopy();
            Bytes value = dataIn.readBytesNoCopy();
            result.add(new Record(key, value));
        }
        int pos = dataIn.getPosition();
        long hashFromFile = dataIn.readLong();
        // after the hash we will have zeroes or garbage
        // the hash is not at the end of file, but after data
        long hashFromDigest = XXHash64Utils.hash(dataPage, 0, pos);
        if (hashFromDigest != hashFromFile) {
            throw new DataStorageManagerException("Corrupted datafile. Bad hash " + hashFromFile + " <> " + hashFromDigest);
        }
        return result;
    }
}

Example 8 with ByteArrayCursor

use of herddb.utils.ByteArrayCursor in project herddb by diennea.

the class BookKeeperDataStorageManager method readIndexPage.

private static <X> X readIndexPage(byte[] dataPage, DataReader<X> reader) throws IOException, DataStorageManagerException {
    try (ByteArrayCursor dataIn = ByteArrayCursor.wrap(dataPage)) {
        // version
        long version = dataIn.readVLong();
        // flags for future implementations
        long flags = dataIn.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted data file");
        }
        X result = reader.read(dataIn);
        int pos = dataIn.getPosition();
        long hashFromFile = dataIn.readLong();
        // after the hash we will have zeroes or garbage
        // the hash is not at the end of file, but after data
        long hashFromDigest = XXHash64Utils.hash(dataPage, 0, pos);
        if (hashFromDigest != hashFromFile) {
            throw new DataStorageManagerException("Corrupted datafile . Bad hash " + hashFromFile + " <> " + hashFromDigest);
        }
        return result;
    }
}

Example 9 with ByteArrayCursor

use of herddb.utils.ByteArrayCursor in project herddb by diennea.

the class FileDataStorageManager method rawReadDataPage.

private List<Record> rawReadDataPage(Path pageFile, InputStream stream) throws IOException, DataStorageManagerException {
    int size = (int) Files.size(pageFile);
    byte[] dataPage = new byte[size];
    int read = stream.read(dataPage);
    if (read != size) {
        throw new IOException("short read, read " + read + " instead of " + size + " bytes from " + pageFile);
    }
    try (ByteArrayCursor dataIn = ByteArrayCursor.wrap(dataPage)) {
        // version
        long version = dataIn.readVLong();
        // flags for future implementations
        long flags = dataIn.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted data file " + pageFile.toAbsolutePath());
        }
        int numRecords = dataIn.readInt();
        List<Record> result = new ArrayList<>(numRecords);
        for (int i = 0; i < numRecords; i++) {
            Bytes key = dataIn.readBytesNoCopy();
            Bytes value = dataIn.readBytesNoCopy();
            result.add(new Record(key, value));
        }
        int pos = dataIn.getPosition();
        long hashFromFile = dataIn.readLong();
        if (hashChecksEnabled && hashFromFile != NO_HASH_PRESENT) {
            // after the hash we will have zeroes or garbage
            // the hash is not at the end of file, but after data
            long hashFromDigest = XXHash64Utils.hash(dataPage, 0, pos);
            if (hashFromDigest != hashFromFile) {
                throw new DataStorageManagerException("Corrupted datafile " + pageFile + ". Bad hash " + hashFromFile + " <> " + hashFromDigest);
            }
        }
        return result;
    }
}

Example 10 with ByteArrayCursor

use of herddb.utils.ByteArrayCursor in project herddb by diennea.

the class Tuple method deserialize.

public static Tuple deserialize(final byte[] data, final String[] fieldNames, final int nColumns) throws IOException {
    try (ByteArrayCursor eoo = ByteArrayCursor.wrap(data)) {
        List<Object> values = new ArrayList<>();
        for (int i = 0; i < nColumns; i++) {
            Object value = RecordSerializer.deserializeTypeAndValue(eoo);
            values.add(value);
        }
        Object[] _values = values.toArray(new Object[nColumns]);
        return new Tuple(fieldNames, _values);
    }
}