Examples with Bytes herddb.utils.Bytes used on opensource projects

Example 81 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class FileDataStorageManager method rawReadDataPage.

private List<Record> rawReadDataPage(Path pageFile, InputStream stream) throws IOException, DataStorageManagerException {
    int size = (int) Files.size(pageFile);
    byte[] dataPage = new byte[size];
    int read = stream.read(dataPage);
    if (read != size) {
        throw new IOException("short read, read " + read + " instead of " + size + " bytes from " + pageFile);
    }
    try (ByteArrayCursor dataIn = ByteArrayCursor.wrap(dataPage)) {
        // version
        long version = dataIn.readVLong();
        // flags for future implementations
        long flags = dataIn.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted data file " + pageFile.toAbsolutePath());
        }
        int numRecords = dataIn.readInt();
        List<Record> result = new ArrayList<>(numRecords);
        for (int i = 0; i < numRecords; i++) {
            Bytes key = dataIn.readBytesNoCopy();
            Bytes value = dataIn.readBytesNoCopy();
            result.add(new Record(key, value));
        }
        int pos = dataIn.getPosition();
        long hashFromFile = dataIn.readLong();
        if (hashChecksEnabled && hashFromFile != NO_HASH_PRESENT) {
            // after the hash we will have zeroes or garbage
            // the hash is not at the end of file, but after data
            long hashFromDigest = XXHash64Utils.hash(dataPage, 0, pos);
            if (hashFromDigest != hashFromFile) {
                throw new DataStorageManagerException("Corrupted datafile " + pageFile + ". Bad hash " + hashFromFile + " <> " + hashFromDigest);
            }
        }
        return result;
    }
}

Example 82 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class BLinkKeyToPageIndex method checkpoint.

@Override
public List<PostCheckpointAction> checkpoint(LogSequenceNumber sequenceNumber, boolean pin) throws DataStorageManagerException {
    try {
        /* Tree can be null if no data was inserted (tree creation deferred to check evaluate key size) */
        final BLink<Bytes, Long> tree = this.tree;
        if (tree == null) {
            return Collections.emptyList();
        }
        BLinkMetadata<Bytes> metadata = getTree().checkpoint();
        byte[] metaPage = MetadataSerializer.INSTANCE.write(metadata);
        Set<Long> activePages = new HashSet<>();
        metadata.nodes.forEach(node -> activePages.add(node.storeId));
        IndexStatus indexStatus = new IndexStatus(indexName, sequenceNumber, newPageId.get(), activePages, metaPage);
        List<PostCheckpointAction> result = new ArrayList<>();
        result.addAll(dataStorageManager.indexCheckpoint(tableSpace, indexName, indexStatus, pin));
        LOGGER.log(Level.INFO, "checkpoint index {0} finished: logpos {1}, {2} pages", new Object[] { indexName, sequenceNumber, Integer.toString(metadata.nodes.size()) });
        LOGGER.log(Level.FINE, "checkpoint index {0} finished: logpos {1}, pages {2}", new Object[] { indexName, sequenceNumber, activePages.toString() });
        return result;
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
}

Example 83 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class BLinkKeyToPageIndex method scanner.

@Override
public Stream<Entry<Bytes, Long>> scanner(IndexOperation operation, StatementEvaluationContext context, TableContext tableContext, AbstractIndexManager index) throws DataStorageManagerException, StatementExecutionException {
    if (operation instanceof PrimaryIndexSeek) {
        PrimaryIndexSeek seek = (PrimaryIndexSeek) operation;
        byte[] seekValue = computeKeyValue(seek.value, context, tableContext);
        if (seekValue == null) {
            return Stream.empty();
        }
        Bytes key = Bytes.from_array(seekValue);
        Long pageId = getTree().search(key);
        if (pageId == null) {
            return Stream.empty();
        }
        return Stream.of(new AbstractMap.SimpleImmutableEntry<>(key, pageId));
    }
    if (operation instanceof PrimaryIndexPrefixScan) {
        PrimaryIndexPrefixScan scan = (PrimaryIndexPrefixScan) operation;
        // SQLRecordKeyFunction value = sis.value;
        byte[] refvalue = computeKeyValue(scan.value, context, tableContext);
        if (refvalue == null) {
            return Stream.empty();
        }
        Bytes firstKey = Bytes.from_array(refvalue);
        Bytes lastKey = firstKey.next();
        return getTree().scan(firstKey, lastKey);
    }
    // every predicate (WHEREs...) will always be evaluated anyway on every record, in order to guarantee correctness
    if (index != null) {
        return index.recordSetScanner(operation, context, tableContext, this);
    }
    if (operation == null) {
        Stream<Map.Entry<Bytes, Long>> baseStream = getTree().scan(null, null);
        return baseStream;
    } else if (operation instanceof PrimaryIndexRangeScan) {
        Bytes refminvalue;
        PrimaryIndexRangeScan sis = (PrimaryIndexRangeScan) operation;
        SQLRecordKeyFunction minKey = sis.minValue;
        if (minKey != null) {
            refminvalue = Bytes.from_array(minKey.computeNewValue(null, context, tableContext));
        } else {
            refminvalue = null;
        }
        Bytes refmaxvalue;
        SQLRecordKeyFunction maxKey = sis.maxValue;
        if (maxKey != null) {
            refmaxvalue = Bytes.from_array(maxKey.computeNewValue(null, context, tableContext));
        } else {
            refmaxvalue = null;
        }
        return getTree().scan(refminvalue, refmaxvalue, refmaxvalue != null);
    }
    throw new DataStorageManagerException("operation " + operation + " not implemented on " + this.getClass());
}

Example 84 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class BLinkKeyToPageIndex method start.

@Override
public void start(LogSequenceNumber sequenceNumber, boolean created) throws DataStorageManagerException {
    if (!created) {
        LOGGER.log(Level.INFO, " start index {0}", new Object[] { indexName });
    }
    /* Actually the same size */
    final long pageSize = memoryManager.getMaxLogicalPageSize();
    if (LogSequenceNumber.START_OF_TIME.equals(sequenceNumber)) {
        /* Empty index (booting from the start) */
        tree = new BLink<>(pageSize, SizeEvaluatorImpl.INSTANCE, memoryManager.getPKPageReplacementPolicy(), indexDataStorage);
        if (!created) {
            LOGGER.log(Level.INFO, "loaded empty index {0}", new Object[] { indexName });
        }
    } else {
        IndexStatus status = dataStorageManager.getIndexStatus(tableSpace, indexName, sequenceNumber);
        try {
            BLinkMetadata<Bytes> metadata = MetadataSerializer.INSTANCE.read(status.indexData);
            tree = new BLink<>(pageSize, SizeEvaluatorImpl.INSTANCE, memoryManager.getPKPageReplacementPolicy(), indexDataStorage, metadata);
        } catch (IOException e) {
            throw new DataStorageManagerException(e);
        }
        newPageId.set(status.newPageId);
        LOGGER.log(Level.INFO, "loaded index {0}: {1} keys", new Object[] { indexName, tree.size() });
    }
}

Example 85 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class BRINIndexManager method scanner.

@Override
protected Stream<Bytes> scanner(IndexOperation operation, StatementEvaluationContext context, TableContext tableContext) throws StatementExecutionException {
    if (operation instanceof SecondaryIndexSeek) {
        SecondaryIndexSeek sis = (SecondaryIndexSeek) operation;
        SQLRecordKeyFunction value = sis.value;
        byte[] refvalue = value.computeNewValue(null, context, tableContext);
        return data.query(Bytes.from_array(refvalue));
    } else if (operation instanceof SecondaryIndexPrefixScan) {
        SecondaryIndexPrefixScan sis = (SecondaryIndexPrefixScan) operation;
        SQLRecordKeyFunction value = sis.value;
        byte[] refvalue = value.computeNewValue(null, context, tableContext);
        Bytes firstKey = Bytes.from_array(refvalue);
        Bytes lastKey = firstKey.next();
        return data.query(firstKey, lastKey);
    } else if (operation instanceof SecondaryIndexRangeScan) {
        Bytes firstKey = null;
        Bytes lastKey = null;
        SecondaryIndexRangeScan sis = (SecondaryIndexRangeScan) operation;
        SQLRecordKeyFunction minKey = sis.minValue;
        if (minKey != null) {
            byte[] refminvalue = minKey.computeNewValue(null, context, tableContext);
            firstKey = Bytes.from_array(refminvalue);
        }
        SQLRecordKeyFunction maxKey = sis.maxValue;
        if (maxKey != null) {
            byte[] refmaxvalue = maxKey.computeNewValue(null, context, tableContext);
            lastKey = Bytes.from_array(refmaxvalue);
        }
        LOGGER.log(Level.FINE, "range scan on {0}.{1}, from {2} to {1}", new Object[] { index.table, index.name, firstKey, lastKey });
        return data.query(firstKey, lastKey);
    } else {
        throw new UnsupportedOperationException("unsuppported index access type " + operation);
    }
}