Examples with Bytes herddb.utils.Bytes used on opensource projects

Example 31 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class MemoryHashIndexManager method doStart.

@Override
protected boolean doStart(LogSequenceNumber sequenceNumber) throws DataStorageManagerException {
    LOGGER.log(Level.INFO, "loading in memory all the keys for mem index {0}", new Object[] { index.name });
    bootSequenceNumber = sequenceNumber;
    dataStorageManager.initIndex(tableSpaceUUID, index.uuid);
    if (LogSequenceNumber.START_OF_TIME.equals(sequenceNumber)) {
        /* Empty index (booting from the start) */
        LOGGER.log(Level.INFO, "loaded empty index {0}", new Object[] { index.name });
        return true;
    } else {
        IndexStatus status;
        try {
            status = dataStorageManager.getIndexStatus(tableSpaceUUID, index.uuid, sequenceNumber);
        } catch (DataStorageManagerException e) {
            LOGGER.log(Level.SEVERE, "cannot load index {0} due to {1}, it will be rebuilt", new Object[] { index.name, e });
            return false;
        }
        for (long pageId : status.activePages) {
            LOGGER.log(Level.INFO, "recovery index {0}, load {1}", new Object[] { index.name, pageId });
            Map<Bytes, List<Bytes>> read = dataStorageManager.readIndexPage(tableSpaceUUID, index.uuid, pageId, in -> {
                Map<Bytes, List<Bytes>> deserialized = new HashMap<>();
                // version
                long version = in.readVLong();
                // flags for future implementations
                long flags = in.readVLong();
                if (version != 1 || flags != 0) {
                    throw new DataStorageManagerException("corrupted index page");
                }
                int size = in.readVInt();
                for (int i = 0; i < size; i++) {
                    Bytes indexKey = in.readBytesNoCopy();
                    int entrySize = in.readVInt();
                    List<Bytes> value = new ArrayList<>(entrySize);
                    for (int kk = 0; kk < entrySize; kk++) {
                        Bytes tableKey = in.readBytesNoCopy();
                        value.add(tableKey);
                    }
                    deserialized.put(indexKey, value);
                }
                return deserialized;
            });
            data.putAll(read);
        }
        newPageId.set(status.newPageId);
        LOGGER.log(Level.INFO, "loaded {0} keys for index {1}", new Object[] { data.size(), index.name });
        return true;
    }
}

Example 32 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class BRINIndexManager method rebuild.

@Override
public void rebuild() throws DataStorageManagerException {
    long _start = System.currentTimeMillis();
    LOGGER.log(Level.FINE, "building index {0}", index.name);
    dataStorageManager.initIndex(tableSpaceUUID, index.uuid);
    data.reset();
    Table table = tableManager.getTable();
    AtomicLong count = new AtomicLong();
    tableManager.scanForIndexRebuild(r -> {
        DataAccessor values = r.getDataAccessor(table);
        Bytes key = RecordSerializer.serializeIndexKey(values, table, table.primaryKey);
        Bytes indexKey = RecordSerializer.serializeIndexKey(values, index, index.columnNames);
        // LOGGER.log(Level.SEVERE, "adding " + key + " -> " + values);
        recordInserted(key, indexKey);
        count.incrementAndGet();
    });
    long _stop = System.currentTimeMillis();
    if (count.intValue() > 0) {
        LOGGER.log(Level.INFO, "building index {0} took {1}, scanned {2} records", new Object[] { index.name, (_stop - _start) + " ms", count });
    }
}

Example 33 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class MemoryDataStorageManager method writeIndexPage.

@Override
public void writeIndexPage(String tableSpace, String indexName, long pageId, DataWriter writer) throws DataStorageManagerException {
    Bytes page_wrapper;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream(1024);
        ExtendedDataOutputStream eout = new ExtendedDataOutputStream(out)) {
        writer.write(eout);
        eout.flush();
        page_wrapper = Bytes.from_array(out.toByteArray());
    } catch (IOException ex) {
        throw new DataStorageManagerException(ex);
    }
    indexpages.put(tableSpace + "." + indexName + "_" + pageId, page_wrapper);
}

Example 34 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class Transaction method serialize.

public synchronized void serialize(ExtendedDataOutputStream out) throws IOException {
    // version
    out.writeVLong(1);
    // flags for future implementations
    out.writeVLong(0);
    out.writeZLong(transactionId);
    if (lastSequenceNumber != null) {
        out.writeZLong(lastSequenceNumber.ledgerId);
        out.writeZLong(lastSequenceNumber.offset);
    } else {
        out.writeZLong(0);
        out.writeZLong(0);
    }
    out.writeVInt(changedRecords.size());
    for (Map.Entry<String, Map<Bytes, Record>> table : changedRecords.entrySet()) {
        out.writeUTF(table.getKey());
        out.writeVInt(table.getValue().size());
        for (Record r : table.getValue().values()) {
            out.writeArray(r.key);
            out.writeArray(r.value);
        }
    }
    out.writeVInt(newRecords.size());
    for (Map.Entry<String, Map<Bytes, Record>> table : newRecords.entrySet()) {
        out.writeUTF(table.getKey());
        out.writeVInt(table.getValue().size());
        for (Record r : table.getValue().values()) {
            out.writeArray(r.key);
            out.writeArray(r.value);
        }
    }
    out.writeVInt(deletedRecords.size());
    for (Map.Entry<String, Set<Bytes>> table : deletedRecords.entrySet()) {
        out.writeUTF(table.getKey());
        out.writeVInt(table.getValue().size());
        for (Bytes key : table.getValue()) {
            out.writeArray(key);
        }
    }
    if (newTables == null) {
        out.writeVInt(0);
    } else {
        out.writeVInt(newTables.size());
        for (Table table : newTables.values()) {
            out.writeArray(table.serialize());
        }
    }
    if (droppedTables == null) {
        out.writeVInt(0);
    } else {
        out.writeVInt(droppedTables.size());
        for (String table : droppedTables) {
            out.writeUTF(table);
        }
    }
    if (newIndexes == null) {
        out.writeVInt(0);
    } else {
        out.writeVInt(newIndexes.size());
        for (Index index : newIndexes.values()) {
            out.writeArray(index.serialize());
        }
    }
    if (droppedIndexes == null) {
        out.writeVInt(0);
    } else {
        out.writeVInt(droppedIndexes.size());
        for (String index : droppedIndexes) {
            out.writeUTF(index);
        }
    }
}

Example 35 with Bytes

use of herddb.utils.Bytes in project herddb by diennea.

the class BookKeeperDataStorageManager method rawReadDataPage.

private static List<Record> rawReadDataPage(byte[] dataPage) throws IOException, DataStorageManagerException {
    try (ByteArrayCursor dataIn = ByteArrayCursor.wrap(dataPage)) {
        // version
        long version = dataIn.readVLong();
        // flags for future implementations
        long flags = dataIn.readVLong();
        if (version != 1 || flags != 0) {
            throw new DataStorageManagerException("corrupted data");
        }
        int numRecords = dataIn.readInt();
        List<Record> result = new ArrayList<>(numRecords);
        for (int i = 0; i < numRecords; i++) {
            Bytes key = dataIn.readBytesNoCopy();
            Bytes value = dataIn.readBytesNoCopy();
            result.add(new Record(key, value));
        }
        int pos = dataIn.getPosition();
        long hashFromFile = dataIn.readLong();
        // after the hash we will have zeroes or garbage
        // the hash is not at the end of file, but after data
        long hashFromDigest = XXHash64Utils.hash(dataPage, 0, pos);
        if (hashFromDigest != hashFromFile) {
            throw new DataStorageManagerException("Corrupted datafile. Bad hash " + hashFromFile + " <> " + hashFromDigest);
        }
        return result;
    }
}