Examples with ExtendedDataOutputStream herddb.utils.ExtendedDataOutputStream used on opensource projects

Example 1 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method writeIndexPage.

@Override
public void writeIndexPage(String tableSpace, String indexName, long pageId, DataWriter writer) throws DataStorageManagerException {
    Bytes page_wrapper;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream(1024);
        ExtendedDataOutputStream eout = new ExtendedDataOutputStream(out)) {
        writer.write(eout);
        eout.flush();
        page_wrapper = Bytes.from_array(out.toByteArray());
    } catch (IOException ex) {
        throw new DataStorageManagerException(ex);
    }
    Bytes prev = indexpages.putIfAbsent(tableSpace + "." + indexName + "_" + pageId, page_wrapper);
    if (prev != null) {
        throw new DataStorageManagerException("pages are immutable");
    }
}

Example 2 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method indexCheckpoint.

@Override
public List<PostCheckpointAction> indexCheckpoint(String tableSpace, String indexName, IndexStatus indexStatus, boolean pin) throws DataStorageManagerException {
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinIndexAndGetPages(tableSpace, indexName, indexStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinIndexAndGetCheckpoints(tableSpace, indexName, indexStatus, pin);
    List<Long> pagesForIndex = new ArrayList<>();
    String prefix = tableSpace + "." + indexName + "_";
    for (String key : indexpages.keySet()) {
        if (key.startsWith(prefix)) {
            long pageId = Long.parseLong(key.substring(prefix.length()));
            if (!pins.containsKey(pageId)) {
                pagesForIndex.add(pageId);
            }
        }
    }
    pagesForIndex.removeAll(indexStatus.activePages);
    List<PostCheckpointAction> result = new ArrayList<>();
    for (long pageId : pagesForIndex) {
        result.add(new PostCheckpointAction(indexName, "drop page " + pageId) {

            @Override
            public void run() {
                // remove only after checkpoint completed
                indexpages.remove(prefix + pageId);
            }
        });
    }
    for (String oldStatus : indexStatuses.keySet()) {
        if (oldStatus.startsWith(prefix)) {
            /* Check for checkpoint skip only if match expected structure */
            final LogSequenceNumber log = evaluateLogSequenceNumber(prefix.substring(0, prefix.length()));
            if (log != null) {
                /* If is pinned skip this status*/
                if (checkpoints.contains(log)) {
                    continue;
                }
            }
            result.add(new PostCheckpointAction(indexName, "drop index checkpoint " + oldStatus) {

                @Override
                public void run() {
                    // remove only after checkpoint completed
                    indexStatuses.remove(oldStatus);
                }
            });
        }
    }
    VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1024);
    try (ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        indexStatus.serialize(dataOutputKeys);
        dataOutputKeys.flush();
        oo.write(oo.xxhash64());
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Uses a copy to limit byte[] size at the min needed */
    indexStatuses.put(checkpointName(tableSpace, indexName, indexStatus.sequenceNumber), oo.toByteArray());
    return result;
}

Example 3 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method tableCheckpoint.

@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
    List<Long> pagesForTable = new ArrayList<>();
    String prefix = tableSpace + "." + tableName + "_";
    for (String key : pages.keySet()) {
        if (key.startsWith(prefix)) {
            long pageId = Long.parseLong(key.substring(prefix.length()));
            if (!pins.containsKey(pageId)) {
                pagesForTable.add(pageId);
            }
        }
    }
    pagesForTable.removeAll(tableStatus.activePages.keySet());
    List<PostCheckpointAction> result = new ArrayList<>();
    for (long pageId : pagesForTable) {
        result.add(new PostCheckpointAction(tableName, "drop page " + pageId) {

            @Override
            public void run() {
                // remove only after checkpoint completed
                pages.remove(prefix + pageId);
                LOGGER.log(Level.SEVERE, "removing " + (prefix + pageId));
            }
        });
    }
    for (String oldStatus : tableStatuses.keySet()) {
        if (oldStatus.startsWith(prefix)) {
            /* Check for checkpoint skip only if match expected structure */
            final LogSequenceNumber log = evaluateLogSequenceNumber(prefix.substring(0, prefix.length()));
            if (log != null) {
                /* If is pinned skip this status*/
                if (checkpoints.contains(log)) {
                    continue;
                }
            }
            result.add(new PostCheckpointAction(tableName, "drop table checkpoint " + oldStatus) {

                @Override
                public void run() {
                    // remove only after checkpoint completed
                    tableStatuses.remove(oldStatus);
                }
            });
        }
    }
    VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1024);
    try (ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        tableStatus.serialize(dataOutputKeys);
        dataOutputKeys.flush();
        oo.write(oo.xxhash64());
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Uses a copy to limit byte[] size at the min needed */
    tableStatuses.put(checkpointName(tableSpace, tableName, tableStatus.sequenceNumber), oo.toByteArray());
    return result;
}

Example 4 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class RecordSerializer method serializeValue.

public static Bytes serializeValue(Map<String, Object> record, Table table) {
    ByteArrayOutputStream value = new ByteArrayOutputStream();
    try (ExtendedDataOutputStream doo = new ExtendedDataOutputStream(value)) {
        for (Column c : table.columns) {
            Object v = record.get(c.name);
            if (v != null && !table.isPrimaryKeyColumn(c.name)) {
                doo.writeVInt(c.serialPosition);
                serializeTypeAndValue(v, c.type, doo);
            }
        }
    } catch (IOException err) {
        throw new RuntimeException(err);
    }
    return new Bytes(value.toByteArray());
}

Example 5 with ExtendedDataOutputStream

use of herddb.utils.ExtendedDataOutputStream in project herddb by diennea.

the class RecordSerializer method serializePrimaryKey.

public static Bytes serializePrimaryKey(Map<String, Object> record, ColumnsList table, String[] columns) {
    ByteArrayOutputStream key = new ByteArrayOutputStream();
    String[] primaryKey = table.getPrimaryKey();
    if (primaryKey.length == 1) {
        String pkColumn = primaryKey[0];
        if (columns.length != 1 && !columns[0].equals(pkColumn)) {
            throw new IllegalArgumentException("SQLTranslator error, " + Arrays.toString(columns) + " != " + Arrays.asList(pkColumn));
        }
        Column c = table.getColumn(pkColumn);
        Object v = record.get(c.name);
        if (v == null) {
            throw new IllegalArgumentException("key field " + pkColumn + " cannot be null. Record data: " + record);
        }
        byte[] fieldValue = serialize(v, c.type);
        return new Bytes(fieldValue);
    } else {
        // beware that we can serialize even only a part of the PK, for instance of a prefix index scan
        try (ExtendedDataOutputStream doo_key = new ExtendedDataOutputStream(key)) {
            int i = 0;
            for (String pkColumn : columns) {
                if (!pkColumn.equals(primaryKey[i])) {
                    throw new IllegalArgumentException("SQLTranslator error, " + Arrays.toString(columns) + " != " + Arrays.asList(primaryKey));
                }
                Column c = table.getColumn(pkColumn);
                Object v = record.get(c.name);
                if (v == null) {
                    throw new IllegalArgumentException("key field " + pkColumn + " cannot be null. Record data: " + record);
                }
                byte[] fieldValue = serialize(v, c.type);
                doo_key.writeArray(fieldValue);
                i++;
            }
        } catch (IOException err) {
            throw new RuntimeException(err);
        }
        return new Bytes(key.toByteArray());
    }
}