Examples with VisibleByteArrayOutputStream herddb.utils.VisibleByteArrayOutputStream used on opensource projects

Example 1 with VisibleByteArrayOutputStream

use of herddb.utils.VisibleByteArrayOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method indexCheckpoint.

@Override
public List<PostCheckpointAction> indexCheckpoint(String tableSpace, String indexName, IndexStatus indexStatus, boolean pin) throws DataStorageManagerException {
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinIndexAndGetPages(tableSpace, indexName, indexStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinIndexAndGetCheckpoints(tableSpace, indexName, indexStatus, pin);
    List<Long> pagesForIndex = new ArrayList<>();
    String prefix = tableSpace + "." + indexName + "_";
    for (String key : indexpages.keySet()) {
        if (key.startsWith(prefix)) {
            long pageId = Long.parseLong(key.substring(prefix.length()));
            if (!pins.containsKey(pageId)) {
                pagesForIndex.add(pageId);
            }
        }
    }
    pagesForIndex.removeAll(indexStatus.activePages);
    List<PostCheckpointAction> result = new ArrayList<>();
    for (long pageId : pagesForIndex) {
        result.add(new PostCheckpointAction(indexName, "drop page " + pageId) {

            @Override
            public void run() {
                // remove only after checkpoint completed
                indexpages.remove(prefix + pageId);
            }
        });
    }
    for (String oldStatus : indexStatuses.keySet()) {
        if (oldStatus.startsWith(prefix)) {
            /* Check for checkpoint skip only if match expected structure */
            final LogSequenceNumber log = evaluateLogSequenceNumber(prefix.substring(0, prefix.length()));
            if (log != null) {
                /* If is pinned skip this status*/
                if (checkpoints.contains(log)) {
                    continue;
                }
            }
            result.add(new PostCheckpointAction(indexName, "drop index checkpoint " + oldStatus) {

                @Override
                public void run() {
                    // remove only after checkpoint completed
                    indexStatuses.remove(oldStatus);
                }
            });
        }
    }
    VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1024);
    try (ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        indexStatus.serialize(dataOutputKeys);
        dataOutputKeys.flush();
        oo.write(oo.xxhash64());
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Uses a copy to limit byte[] size at the min needed */
    indexStatuses.put(checkpointName(tableSpace, indexName, indexStatus.sequenceNumber), oo.toByteArray());
    return result;
}

Example 2 with VisibleByteArrayOutputStream

use of herddb.utils.VisibleByteArrayOutputStream in project herddb by diennea.

the class MemoryDataStorageManager method tableCheckpoint.

@Override
public List<PostCheckpointAction> tableCheckpoint(String tableSpace, String tableName, TableStatus tableStatus, boolean pin) throws DataStorageManagerException {
    /* Checkpoint pinning */
    final Map<Long, Integer> pins = pinTableAndGetPages(tableSpace, tableName, tableStatus, pin);
    final Set<LogSequenceNumber> checkpoints = pinTableAndGetCheckpoints(tableSpace, tableName, tableStatus, pin);
    List<Long> pagesForTable = new ArrayList<>();
    String prefix = tableSpace + "." + tableName + "_";
    for (String key : pages.keySet()) {
        if (key.startsWith(prefix)) {
            long pageId = Long.parseLong(key.substring(prefix.length()));
            if (!pins.containsKey(pageId)) {
                pagesForTable.add(pageId);
            }
        }
    }
    pagesForTable.removeAll(tableStatus.activePages.keySet());
    List<PostCheckpointAction> result = new ArrayList<>();
    for (long pageId : pagesForTable) {
        result.add(new PostCheckpointAction(tableName, "drop page " + pageId) {

            @Override
            public void run() {
                // remove only after checkpoint completed
                pages.remove(prefix + pageId);
                LOGGER.log(Level.SEVERE, "removing " + (prefix + pageId));
            }
        });
    }
    for (String oldStatus : tableStatuses.keySet()) {
        if (oldStatus.startsWith(prefix)) {
            /* Check for checkpoint skip only if match expected structure */
            final LogSequenceNumber log = evaluateLogSequenceNumber(prefix.substring(0, prefix.length()));
            if (log != null) {
                /* If is pinned skip this status*/
                if (checkpoints.contains(log)) {
                    continue;
                }
            }
            result.add(new PostCheckpointAction(tableName, "drop table checkpoint " + oldStatus) {

                @Override
                public void run() {
                    // remove only after checkpoint completed
                    tableStatuses.remove(oldStatus);
                }
            });
        }
    }
    VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1024);
    try (ExtendedDataOutputStream dataOutputKeys = new ExtendedDataOutputStream(oo)) {
        tableStatus.serialize(dataOutputKeys);
        dataOutputKeys.flush();
        oo.write(oo.xxhash64());
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    /* Uses a copy to limit byte[] size at the min needed */
    tableStatuses.put(checkpointName(tableSpace, tableName, tableStatus.sequenceNumber), oo.toByteArray());
    return result;
}

Example 3 with VisibleByteArrayOutputStream

use of herddb.utils.VisibleByteArrayOutputStream in project herddb by diennea.

the class LedgersInfo method serialize.

public synchronized byte[] serialize() {
    try {
        ObjectMapper mapper = new ObjectMapper();
        VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream();
        mapper.writeValue(oo, this);
        return oo.toByteArray();
    } catch (IOException impossible) {
        throw new RuntimeException(impossible);
    }
}

Example 4 with VisibleByteArrayOutputStream

use of herddb.utils.VisibleByteArrayOutputStream in project herddb by diennea.

the class Tuple method serialize.

public static VisibleByteArrayOutputStream serialize(DataAccessor tuple, Column[] columns) throws IOException {
    VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1024);
    try (ExtendedDataOutputStream eoo = new ExtendedDataOutputStream(oo)) {
        int i = 0;
        String[] fieldNames = tuple.getFieldNames();
        for (String fieldName : fieldNames) {
            if (!columns[i].name.toLowerCase().equals(fieldName)) {
                throw new IOException("invalid schema for tuple " + Arrays.toString(fieldNames) + " <> " + Arrays.toString(columns));
            }
            Object value = tuple.get(fieldName);
            if (value == null) {
                eoo.writeVInt(ColumnTypes.NULL);
            } else {
                byte columnType;
                if (value instanceof String) {
                    columnType = ColumnTypes.STRING;
                } else if (value instanceof RawString) {
                    columnType = ColumnTypes.STRING;
                } else if (value instanceof Integer) {
                    columnType = ColumnTypes.INTEGER;
                } else if (value instanceof Long) {
                    columnType = ColumnTypes.LONG;
                } else if (value instanceof java.sql.Timestamp) {
                    columnType = ColumnTypes.TIMESTAMP;
                } else if (value instanceof Double) {
                    columnType = ColumnTypes.DOUBLE;
                } else if (value instanceof Boolean) {
                    columnType = ColumnTypes.BOOLEAN;
                } else if (value instanceof byte[]) {
                    columnType = ColumnTypes.BYTEARRAY;
                } else {
                    throw new IOException("unsupported class " + value.getClass());
                }
                RecordSerializer.serializeTypeAndValue(value, columnType, eoo);
            }
            i++;
        }
    }
    return oo;
}

Example 5 with VisibleByteArrayOutputStream

use of herddb.utils.VisibleByteArrayOutputStream in project herddb by diennea.

the class VisibleByteArrayOutputStreamTest method testMd5.

@Test
public void testMd5() throws Exception {
    byte[] content = "foo".getBytes(StandardCharsets.UTF_8);
    byte[] md5;
    try (VisibleByteArrayOutputStream oo = new VisibleByteArrayOutputStream(1000)) {
        oo.write(content);
        assertArrayEquals(content, oo.toByteArray());
        md5 = oo.xxhash64();
        System.out.println("hash:" + Arrays.toString(md5));
        System.out.println("content:" + Arrays.toString(content));
    }
    byte[] expected = XXHash64Utils.digest(content, 0, content.length);
    System.out.println("expected:" + Arrays.toString(expected));
    assertArrayEquals(expected, md5);
}