Examples with DumpedTableMetadata herddb.backup.DumpedTableMetadata used on opensource projects

Example 1 with DumpedTableMetadata

use of herddb.backup.DumpedTableMetadata in project herddb by diennea.

the class DownloadTableSpaceTest method downloadTablespaceTest.

@Test
public void downloadTablespaceTest() throws Exception {
    ServerConfiguration serverconfig_1 = new ServerConfiguration(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_PORT, 7867);
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        for (int i = 0; i < 1000; i++) {
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", i)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        }
        List<Map<String, Object>> logical_data = new ArrayList<>();
        AtomicBoolean start = new AtomicBoolean();
        try (HDBClient client = new HDBClient(new ClientConfiguration(folder.newFolder().toPath()));
            HDBConnection con = client.openConnection()) {
            client.setClientSideMetadataProvider(new ZookeeperClientSideMetadataProvider(testEnv.getAddress(), testEnv.getTimeout(), testEnv.getPath()));
            CountDownLatch count = new CountDownLatch(1);
            con.dumpTableSpace(TableSpace.DEFAULT, new TableSpaceDumpReceiver() {

                Table table;

                @Override
                public void start(LogSequenceNumber logSequenceNumber) throws DataStorageManagerException {
                    System.out.println("start at " + logSequenceNumber);
                    start.set(true);
                }

                @Override
                public void finish(LogSequenceNumber logSequenceNumber) {
                    System.out.println("finish!");
                    count.countDown();
                }

                @Override
                public void endTable() {
                    System.out.println("endTable");
                    table = null;
                }

                @Override
                public void receiveTableDataChunk(List<Record> records) {
                    // System.out.println("receiveTableDataChunk " + records);
                    for (Record r : records) {
                        Map<String, Object> bean = r.toBean(table);
                        // System.out.println("received:" + bean);
                        logical_data.add(bean);
                    }
                }

                @Override
                public void beginTable(DumpedTableMetadata table, Map<String, Object> stats) {
                    System.out.println("beginTable " + table);
                    this.table = table.table;
                }
            }, 89, false);
            assertTrue(count.await(20, TimeUnit.SECONDS));
            assertEquals(1000, logical_data.size());
            assertTrue(start.get());
        }
    }
}

Example 2 with DumpedTableMetadata

use of herddb.backup.DumpedTableMetadata in project herddb by diennea.

the class RoutedClientSideConnection method messageReceived.

@Override
@SuppressFBWarnings(value = "SF_SWITCH_NO_DEFAULT")
public void messageReceived(Message message, Channel _channel) {
    switch(message.type) {
        case Message.TYPE_TABLESPACE_DUMP_DATA:
            {
                String dumpId = (String) message.parameters.get("dumpId");
                TableSpaceDumpReceiver receiver = dumpReceivers.get(dumpId);
                LOGGER.log(Level.FINE, "receiver for {0}: {1}", new Object[] { dumpId, receiver });
                if (receiver == null) {
                    if (_channel != null) {
                        _channel.sendReplyMessage(message, Message.ERROR(clientId, new Exception("no such dump receiver " + dumpId)));
                    }
                    return;
                }
                try {
                    Map<String, Object> values = (Map<String, Object>) message.parameters.get("values");
                    String command = (String) values.get("command") + "";
                    boolean sendAck = true;
                    switch(command) {
                        case "start":
                            {
                                long ledgerId = (long) values.get("ledgerid");
                                long offset = (long) values.get("offset");
                                receiver.start(new LogSequenceNumber(ledgerId, offset));
                                break;
                            }
                        case "beginTable":
                            {
                                byte[] tableDefinition = (byte[]) values.get("table");
                                Table table = Table.deserialize(tableDefinition);
                                Long estimatedSize = (Long) values.get("estimatedSize");
                                long dumpLedgerId = (Long) values.get("dumpLedgerid");
                                long dumpOffset = (Long) values.get("dumpOffset");
                                List<byte[]> indexesDef = (List<byte[]>) values.get("indexes");
                                List<Index> indexes = indexesDef.stream().map(Index::deserialize).collect(Collectors.toList());
                                Map<String, Object> stats = new HashMap<>();
                                stats.put("estimatedSize", estimatedSize);
                                stats.put("dumpLedgerId", dumpLedgerId);
                                stats.put("dumpOffset", dumpOffset);
                                receiver.beginTable(new DumpedTableMetadata(table, new LogSequenceNumber(dumpLedgerId, dumpOffset), indexes), stats);
                                break;
                            }
                        case "endTable":
                            {
                                receiver.endTable();
                                break;
                            }
                        case "finish":
                            {
                                long ledgerId = (long) values.get("ledgerid");
                                long offset = (long) values.get("offset");
                                receiver.finish(new LogSequenceNumber(ledgerId, offset));
                                sendAck = false;
                                break;
                            }
                        case "data":
                            {
                                List<KeyValue> data = (List<KeyValue>) values.get("records");
                                List<Record> records = new ArrayList<>(data.size());
                                for (KeyValue kv : data) {
                                    records.add(new Record(new Bytes(kv.key), new Bytes(kv.value)));
                                }
                                receiver.receiveTableDataChunk(records);
                                break;
                            }
                        case "txlog":
                            {
                                List<KeyValue> data = (List<KeyValue>) values.get("records");
                                List<DumpedLogEntry> records = new ArrayList<>(data.size());
                                for (KeyValue kv : data) {
                                    records.add(new DumpedLogEntry(LogSequenceNumber.deserialize(kv.key), kv.value));
                                }
                                receiver.receiveTransactionLogChunk(records);
                                break;
                            }
                        case "transactions":
                            {
                                String tableSpace = (String) values.get("tableSpace");
                                List<byte[]> data = (List<byte[]>) values.get("transactions");
                                List<Transaction> transactions = data.stream().map(array -> {
                                    return Transaction.deserialize(tableSpace, array);
                                }).collect(Collectors.toList());
                                receiver.receiveTransactionsAtDump(transactions);
                                break;
                            }
                        default:
                            throw new DataStorageManagerException("invalid dump command:" + command);
                    }
                    if (_channel != null && sendAck) {
                        _channel.sendReplyMessage(message, Message.ACK(clientId));
                    }
                } catch (DataStorageManagerException error) {
                    LOGGER.log(Level.SEVERE, "error while handling dump data", error);
                    if (_channel != null) {
                        _channel.sendReplyMessage(message, Message.ERROR(clientId, error));
                    }
                }
            }
            break;
    }
}