Examples with SnapshotTableTask org.voltdb.SnapshotTableTask used on opensource projects

Example 1 with SnapshotTableTask

use of org.voltdb.SnapshotTableTask in project voltdb by VoltDB.

the class CSVSnapshotWritePlan method createDeferredSetup.

private Callable<Boolean> createDeferredSetup(final String file_path, final String pathType, final String file_nonce, final Table[] tables, final long txnId, final Map<Integer, Long> partitionTransactionIds, final SystemProcedureExecutionContext context, final ExtensibleSnapshotDigestData extraSnapshotData, final long timestamp, final AtomicInteger numTables, final SnapshotRegistry.Snapshot snapshotRecord, final ArrayList<SnapshotTableTask> partitionedSnapshotTasks, final ArrayList<SnapshotTableTask> replicatedSnapshotTasks) {
    return new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            NativeSnapshotWritePlan.createFileBasedCompletionTasks(file_path, pathType, file_nonce, txnId, partitionTransactionIds, context, extraSnapshotData, null, timestamp, context.getNumberOfPartitions(), tables);
            for (SnapshotTableTask task : replicatedSnapshotTasks) {
                final SnapshotDataTarget target = createDataTargetForTable(file_path, file_nonce, context.getHostId(), numTables, snapshotRecord, task.m_table);
                task.setTarget(target);
            }
            for (SnapshotTableTask task : partitionedSnapshotTasks) {
                final SnapshotDataTarget target = createDataTargetForTable(file_path, file_nonce, context.getHostId(), numTables, snapshotRecord, task.m_table);
                task.setTarget(target);
            }
            return true;
        }
    };
}

Example 2 with SnapshotTableTask

use of org.voltdb.SnapshotTableTask in project voltdb by VoltDB.

the class SnapshotWritePlan method createAllDevNullTargets.

/**
     * In case the deferred setup phase fails, some data targets may have not been created yet.
     * This method will close all existing data targets and replace all with DevNullDataTargets
     * so that snapshot can be drained.
     */
public void createAllDevNullTargets() {
    Map<Integer, SnapshotDataTarget> targets = Maps.newHashMap();
    final AtomicInteger numTargets = new AtomicInteger();
    for (Deque<SnapshotTableTask> tasksForSite : m_taskListsForHSIds.values()) {
        for (SnapshotTableTask task : tasksForSite) {
            // Close any created targets and replace them with DevNull, go web-scale
            if (task.getTarget(true) != null) {
                try {
                    task.getTarget().close();
                } catch (Exception e) {
                    SNAP_LOG.error("Failed closing data target after error", e);
                }
            }
            SnapshotDataTarget target = targets.get(task.m_table.getRelativeIndex());
            if (target == null) {
                target = new DevNullSnapshotTarget();
                final Runnable onClose = new TargetStatsClosure(target, task.m_table.getTypeName(), numTargets, m_snapshotRecord);
                target.setOnCloseHandler(onClose);
                targets.put(task.m_table.getRelativeIndex(), target);
                m_targets.add(target);
                numTargets.incrementAndGet();
            }
            task.setTarget(target);
        }
    }
}

Example 3 with SnapshotTableTask

use of org.voltdb.SnapshotTableTask in project voltdb by VoltDB.

the class NativeSnapshotWritePlan method createSetupInternal.

Callable<Boolean> createSetupInternal(String file_path, String pathType, String file_nonce, long txnId, Map<Integer, Long> partitionTransactionIds, JSONObject jsData, SystemProcedureExecutionContext context, final VoltTable result, ExtensibleSnapshotDigestData extraSnapshotData, SiteTracker tracker, HashinatorSnapshotData hashinatorData, long timestamp, int newPartitionCount) {
    assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());
    if (TheHashinator.getConfiguredHashinatorType() == HashinatorType.ELASTIC && hashinatorData == null) {
        throw new RuntimeException("No hashinator data provided for elastic hashinator type.");
    }
    final SnapshotRequestConfig config = new SnapshotRequestConfig(jsData, context.getDatabase());
    final Table[] tableArray;
    if (config.tables.length == 0 && (jsData == null || !jsData.has("tables"))) {
        tableArray = SnapshotUtil.getTablesToSave(context.getDatabase()).toArray(new Table[0]);
    } else {
        tableArray = config.tables;
    }
    m_snapshotRecord = SnapshotRegistry.startSnapshot(txnId, context.getHostId(), file_path, file_nonce, SnapshotFormat.NATIVE, tableArray);
    final ArrayList<SnapshotTableTask> partitionedSnapshotTasks = new ArrayList<SnapshotTableTask>();
    final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
    for (final Table table : tableArray) {
        final SnapshotTableTask task = new SnapshotTableTask(table, new SnapshotDataFilter[0], null, false);
        SNAP_LOG.debug("ADDING TASK: " + task);
        if (table.getIsreplicated()) {
            replicatedSnapshotTasks.add(task);
        } else {
            partitionedSnapshotTasks.add(task);
        }
        result.addRow(context.getHostId(), CoreUtils.getHostnameOrAddress(), table.getTypeName(), "SUCCESS", "");
    }
    if (tableArray.length > 0 && replicatedSnapshotTasks.isEmpty() && partitionedSnapshotTasks.isEmpty()) {
        SnapshotRegistry.discardSnapshot(m_snapshotRecord);
    }
    // Native snapshots place the partitioned tasks on every site and round-robin the
    // replicated tasks across all the sites on every host
    placePartitionedTasks(partitionedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
    placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
    boolean isTruncationSnapshot = true;
    if (jsData != null) {
        isTruncationSnapshot = jsData.has("truncReqId");
    }
    // All IO work will be deferred and be run on the dedicated snapshot IO thread
    return createDeferredSetup(file_path, pathType, file_nonce, txnId, partitionTransactionIds, context, extraSnapshotData, tracker, hashinatorData, timestamp, newPartitionCount, tableArray, m_snapshotRecord, partitionedSnapshotTasks, replicatedSnapshotTasks, isTruncationSnapshot);
}

Example 4 with SnapshotTableTask

use of org.voltdb.SnapshotTableTask in project voltdb by VoltDB.

the class IndexSnapshotWritePlan method createTasksForTable.

/**
     * For each site, generate a task for each target it has for this table.
     */
private void createTasksForTable(Table table, Collection<IndexSnapshotRequestConfig.PartitionRanges> partitionRanges, Map<Integer, Long> pidToLocalHSIDs, AtomicInteger numTables, SnapshotRegistry.Snapshot snapshotRecord) {
    // no work on this node
    if (pidToLocalHSIDs.isEmpty()) {
        return;
    }
    // create a null data target
    final DevNullSnapshotTarget dataTarget = new DevNullSnapshotTarget();
    final Runnable onClose = new TargetStatsClosure(dataTarget, table.getTypeName(), numTables, snapshotRecord);
    dataTarget.setOnCloseHandler(onClose);
    m_targets.add(dataTarget);
    // go over all local sites, create a task for each source site
    for (IndexSnapshotRequestConfig.PartitionRanges partitionRange : partitionRanges) {
        Long localHSId = pidToLocalHSIDs.get(partitionRange.partitionId);
        // The partition may not exist on this node. If so, keep calm and carry on
        if (localHSId != null) {
            // based on the source partition, the predicate is different
            final SnapshotTableTask task = new SnapshotTableTask(table, new SnapshotDataFilter[0], createIndexExpressionForTable(table, partitionRange.ranges), false);
            task.setTarget(dataTarget);
            placeTask(task, Arrays.asList(localHSId));
        }
    }
}

Example 5 with SnapshotTableTask

use of org.voltdb.SnapshotTableTask in project voltdb by VoltDB.

the class NativeSnapshotWritePlan method createDeferredSetup.

private Callable<Boolean> createDeferredSetup(final String file_path, final String pathType, final String file_nonce, final long txnId, final Map<Integer, Long> partitionTransactionIds, final SystemProcedureExecutionContext context, final ExtensibleSnapshotDigestData extraSnapshotData, final SiteTracker tracker, final HashinatorSnapshotData hashinatorData, final long timestamp, final int newPartitionCount, final Table[] tables, final SnapshotRegistry.Snapshot snapshotRecord, final ArrayList<SnapshotTableTask> partitionedSnapshotTasks, final ArrayList<SnapshotTableTask> replicatedSnapshotTasks, final boolean isTruncationSnapshot) {
    return new Callable<Boolean>() {

        private final HashMap<Integer, SnapshotDataTarget> m_createdTargets = Maps.newHashMap();

        @Override
        public Boolean call() throws Exception {
            final AtomicInteger numTables = new AtomicInteger(tables.length);
            NativeSnapshotWritePlan.createFileBasedCompletionTasks(file_path, pathType, file_nonce, txnId, partitionTransactionIds, context, extraSnapshotData, hashinatorData, timestamp, newPartitionCount, tables);
            for (SnapshotTableTask task : replicatedSnapshotTasks) {
                SnapshotDataTarget target = getSnapshotDataTarget(numTables, task);
                task.setTarget(target);
            }
            for (SnapshotTableTask task : partitionedSnapshotTasks) {
                SnapshotDataTarget target = getSnapshotDataTarget(numTables, task);
                task.setTarget(target);
            }
            if (isTruncationSnapshot) {
                // Only sync the DR Log on Native Snapshots
                SnapshotSiteProcessor.m_tasksOnSnapshotCompletion.offer(new Runnable() {

                    @Override
                    public void run() {
                        context.forceAllDRNodeBuffersToDisk(false);
                    }
                });
            }
            // Sync export buffer for all types of snapshot
            SnapshotSiteProcessor.m_tasksOnSnapshotCompletion.offer(new Runnable() {

                @Override
                public void run() {
                    ExportManager.sync(false);
                }
            });
            return true;
        }

        private SnapshotDataTarget getSnapshotDataTarget(AtomicInteger numTables, SnapshotTableTask task) throws IOException {
            SnapshotDataTarget target = m_createdTargets.get(task.m_table.getRelativeIndex());
            if (target == null) {
                target = createDataTargetForTable(file_path, file_nonce, task.m_table, txnId, context.getHostId(), context.getCluster().getTypeName(), context.getDatabase().getTypeName(), context.getNumberOfPartitions(), DrRoleType.XDCR.value().equals(context.getCluster().getDrrole()), tracker, timestamp, numTables, snapshotRecord);
                m_createdTargets.put(task.m_table.getRelativeIndex(), target);
            }
            return target;
        }
    };
}