Search in sources :

Example 1 with DiskPartitionBackedSource

use of edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource in project twister2 by DSC-SPIDAL.

the class KeyedBatchIteratorLinkWrapper method persist.

@Override
public KeyedPersistedTSet<K, V> persist() {
    // handling checkpointing
    if (getTSetEnv().isCheckpointingEnabled()) {
        String persistVariableName = this.getId() + "-persisted";
        BatchChkPntEnvironment chkEnv = (BatchChkPntEnvironment) getTSetEnv();
        Boolean persisted = chkEnv.initVariable(persistVariableName, false);
        if (persisted) {
            // create a source function with the capability to read from disk
            DiskPartitionBackedSource<Tuple<K, V>> sourceFn = new DiskPartitionBackedSource<>(this.getId());
            // pass the source fn to the checkpointed tset (that would create a source tset from the
            // source function, the same way as a persisted tset. This preserves the order of tsets
            // that are being created in the checkpointed env)
            KeyedCheckpointedTSet<K, V> checkTSet = new KeyedCheckpointedTSet<>(getTSetEnv(), sourceFn, this.getTargetParallelism(), getSchema());
            // adding checkpointed tset to the graph, so that the IDs would not change
            addChildToGraph(checkTSet);
            // run only the checkpointed tset so that it would populate the inputs in the executor
            getTSetEnv().runOne(checkTSet);
            return checkTSet;
        } else {
            KeyedPersistedTSet<K, V> storable = this.doPersist();
            chkEnv.updateVariable(persistVariableName, true);
            chkEnv.commit();
            return storable;
        }
    }
    return doPersist();
}
Also used : DiskPartitionBackedSource(edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource) KeyedCheckpointedTSet(edu.iu.dsc.tws.tset.sets.batch.KeyedCheckpointedTSet) BatchChkPntEnvironment(edu.iu.dsc.tws.tset.env.BatchChkPntEnvironment) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 2 with DiskPartitionBackedSource

use of edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource in project twister2 by DSC-SPIDAL.

the class RowBatchTLinkImpl method persist.

/*
   * Similar to cache, but stores data in disk rather than in memory.
   */
public StorableTBase<Row> persist() {
    // handling checkpointing
    if (getTSetEnv().isCheckpointingEnabled()) {
        String persistVariableName = this.getId() + "-persisted";
        BatchChkPntEnvironment chkEnv = (BatchChkPntEnvironment) getTSetEnv();
        Boolean persisted = chkEnv.initVariable(persistVariableName, false);
        if (persisted) {
            // create a source function with the capability to read from disk
            DiskPartitionBackedSource<Row> sourceFn = new DiskPartitionBackedSource<>(this.getId());
            // pass the source fn to the checkpointed tset (that would create a source tset from the
            // source function, the same way as a persisted tset. This preserves the order of tsets
            // that are being created in the checkpointed env)
            CheckpointedTSet<Row> checkTSet = new CheckpointedTSet<>(getTSetEnv(), sourceFn, this.getTargetParallelism(), getSchema());
            // adding checkpointed tset to the graph, so that the IDs would not change
            addChildToGraph(checkTSet);
            // run only the checkpointed tset so that it would populate the inputs in the executor
            getTSetEnv().runOne(checkTSet);
            return checkTSet;
        } else {
            StorableTBase<Row> storable = this.doPersist();
            chkEnv.updateVariable(persistVariableName, true);
            chkEnv.commit();
            return storable;
        }
    }
    return doPersist();
}
Also used : DiskPartitionBackedSource(edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource) BatchChkPntEnvironment(edu.iu.dsc.tws.tset.env.BatchChkPntEnvironment) Row(edu.iu.dsc.tws.common.table.Row) CheckpointedTSet(edu.iu.dsc.tws.tset.sets.batch.CheckpointedTSet)

Example 3 with DiskPartitionBackedSource

use of edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource in project twister2 by DSC-SPIDAL.

the class BatchTLinkImpl method persist.

/*
   * Similar to cache, but stores data in disk rather than in memory.
   */
@Override
public StorableTBase<T0> persist() {
    // handling checkpointing
    if (getTSetEnv().isCheckpointingEnabled()) {
        String persistVariableName = this.getId() + "-persisted";
        BatchChkPntEnvironment chkEnv = (BatchChkPntEnvironment) getTSetEnv();
        Boolean persisted = chkEnv.initVariable(persistVariableName, false);
        if (persisted) {
            // create a source function with the capability to read from disk
            DiskPartitionBackedSource<T0> sourceFn = new DiskPartitionBackedSource<>(this.getId());
            // pass the source fn to the checkpointed tset (that would create a source tset from the
            // source function, the same way as a persisted tset. This preserves the order of tsets
            // that are being created in the checkpointed env)
            CheckpointedTSet<T0> checkTSet = new CheckpointedTSet<>(getTSetEnv(), sourceFn, this.getTargetParallelism(), getSchema());
            // adding checkpointed tset to the graph, so that the IDs would not change
            addChildToGraph(checkTSet);
            // run only the checkpointed tset so that it would populate the inputs in the executor
            getTSetEnv().runOne(checkTSet);
            return checkTSet;
        } else {
            StorableTBase<T0> storable = this.doPersist();
            chkEnv.updateVariable(persistVariableName, true);
            chkEnv.commit();
            return storable;
        }
    }
    return doPersist();
}
Also used : DiskPartitionBackedSource(edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource) BatchChkPntEnvironment(edu.iu.dsc.tws.tset.env.BatchChkPntEnvironment) CheckpointedTSet(edu.iu.dsc.tws.tset.sets.batch.CheckpointedTSet)

Example 4 with DiskPartitionBackedSource

use of edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource in project twister2 by DSC-SPIDAL.

the class KeyedPipeTLink method persist.

@Override
public KeyedPersistedTSet<K, V> persist() {
    // handling checkpointing
    if (getTSetEnv().isCheckpointingEnabled()) {
        String persistVariableName = this.getId() + "-persisted";
        BatchChkPntEnvironment chkEnv = (BatchChkPntEnvironment) getTSetEnv();
        Boolean persisted = chkEnv.initVariable(persistVariableName, false);
        if (persisted) {
            // create a source function with the capability to read from disk
            DiskPartitionBackedSource<Tuple<K, V>> sourceFn = new DiskPartitionBackedSource<>(this.getId());
            // pass the source fn to the checkpointed tset (that would create a source tset from the
            // source function, the same way as a persisted tset. This preserves the order of tsets
            // that are being created in the checkpointed env)
            KeyedCheckpointedTSet<K, V> checkTSet = new KeyedCheckpointedTSet<>(getTSetEnv(), sourceFn, this.getTargetParallelism(), getSchema());
            // adding checkpointed tset to the graph, so that the IDs would not change
            addChildToGraph(checkTSet);
            // run only the checkpointed tset so that it would populate the inputs in the executor
            getTSetEnv().runOne(checkTSet);
            return checkTSet;
        } else {
            KeyedPersistedTSet<K, V> storable = this.doPersist();
            chkEnv.updateVariable(persistVariableName, true);
            chkEnv.commit();
            return storable;
        }
    }
    return doPersist();
}
Also used : DiskPartitionBackedSource(edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource) KeyedCheckpointedTSet(edu.iu.dsc.tws.tset.sets.batch.KeyedCheckpointedTSet) BatchChkPntEnvironment(edu.iu.dsc.tws.tset.env.BatchChkPntEnvironment) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Aggregations

BatchChkPntEnvironment (edu.iu.dsc.tws.tset.env.BatchChkPntEnvironment)4 DiskPartitionBackedSource (edu.iu.dsc.tws.tset.sources.DiskPartitionBackedSource)4 Tuple (edu.iu.dsc.tws.api.comms.structs.Tuple)2 CheckpointedTSet (edu.iu.dsc.tws.tset.sets.batch.CheckpointedTSet)2 KeyedCheckpointedTSet (edu.iu.dsc.tws.tset.sets.batch.KeyedCheckpointedTSet)2 Row (edu.iu.dsc.tws.common.table.Row)1