Examples with LocalFixedInputPartitioner edu.iu.dsc.tws.data.api.formatters.LocalFixedInputPartitioner used on opensource projects

Example 1 with LocalFixedInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalFixedInputPartitioner in project twister2 by DSC-SPIDAL.

the class GraphDataSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    this.source = runtime.createInput(cfg, context, new LocalFixedInputPartitioner(new Path(getDataDirectory()), context.getParallelism(), cfg, dsize));
}

Example 2 with LocalFixedInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalFixedInputPartitioner in project twister2 by DSC-SPIDAL.

the class DataLoadingTask method prepare.

@Override
public void prepare(TSetContext context) {
    super.prepare(context);
    this.config = context.getConfig();
    this.parallelism = context.getParallelism();
    LOG.info(String.format("%d, %d, %d", context.getIndex(), this.svmJobParameters.getParallelism(), context.getParallelism()));
    // dimension is +1 features as the input data comes along with the label
    this.dimension = this.binaryBatchModel.getFeatures() + 1;
    if ("train".equalsIgnoreCase(this.dataType)) {
        this.dataSize = this.binaryBatchModel.getSamples();
        this.localPoints = new double[this.dataSize / parallelism][this.dimension];
        LOG.info(String.format("Data Size : %d, Array Shape [%d,%d]", this.dataSize, this.localPoints.length, this.dimension));
        this.source = new DataSource(config, new LocalFixedInputPartitioner(new Path(this.svmJobParameters.getTrainingDataDir()), this.parallelism, config, dataSize), this.parallelism);
    }
    if ("test".equalsIgnoreCase(this.dataType)) {
        this.dataSize = this.svmJobParameters.getTestingSamples();
        this.localPoints = new double[this.dataSize / parallelism][this.dimension];
        this.source = new DataSource(config, new LocalFixedInputPartitioner(new Path(this.svmJobParameters.getTestingDataDir()), this.parallelism, config, dataSize), this.parallelism);
    }
}

Example 3 with LocalFixedInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalFixedInputPartitioner in project twister2 by DSC-SPIDAL.

the class SVMDataObjectSource method prepare.

@Override
public void prepare(Config cfg, TaskContext ctx) {
    super.prepare(cfg, ctx);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    this.source = runtime.createInput(cfg, context, new LocalFixedInputPartitioner(new Path(getDataDirectory()), context.getParallelism(), config, this.datasize));
}

Example 4 with LocalFixedInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalFixedInputPartitioner in project twister2 by DSC-SPIDAL.

the class WeightVectorLoad method prepare.

@Override
public void prepare(TSetContext context) {
    super.prepare(context);
    this.config = context.getConfig();
    this.parallelism = context.getParallelism();
    LOG.info(String.format("%d, %d, %d", this.getTSetContext().getIndex(), this.svmJobParameters.getParallelism(), context.getParallelism()));
    this.dimension = this.binaryBatchModel.getFeatures();
    this.localPoints = new double[this.dimension];
    this.source = new DataSource<double[], InputSplit<double[]>>(config, new LocalFixedInputPartitioner(new Path(this.svmJobParameters.getWeightVectorDataDir()), this.parallelism, config, 1), this.parallelism);
}