Examples with BenchmarkResultsRecorder edu.iu.dsc.tws.examples.utils.bench.BenchmarkResultsRecorder used on opensource projects

Example 1 with BenchmarkResultsRecorder

use of edu.iu.dsc.tws.examples.utils.bench.BenchmarkResultsRecorder in project twister2 by DSC-SPIDAL.

the class TeraSort method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerID = workerEnv.getWorkerId();
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    Config config = workerEnv.getConfig();
    resultsRecorder = new BenchmarkResultsRecorder(config, workerID == 0);
    Timing.setDefaultTimingUnit(TimingUnit.MILLI_SECONDS);
    final String filePath = config.getStringValue(ARG_INPUT_FILE, null);
    final int keySize = config.getIntegerValue(ARG_KEY_SIZE, 10);
    final int valueSize = config.getIntegerValue(ARG_VALUE_SIZE, 90);
    // Sampling Graph : if file based only
    TaskPartitioner taskPartitioner;
    if (filePath != null) {
        ComputeGraphBuilder samplingGraph = ComputeGraphBuilder.newBuilder(config);
        samplingGraph.setMode(OperationMode.BATCH);
        Sampler samplerTask = new Sampler();
        samplingGraph.addSource(TASK_SAMPLER, samplerTask, config.getIntegerValue(ARG_TASKS_SOURCES, 4));
        SamplerReduce samplerReduce = new SamplerReduce();
        samplingGraph.addCompute(TASK_SAMPLER_REDUCE, samplerReduce, config.getIntegerValue(ARG_RESOURCE_INSTANCES, 4)).allreduce(TASK_SAMPLER).viaEdge(EDGE).withReductionFunction(byte[].class, (minMax1, minMax2) -> {
            byte[] min1 = Arrays.copyOfRange(minMax1, 0, keySize);
            byte[] max1 = Arrays.copyOfRange(minMax1, keySize, minMax1.length);
            byte[] min2 = Arrays.copyOfRange(minMax2, 0, keySize);
            byte[] max2 = Arrays.copyOfRange(minMax2, keySize, minMax2.length);
            byte[] newMinMax = new byte[keySize * 2];
            byte[] min = min1;
            byte[] max = max1;
            if (ByteArrayComparator.getInstance().compare(min1, min2) > 0) {
                min = min2;
            }
            if (ByteArrayComparator.getInstance().compare(max1, max2) < 0) {
                max = max2;
            }
            System.arraycopy(min, 0, newMinMax, 0, keySize);
            System.arraycopy(max, 0, newMinMax, keySize, keySize);
            return newMinMax;
        });
        ComputeGraph sampleGraphBuild = samplingGraph.build();
        ExecutionPlan sampleTaskPlan = cEnv.getTaskExecutor().plan(sampleGraphBuild);
        cEnv.getTaskExecutor().execute(sampleGraphBuild, sampleTaskPlan);
        DataObject<byte[]> output = cEnv.getTaskExecutor().getOutput("sample-reduce");
        LOG.info("Sample output received");
        taskPartitioner = new TaskPartitionerForSampledData(output.getPartitions()[0].getConsumer().next(), keySize);
    } else {
        taskPartitioner = new TaskPartitionerForRandom();
    }
    // Sort Graph
    ComputeGraphBuilder teraSortTaskGraph = ComputeGraphBuilder.newBuilder(config);
    teraSortTaskGraph.setMode(OperationMode.BATCH);
    BaseSource dataSource;
    if (filePath == null) {
        dataSource = new RandomDataSource();
    } else {
        dataSource = new FileDataSource();
    }
    teraSortTaskGraph.addSource(TASK_SOURCE, dataSource, config.getIntegerValue(ARG_TASKS_SOURCES, 4));
    Receiver receiver = new Receiver();
    KeyedGatherConfig keyedGatherConfig = teraSortTaskGraph.addCompute(TASK_RECV, receiver, config.getIntegerValue(ARG_TASKS_SINKS, 4)).keyedGather(TASK_SOURCE).viaEdge(EDGE).withDataType(MessageTypes.BYTE_ARRAY).withKeyType(MessageTypes.BYTE_ARRAY).withTaskPartitioner(taskPartitioner).useDisk(true).sortBatchByKey(ByteArrayComparator.getInstance()).groupBatchByKey(false);
    if (config.getBooleanValue(ARG_FIXED_SCHEMA, false)) {
        LOG.info("Using fixed schema feature with message size : " + (keySize + valueSize) + " and key size : " + keySize);
        keyedGatherConfig.withMessageSchema(MessageSchema.ofSize(keySize + valueSize, keySize));
    }
    ComputeGraph computeGraph = teraSortTaskGraph.build();
    ExecutionPlan executionPlan = cEnv.getTaskExecutor().plan(computeGraph);
    cEnv.getTaskExecutor().execute(computeGraph, executionPlan);
    cEnv.close();
    LOG.info("Finished Sorting...");
}

Example 2 with BenchmarkResultsRecorder

use of edu.iu.dsc.tws.examples.utils.bench.BenchmarkResultsRecorder in project twister2 by DSC-SPIDAL.

the class BenchTaskWorker method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    if (resultsRecorder == null) {
        resultsRecorder = new BenchmarkResultsRecorder(config, workerId == 0);
    }
    Timing.setDefaultTimingUnit(TimingUnit.NANO_SECONDS);
    jobParameters = JobParameters.build(config);
    computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    if (jobParameters.isStream()) {
        computeGraphBuilder.setMode(OperationMode.STREAMING);
    } else {
        computeGraphBuilder.setMode(OperationMode.BATCH);
    }
    inputDataArray = DataGenerator.generateIntData(jobParameters.getSize());
    buildTaskGraph();
    computeGraph = computeGraphBuilder.build();
    executionPlan = cEnv.getTaskExecutor().plan(computeGraph);
    IExecution execution = cEnv.getTaskExecutor().createExecution(computeGraph, executionPlan).iExecute();
    if (jobParameters.isStream()) {
        while (execution.progress() && (sendersInProgress.get() != 0 || receiversInProgress.get() != 0)) {
        // do nothing
        }
        // now just spin for several iterations to progress the remaining communication.
        // todo fix streaming to return false, when comm is done
        long timeNow = System.currentTimeMillis();
        LOG.info("Streaming Example task will wait 10secs to finish communication...");
        while (System.currentTimeMillis() - timeNow < 10000) {
            execution.progress();
        }
    } else {
        while (execution.progress()) {
        // do nothing
        }
    }
    LOG.info("Stopping execution....");
    execution.stop();
    execution.close();
}

Example 3 with BenchmarkResultsRecorder

use of edu.iu.dsc.tws.examples.utils.bench.BenchmarkResultsRecorder in project twister2 by DSC-SPIDAL.

the class KeyedBenchWorker method execute.

@Override
public void execute(WorkerEnvironment workerEnvironment) {
    this.workerEnv = workerEnvironment;
    workerId = workerEnv.getWorkerId();
    Config cfg = workerEnv.getConfig();
    Timing.setDefaultTimingUnit(TimingUnit.NANO_SECONDS);
    this.resultsRecorder = new BenchmarkResultsRecorder(cfg, workerId == 0);
    // create the job parameters
    this.jobParameters = JobParameters.build(cfg);
    // lets create the task plan
    this.logicalPlan = Utils.createStageLogicalPlan(workerEnv, jobParameters.getTaskStages());
    this.inputDataArray = DataGenerator.generateIntData(jobParameters.getSize());
    // collect experiment data
    experimentData = new ExperimentData();
    // now lets execute
    compute(workerEnv);
    // now progress
    progress();
    // wait for the sync
    try {
        workerEnv.getWorkerController().waitOnBarrier();
    } catch (TimeoutException timeoutException) {
        LOG.log(Level.SEVERE, timeoutException.getMessage(), timeoutException);
    }
    // let allows the specific example to close
    close();
    // lets terminate the communicator
    workerEnv.close();
}

Example 4 with BenchmarkResultsRecorder

use of edu.iu.dsc.tws.examples.utils.bench.BenchmarkResultsRecorder in project twister2 by DSC-SPIDAL.

the class BenchWorker method execute.

@Override
public void execute(WorkerEnvironment workerEnvironment) {
    this.workerEnv = workerEnvironment;
    workerId = workerEnv.getWorkerId();
    Config cfg = workerEnv.getConfig();
    Timing.setDefaultTimingUnit(TimingUnit.NANO_SECONDS);
    // create the job parameters
    this.jobParameters = JobParameters.build(cfg);
    this.resultsRecorder = new BenchmarkResultsRecorder(cfg, workerId == 0);
    // lets create the task plan
    this.logicalPlan = Utils.createStageLogicalPlan(workerEnv, jobParameters.getTaskStages());
    // todo collect experiment data : will be removed
    experimentData = new ExperimentData();
    if (jobParameters.isStream()) {
        experimentData.setOperationMode(OperationMode.STREAMING);
    } else {
        experimentData.setOperationMode(OperationMode.BATCH);
    }
    // todo above will be removed
    this.inputDataArray = generateData();
    // todo below will be removed
    experimentData.setInput(this.inputDataArray);
    experimentData.setTaskStages(jobParameters.getTaskStages());
    experimentData.setIterations(jobParameters.getIterations());
    // todo above will be removed
    // now lets execute
    compute(workerEnv);
    // now communicationProgress
    progress();
    // wait for the sync
    try {
        workerEnv.getWorkerController().waitOnBarrier();
    } catch (TimeoutException timeoutException) {
        LOG.log(Level.SEVERE, timeoutException, () -> timeoutException.getMessage());
    }
    // let allows the specific example to close
    close();
    // lets terminate the communicator
    workerEnv.close();
}