Examples with Group edu.snu.mist.core.task.groupaware.Group used on opensource projects

Example 11 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class FirstFitRebalancerImpl method calculateLoadOfGroupsForLogging.

/**
 * Calculate the load of groups.
 * @param groups groups
 * @return total load
 */
private double calculateLoadOfGroupsForLogging(final Collection<Group> groups) {
    double sum = 0;
    for (final Group group : groups) {
        final double fixedLoad = group.getLoad();
        sum += fixedLoad;
    }
    return sum;
}

Example 12 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class FirstFitRebalancerImpl method firstFitHeuristic.

private Map<Group, EventProcessor> firstFitHeuristic(final List<EventProcessor> eventProcessors, final Map<EventProcessor, Double> bins, final List<Group> items) {
    final Map<Group, EventProcessor> mapping = new HashMap<>(items.size());
    final Iterator<Group> iterator = items.iterator();
    while (iterator.hasNext()) {
        final Group item = iterator.next();
        // find the first bin that can hold the item
        for (final EventProcessor eventProcessor : eventProcessors) {
            final double size = bins.get(eventProcessor);
            final double itemSize = item.getLoad();
            if (size >= itemSize) {
                // This is the first bin that can hold the item!
                iterator.remove();
                mapping.put(item, eventProcessor);
                bins.put(eventProcessor, size - itemSize);
                break;
            }
        }
    }
    if (!items.isEmpty()) {
        // Second
        final Iterator<Group> secondIter = items.iterator();
        while (secondIter.hasNext()) {
            final Group item = secondIter.next();
            // find the first bin that can hold the item
            for (final EventProcessor eventProcessor : eventProcessors) {
                final double size = bins.get(eventProcessor);
                final double itemSize = item.getLoad();
                if (size > 0) {
                    // This is the first bin that can hold the item!
                    secondIter.remove();
                    mapping.put(item, eventProcessor);
                    bins.put(eventProcessor, size - itemSize);
                    break;
                }
            }
        }
    }
    if (!items.isEmpty()) {
        throw new RuntimeException("First-fit algorithm is incorrect");
    }
    return mapping;
}

Example 13 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class AffinityRunnable method run.

/**
 * It executes the events of the selected group during the scheduling period, and re-select another group.
 */
@Override
public void run() {
    try {
        while (!Thread.currentThread().isInterrupted() && !closed) {
            // Pick an active group
            final Group groupInfo = nextGroupSelector.getNextExecutableGroup();
            final long startTime = System.nanoTime();
            numProcessedEvents = groupInfo.processAllEvent();
            final long endTime = System.nanoTime();
            groupInfo.getProcessingEvent().addAndGet(numProcessedEvents);
            groupInfo.getProcessingTime().getAndAdd(endTime - startTime);
            groupInfo.setReady();
        }
    } catch (final Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e + ", OperatorChainManager should not return null");
    }
}

Example 14 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class DefaultEventProcessor method run.

/**
 * It executes the events of the selected group during the scheduling period, and re-select another group.
 */
@Override
public void run() {
    try {
        while (!Thread.currentThread().isInterrupted() && !closed) {
            // Pick an active group
            final Group groupInfo = nextGroupSelector.getNextExecutableGroup();
            final long startTime = System.nanoTime();
            numProcessedEvents = groupInfo.processAllEvent(timeout);
            final long endTime = System.nanoTime();
            groupInfo.getProcessingEvent().addAndGet(numProcessedEvents);
            groupInfo.getProcessingTime().getAndAdd(endTime - startTime);
            groupInfo.setReady();
        /*
        if (LOG.isLoggable(Level.INFO)) {
          LOG.log(Level.INFO, "{0} Process Group {1}, # Processed Events: {2}",
              new Object[]{Thread.currentThread().getName(), groupInfo,  numProcessedEvents});
        }
        */
        }
    } catch (final Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e + ", OperatorChainManager should not return null");
    }
}

Example 15 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class DefaultGroupCheckpointStore method checkpointGroupStates.

@Override
public CheckpointResult checkpointGroupStates(final Tuple<String, Group> tuple) {
    final String groupId = tuple.getKey();
    final Group group = tuple.getValue();
    final GroupCheckpoint checkpoint = group.checkpoint();
    try {
        // Write the file.
        final File storedFile = getGroupCheckpointFile(groupId);
        if (storedFile.exists()) {
            storedFile.delete();
            LOG.log(Level.INFO, "Checkpoint deleted for groupId: {0}", groupId);
        }
        final DataFileWriter<GroupCheckpoint> dataFileWriter = new DataFileWriter<>(groupCheckpointDatumWriter);
        dataFileWriter.create(checkpoint.getSchema(), storedFile);
        dataFileWriter.append(checkpoint);
        dataFileWriter.close();
        LOG.log(Level.INFO, "Checkpoint completed for groupId: {0}", groupId);
    } catch (final Exception e) {
        e.printStackTrace();
        return CheckpointResult.newBuilder().setIsSuccess(false).setMsg("Unsuccessful in checkpointing group " + tuple.getKey()).setPathToCheckpoint("").build();
    }
    // Delete all the unnecessary states within the stateMaps of stateful operators.
    for (final ExecutionDag ed : group.getExecutionDags().values()) {
        for (final ExecutionVertex ev : ed.getDag().getVertices()) {
            if (ev.getType() == ExecutionVertex.Type.OPERATOR) {
                final Operator op = ((DefaultPhysicalOperatorImpl) ev).getOperator();
                if (op instanceof StateHandler) {
                    final StateHandler stateHandler = (StateHandler) op;
                    stateHandler.removeOldStates(checkpoint.getCheckpointTimestamp());
                }
            }
        }
    }
    return CheckpointResult.newBuilder().setIsSuccess(true).setMsg("Successfully checkpointed group " + tuple.getKey()).setPathToCheckpoint(getGroupCheckpointFile(groupId).toString()).build();
}