use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.
the class Mist method getCommandLineConf.
/**
* Gets configurations from command line args.
*/
private static Configuration getCommandLineConf(final String[] args) throws Exception {
final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
CommandLine commandLine = new CommandLine(jcb).registerShortNameOfClass(DriverRuntimeType.class);
commandLine = MistDriverConfigs.addCommandLineConf(commandLine);
commandLine = MistTaskConfigs.addCommandLineConf(commandLine);
commandLine = MistMasterConfigs.addCommandLineConf(commandLine);
commandLine = commandLine.processCommandLine(args);
if (commandLine == null) {
// Option '?' was entered and processCommandLine printed the help.
return null;
}
return jcb.build();
}
use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.
the class MistMasterConfigs method getConfiguration.
public Configuration getConfiguration() {
final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
jcb.bindNamedParameter(OverloadedTaskThreshold.class, String.valueOf(overloadedTaskThreshold));
if (queryAllocationOption.equals("rr")) {
jcb.bindImplementation(QueryAllocationManager.class, RoundRobinQueryAllocationManager.class);
} else if (queryAllocationOption.equals("pot")) {
jcb.bindImplementation(QueryAllocationManager.class, PowerOfTwoQueryAllocationManager.class);
} else if (queryAllocationOption.equals("aa")) {
jcb.bindImplementation(QueryAllocationManager.class, ApplicationAwareQueryAllocationManager.class);
} else if (queryAllocationOption.equals("min")) {
jcb.bindImplementation(QueryAllocationManager.class, MinLoadQueryAllocationManager.class);
} else {
throw new IllegalArgumentException("Invalid query allocation option!");
}
return jcb.build();
}
use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.
the class MistTaskConfigs method getConfiguration.
/**
* Get the task configuration.
* @return configuration
*/
public Configuration getConfiguration() {
final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
// Parameter
jcb.bindNamedParameter(DefaultNumEventProcessors.class, Integer.toString(numEventProcessors));
jcb.bindNamedParameter(MqttSourceKeepAliveSec.class, Integer.toString(mqttSourceKeepAliveSec));
jcb.bindNamedParameter(MqttSinkKeepAliveSec.class, Integer.toString(mqttSinkKeepAliveSec));
jcb.bindNamedParameter(MqttSourceClientNumPerBroker.class, Integer.toString(mqttSourceClientNumPerBroker));
jcb.bindNamedParameter(MqttSinkClientNumPerBroker.class, Integer.toString(mqttSinkClientNumPerBroker));
jcb.bindNamedParameter(GroupRebalancingPeriod.class, Long.toString(rebalancingPeriod));
jcb.bindNamedParameter(PeriodicCheckpointPeriod.class, Long.toString(checkpointPeriod));
// Implementation
jcb.bindImplementation(ClientToTaskMessage.class, DefaultClientToTaskMessageImpl.class);
return jcb.build();
}
use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.
the class MistLauncher method getDriverConfiguration.
/**
* @return the configuration of the Mist driver.
*/
public static Configuration getDriverConfiguration(final Configuration conf) {
final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder(conf);
jcb.bindImplementation(NameResolver.class, LocalNameResolverImpl.class);
jcb.bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class);
final Configuration driverConf = DriverConfiguration.CONF.set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(MistDriver.class)).set(DriverConfiguration.DRIVER_IDENTIFIER, "MistDriver").set(DriverConfiguration.ON_DRIVER_STARTED, MistDriver.StartHandler.class).set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, MistDriver.EvaluatorAllocatedHandler.class).set(DriverConfiguration.ON_CONTEXT_ACTIVE, MistDriver.ActiveContextHandler.class).set(DriverConfiguration.ON_TASK_RUNNING, MistDriver.RunningTaskHandler.class).build();
return Configurations.merge(driverConf, jcb.build());
}
use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.
the class DefaultGroupSplitterImpl method splitGroup.
@Override
public void splitGroup() {
LOG.info("GROUP SPLIT START");
long rebalanceStart = System.currentTimeMillis();
try {
// Skip if it is an isolated processor that runs an isolated group
final List<EventProcessor> eventProcessors = groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup();
// Overloaded threads
final List<EventProcessor> overloadedThreads = new LinkedList<>();
// Underloaded threads
final PriorityQueue<EventProcessor> underloadedThreads = new PriorityQueue<>(new Comparator<EventProcessor>() {
@Override
public int compare(final EventProcessor o1, final EventProcessor o2) {
final Double load1 = o1.getLoad();
final Double load2 = o2.getLoad();
return load1.compareTo(load2);
}
});
// Calculate each load and total load
for (final EventProcessor eventProcessor : eventProcessors) {
final double load = eventProcessor.getLoad();
if (load > beta) {
overloadedThreads.add(eventProcessor);
} else if (load < alpha) {
underloadedThreads.add(eventProcessor);
}
}
// LOGGING
// logging(eventProcessors, loadTable);
double targetLoad = (alpha + beta) / 2;
int rebNum = 0;
Collections.sort(overloadedThreads, new Comparator<EventProcessor>() {
@Override
public int compare(final EventProcessor o1, final EventProcessor o2) {
if (o1.getLoad() < o2.getLoad()) {
return 1;
} else if (o1.getLoad() > o2.getLoad()) {
return -1;
} else {
return 0;
}
}
});
if (!overloadedThreads.isEmpty() && !underloadedThreads.isEmpty()) {
for (final EventProcessor highLoadThread : overloadedThreads) {
final Collection<Group> highLoadGroups = groupAllocationTable.getValue(highLoadThread);
final List<Group> sortedHighLoadGroups = new LinkedList<>(highLoadGroups);
Collections.sort(sortedHighLoadGroups, new Comparator<Group>() {
@Override
public int compare(final Group o1, final Group o2) {
if (o1.getLoad() < o2.getLoad()) {
return 1;
} else if (o1.getLoad() > o2.getLoad()) {
return -1;
} else {
return 0;
}
}
});
for (final Group highLoadGroup : sortedHighLoadGroups) {
// Split if the load of the high load thread could be less than targetLoad
// when we split the high load group
int n = 0;
if (highLoadThread.getLoad() - highLoadGroup.getLoad() < targetLoad + epsilon && highLoadGroup.size() > 1) {
// Sorting queries
final List<Query> queries = highLoadGroup.getQueries();
final List<Query> sortedQueries = new ArrayList<>(queries);
sortedQueries.sort(new Comparator<Query>() {
@Override
public int compare(final Query o1, final Query o2) {
if (o1.getLoad() < o2.getLoad()) {
return 1;
} else if (o1.getLoad() > o2.getLoad()) {
return -1;
} else {
return 0;
}
}
});
final EventProcessor lowLoadThread = underloadedThreads.poll();
Group sameGroup = hasGroupOfSameApp(highLoadGroup, lowLoadThread);
if (sameGroup == null) {
// Split! Create a new group!
final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
jcb.bindNamedParameter(GroupId.class, groupIdRequestor.requestGroupId(highLoadGroup.getApplicationInfo().getApplicationId()));
final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
// TODO[MIST-1096]: We should inject executionDags, configVertexMap ... for creating a new group.
sameGroup = injector.getInstance(Group.class);
sameGroup.setEventProcessor(lowLoadThread);
highLoadGroup.getApplicationInfo().addGroup(sameGroup);
groupAllocationTable.getValue(lowLoadThread).add(sameGroup);
groupMap.putIfAbsent(sameGroup.getGroupId(), sameGroup);
}
for (final Query movingQuery : sortedQueries) {
if (highLoadThread.getLoad() - movingQuery.getLoad() >= targetLoad - epsilon && lowLoadThread.getLoad() + movingQuery.getLoad() <= targetLoad + epsilon) {
// Move to the existing group!
sameGroup.addQuery(movingQuery);
sameGroup.setLoad(sameGroup.getLoad() + movingQuery.getLoad());
highLoadGroup.delete(movingQuery);
highLoadGroup.setLoad(highLoadGroup.getLoad() - movingQuery.getLoad());
lowLoadThread.setLoad(lowLoadThread.getLoad() + movingQuery.getLoad());
highLoadThread.setLoad(highLoadThread.getLoad() - movingQuery.getLoad());
rebNum += 1;
n += 1;
}
}
// Prevent lots of groups from being reassigned
if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
break;
}
underloadedThreads.add(lowLoadThread);
LOG.log(Level.INFO, "GroupSplit from: {0} to {1}, Splitted Group: {3}, number: {2}", new Object[] { highLoadThread, lowLoadThread, n, highLoadGroup.toString() });
}
}
// Prevent lots of groups from being reassigned
if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
break;
}
}
}
long rebalanceEnd = System.currentTimeMillis();
LOG.log(Level.INFO, "GroupSplit number: {0}, elapsed time: {1}", new Object[] { rebNum, rebalanceEnd - rebalanceStart });
// LOG.log(Level.INFO, "-------------TABLE-------------\n{0}",
// new Object[]{groupAllocationTable.toString()});
} catch (final Exception e) {
LOG.log(Level.WARNING, "Exception " + e);
e.printStackTrace();
throw new RuntimeException(e);
}
}
Aggregations