Examples with JavaConfigurationBuilder org.apache.reef.tang.JavaConfigurationBuilder used on opensource projects

Example 1 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class Mist method getCommandLineConf.

/**
 * Gets configurations from command line args.
 */
private static Configuration getCommandLineConf(final String[] args) throws Exception {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    CommandLine commandLine = new CommandLine(jcb).registerShortNameOfClass(DriverRuntimeType.class);
    commandLine = MistDriverConfigs.addCommandLineConf(commandLine);
    commandLine = MistTaskConfigs.addCommandLineConf(commandLine);
    commandLine = MistMasterConfigs.addCommandLineConf(commandLine);
    commandLine = commandLine.processCommandLine(args);
    if (commandLine == null) {
        // Option '?' was entered and processCommandLine printed the help.
        return null;
    }
    return jcb.build();
}

Example 2 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class MistMasterConfigs method getConfiguration.

public Configuration getConfiguration() {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(OverloadedTaskThreshold.class, String.valueOf(overloadedTaskThreshold));
    if (queryAllocationOption.equals("rr")) {
        jcb.bindImplementation(QueryAllocationManager.class, RoundRobinQueryAllocationManager.class);
    } else if (queryAllocationOption.equals("pot")) {
        jcb.bindImplementation(QueryAllocationManager.class, PowerOfTwoQueryAllocationManager.class);
    } else if (queryAllocationOption.equals("aa")) {
        jcb.bindImplementation(QueryAllocationManager.class, ApplicationAwareQueryAllocationManager.class);
    } else if (queryAllocationOption.equals("min")) {
        jcb.bindImplementation(QueryAllocationManager.class, MinLoadQueryAllocationManager.class);
    } else {
        throw new IllegalArgumentException("Invalid query allocation option!");
    }
    return jcb.build();
}

Example 3 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class MistTaskConfigs method getConfiguration.

/**
 * Get the task configuration.
 * @return configuration
 */
public Configuration getConfiguration() {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    // Parameter
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, Integer.toString(numEventProcessors));
    jcb.bindNamedParameter(MqttSourceKeepAliveSec.class, Integer.toString(mqttSourceKeepAliveSec));
    jcb.bindNamedParameter(MqttSinkKeepAliveSec.class, Integer.toString(mqttSinkKeepAliveSec));
    jcb.bindNamedParameter(MqttSourceClientNumPerBroker.class, Integer.toString(mqttSourceClientNumPerBroker));
    jcb.bindNamedParameter(MqttSinkClientNumPerBroker.class, Integer.toString(mqttSinkClientNumPerBroker));
    jcb.bindNamedParameter(GroupRebalancingPeriod.class, Long.toString(rebalancingPeriod));
    jcb.bindNamedParameter(PeriodicCheckpointPeriod.class, Long.toString(checkpointPeriod));
    // Implementation
    jcb.bindImplementation(ClientToTaskMessage.class, DefaultClientToTaskMessageImpl.class);
    return jcb.build();
}

Example 4 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class MistLauncher method getDriverConfiguration.

/**
 * @return the configuration of the Mist driver.
 */
public static Configuration getDriverConfiguration(final Configuration conf) {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder(conf);
    jcb.bindImplementation(NameResolver.class, LocalNameResolverImpl.class);
    jcb.bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class);
    final Configuration driverConf = DriverConfiguration.CONF.set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(MistDriver.class)).set(DriverConfiguration.DRIVER_IDENTIFIER, "MistDriver").set(DriverConfiguration.ON_DRIVER_STARTED, MistDriver.StartHandler.class).set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, MistDriver.EvaluatorAllocatedHandler.class).set(DriverConfiguration.ON_CONTEXT_ACTIVE, MistDriver.ActiveContextHandler.class).set(DriverConfiguration.ON_TASK_RUNNING, MistDriver.RunningTaskHandler.class).build();
    return Configurations.merge(driverConf, jcb.build());
}

Example 5 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class DefaultGroupSplitterImpl method splitGroup.

@Override
public void splitGroup() {
    LOG.info("GROUP SPLIT START");
    long rebalanceStart = System.currentTimeMillis();
    try {
        // Skip if it is an isolated processor that runs an isolated group
        final List<EventProcessor> eventProcessors = groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup();
        // Overloaded threads
        final List<EventProcessor> overloadedThreads = new LinkedList<>();
        // Underloaded threads
        final PriorityQueue<EventProcessor> underloadedThreads = new PriorityQueue<>(new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                final Double load1 = o1.getLoad();
                final Double load2 = o2.getLoad();
                return load1.compareTo(load2);
            }
        });
        // Calculate each load and total load
        for (final EventProcessor eventProcessor : eventProcessors) {
            final double load = eventProcessor.getLoad();
            if (load > beta) {
                overloadedThreads.add(eventProcessor);
            } else if (load < alpha) {
                underloadedThreads.add(eventProcessor);
            }
        }
        // LOGGING
        // logging(eventProcessors, loadTable);
        double targetLoad = (alpha + beta) / 2;
        int rebNum = 0;
        Collections.sort(overloadedThreads, new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                if (o1.getLoad() < o2.getLoad()) {
                    return 1;
                } else if (o1.getLoad() > o2.getLoad()) {
                    return -1;
                } else {
                    return 0;
                }
            }
        });
        if (!overloadedThreads.isEmpty() && !underloadedThreads.isEmpty()) {
            for (final EventProcessor highLoadThread : overloadedThreads) {
                final Collection<Group> highLoadGroups = groupAllocationTable.getValue(highLoadThread);
                final List<Group> sortedHighLoadGroups = new LinkedList<>(highLoadGroups);
                Collections.sort(sortedHighLoadGroups, new Comparator<Group>() {

                    @Override
                    public int compare(final Group o1, final Group o2) {
                        if (o1.getLoad() < o2.getLoad()) {
                            return 1;
                        } else if (o1.getLoad() > o2.getLoad()) {
                            return -1;
                        } else {
                            return 0;
                        }
                    }
                });
                for (final Group highLoadGroup : sortedHighLoadGroups) {
                    // Split if the load of the high load thread could be less than targetLoad
                    // when we split the high load group
                    int n = 0;
                    if (highLoadThread.getLoad() - highLoadGroup.getLoad() < targetLoad + epsilon && highLoadGroup.size() > 1) {
                        // Sorting queries
                        final List<Query> queries = highLoadGroup.getQueries();
                        final List<Query> sortedQueries = new ArrayList<>(queries);
                        sortedQueries.sort(new Comparator<Query>() {

                            @Override
                            public int compare(final Query o1, final Query o2) {
                                if (o1.getLoad() < o2.getLoad()) {
                                    return 1;
                                } else if (o1.getLoad() > o2.getLoad()) {
                                    return -1;
                                } else {
                                    return 0;
                                }
                            }
                        });
                        final EventProcessor lowLoadThread = underloadedThreads.poll();
                        Group sameGroup = hasGroupOfSameApp(highLoadGroup, lowLoadThread);
                        if (sameGroup == null) {
                            // Split! Create a new group!
                            final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
                            jcb.bindNamedParameter(GroupId.class, groupIdRequestor.requestGroupId(highLoadGroup.getApplicationInfo().getApplicationId()));
                            final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
                            // TODO[MIST-1096]: We should inject executionDags, configVertexMap ... for creating a new group.
                            sameGroup = injector.getInstance(Group.class);
                            sameGroup.setEventProcessor(lowLoadThread);
                            highLoadGroup.getApplicationInfo().addGroup(sameGroup);
                            groupAllocationTable.getValue(lowLoadThread).add(sameGroup);
                            groupMap.putIfAbsent(sameGroup.getGroupId(), sameGroup);
                        }
                        for (final Query movingQuery : sortedQueries) {
                            if (highLoadThread.getLoad() - movingQuery.getLoad() >= targetLoad - epsilon && lowLoadThread.getLoad() + movingQuery.getLoad() <= targetLoad + epsilon) {
                                // Move to the existing group!
                                sameGroup.addQuery(movingQuery);
                                sameGroup.setLoad(sameGroup.getLoad() + movingQuery.getLoad());
                                highLoadGroup.delete(movingQuery);
                                highLoadGroup.setLoad(highLoadGroup.getLoad() - movingQuery.getLoad());
                                lowLoadThread.setLoad(lowLoadThread.getLoad() + movingQuery.getLoad());
                                highLoadThread.setLoad(highLoadThread.getLoad() - movingQuery.getLoad());
                                rebNum += 1;
                                n += 1;
                            }
                        }
                        // Prevent lots of groups from being reassigned
                        if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                            break;
                        }
                        underloadedThreads.add(lowLoadThread);
                        LOG.log(Level.INFO, "GroupSplit from: {0} to {1}, Splitted Group: {3}, number: {2}", new Object[] { highLoadThread, lowLoadThread, n, highLoadGroup.toString() });
                    }
                }
                // Prevent lots of groups from being reassigned
                if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                    break;
                }
            }
        }
        long rebalanceEnd = System.currentTimeMillis();
        LOG.log(Level.INFO, "GroupSplit number: {0}, elapsed time: {1}", new Object[] { rebNum, rebalanceEnd - rebalanceStart });
    // LOG.log(Level.INFO, "-------------TABLE-------------\n{0}",
    // new Object[]{groupAllocationTable.toString()});
    } catch (final Exception e) {
        LOG.log(Level.WARNING, "Exception " + e);
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}