Examples with ClusterActionHandler org.apache.whirr.service.ClusterActionHandler used on opensource projects

Example 1 with ClusterActionHandler

use of org.apache.whirr.service.ClusterActionHandler in project whirr by apache.

the class BootstrapClusterActionTest method testDoActionRetriesSucceed.

@SuppressWarnings("unchecked")
@Test
public void testDoActionRetriesSucceed() throws Exception {
    CompositeConfiguration config = new CompositeConfiguration();
    if (System.getProperty("config") != null) {
        config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
    }
    Configuration conf = new PropertiesConfiguration();
    conf.addProperty("whirr.service-name", "test-service");
    conf.addProperty("whirr.cluster-name", "test-cluster");
    conf.addProperty("whirr.instance-templates", "1 hadoop-namenode+hadoop-jobtracker,4 hadoop-datanode+hadoop-tasktracker");
    conf.addProperty("whirr.instance-templates-max-percent-failures", "60 hadoop-datanode+hadoop-tasktracker");
    conf.addProperty("whirr.provider", "ec2");
    config.addConfiguration(conf);
    ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(conf);
    Set<String> jtnn = new HashSet<String>();
    jtnn.add("hadoop-jobtracker");
    jtnn.add("hadoop-namenode");
    Set<String> dntt = new HashSet<String>();
    dntt.add("hadoop-datanode");
    dntt.add("hadoop-tasktracker");
    TestNodeStarterFactory nodeStarterFactory = null;
    ClusterActionHandler handler = mock(ClusterActionHandler.class);
    LoadingCache<String, ClusterActionHandler> handlerMap = convertMapToLoadingCache(ImmutableMap.<String, ClusterActionHandler>builder().put("hadoop-jobtracker", handler).put("hadoop-namenode", handler).put("hadoop-datanode", handler).put("hadoop-tasktracker", handler).build());
    Function<ClusterSpec, ComputeServiceContext> getCompute = mock(Function.class);
    ComputeServiceContext serviceContext = mock(ComputeServiceContext.class);
    ComputeService computeService = mock(ComputeService.class);
    TemplateBuilder templateBuilder = mock(TemplateBuilder.class);
    Template template = mock(Template.class);
    TemplateOptions templateOptions = mock(TemplateOptions.class);
    when(getCompute.apply(clusterSpec)).thenReturn(serviceContext);
    when(serviceContext.getComputeService()).thenReturn(computeService);
    when(computeService.getContext()).thenReturn(serviceContext);
    when(serviceContext.getBackendType()).thenReturn(TypeToken.class.cast(TypeToken.of(Context.class)));
    when(computeService.templateBuilder()).thenReturn(templateBuilder);
    when(templateBuilder.from((TemplateBuilderSpec) any())).thenReturn(templateBuilder);
    when(templateBuilder.options((TemplateOptions) any())).thenReturn(templateBuilder);
    when(templateBuilder.build()).thenReturn(template);
    when(template.getOptions()).thenReturn(templateOptions);
    // here is a scenario when jt+nn fails once, then the retry is successful
    // and from the dn+tt one node fails, then the retry is successful
    Map<Set<String>, Stack<Integer>> reaction = Maps.newHashMap();
    Stack<Integer> jtnnStack = new Stack<Integer>();
    // then ok
    jtnnStack.push(1);
    // initially fail
    jtnnStack.push(0);
    reaction.put(jtnn, jtnnStack);
    Stack<Integer> ddttStack = new Stack<Integer>();
    // 3 from 4, just enough
    ddttStack.push(3);
    reaction.put(dntt, ddttStack);
    nodeStarterFactory = new TestNodeStarterFactory(reaction);
    BootstrapClusterAction bootstrapper = new BootstrapClusterAction(getCompute, handlerMap, nodeStarterFactory);
    bootstrapper.execute(clusterSpec, null);
    if (nodeStarterFactory != null) {
        nodeStarterFactory.validateCompletion();
    }
}

Example 2 with ClusterActionHandler

use of org.apache.whirr.service.ClusterActionHandler in project whirr by apache.

the class BootstrapClusterActionTest method testDoActionRetriesExceeds.

@SuppressWarnings("unchecked")
@Test(expected = IOException.class)
public void testDoActionRetriesExceeds() throws Exception {
    CompositeConfiguration config = new CompositeConfiguration();
    if (System.getProperty("config") != null) {
        config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
    }
    Configuration conf = new PropertiesConfiguration();
    conf.addProperty("whirr.service-name", "test-service");
    conf.addProperty("whirr.cluster-name", "test-cluster");
    conf.addProperty("whirr.instance-templates", "1 hadoop-namenode+hadoop-jobtracker,4 hadoop-datanode+hadoop-tasktracker");
    conf.addProperty("whirr.instance-templates-max-percent-failures", "60 hadoop-datanode+hadoop-tasktracker");
    conf.addProperty("whirr.provider", "ec2");
    config.addConfiguration(conf);
    ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(conf);
    Set<String> jtnn = new HashSet<String>();
    jtnn.add("hadoop-jobtracker");
    jtnn.add("hadoop-namenode");
    Set<String> dntt = new HashSet<String>();
    dntt.add("hadoop-datanode");
    dntt.add("hadoop-tasktracker");
    TestNodeStarterFactory nodeStarterFactory = null;
    ClusterActionHandler handler = mock(ClusterActionHandler.class);
    LoadingCache<String, ClusterActionHandler> handlerMap = convertMapToLoadingCache(ImmutableMap.<String, ClusterActionHandler>builder().put("hadoop-jobtracker", handler).put("hadoop-namenode", handler).put("hadoop-datanode", handler).put("hadoop-tasktracker", handler).build());
    Function<ClusterSpec, ComputeServiceContext> getCompute = mock(Function.class);
    ComputeServiceContext serviceContext = mock(ComputeServiceContext.class);
    ComputeService computeService = mock(ComputeService.class);
    TemplateBuilder templateBuilder = mock(TemplateBuilder.class);
    Template template = mock(Template.class);
    TemplateOptions templateOptions = mock(TemplateOptions.class);
    when(getCompute.apply(clusterSpec)).thenReturn(serviceContext);
    when(serviceContext.getComputeService()).thenReturn(computeService);
    when(computeService.getContext()).thenReturn(serviceContext);
    when(serviceContext.getBackendType()).thenReturn(TypeToken.class.cast(TypeToken.of(Context.class)));
    when(computeService.templateBuilder()).thenReturn(templateBuilder);
    when(templateBuilder.from((TemplateBuilderSpec) any())).thenReturn(templateBuilder);
    when(templateBuilder.options((TemplateOptions) any())).thenReturn(templateBuilder);
    when(templateBuilder.build()).thenReturn(template);
    when(template.getOptions()).thenReturn(templateOptions);
    // here is a scenario when jt+nn does not fail
    // but the dn+tt one node fails 3, then in the retry fails 2
    // at the end result, the cluster will fail, throwing IOException
    Map<Set<String>, Stack<Integer>> reaction = Maps.newHashMap();
    Stack<Integer> jtnnStack = new Stack<Integer>();
    jtnnStack.push(new Integer(1));
    reaction.put(jtnn, jtnnStack);
    Stack<Integer> ddttStack = new Stack<Integer>();
    // 1 from 4, retryRequired
    ddttStack.push(new Integer(1));
    // 1 from 4, still retryRequired
    ddttStack.push(new Integer(1));
    reaction.put(dntt, ddttStack);
    nodeStarterFactory = new TestNodeStarterFactory(reaction);
    BootstrapClusterAction bootstrapper = new BootstrapClusterAction(getCompute, handlerMap, nodeStarterFactory);
    // this should file with too many retries
    bootstrapper.execute(clusterSpec, null);
    if (nodeStarterFactory != null) {
        nodeStarterFactory.validateCompletion();
    }
}

Example 3 with ClusterActionHandler

use of org.apache.whirr.service.ClusterActionHandler in project whirr by apache.

the class Activator method start.

/**
 * Called when this bundle is started so the Framework can perform the
 * bundle-specific activities necessary to start this bundle. This method
 * can be used to register services or to allocate any resources that this
 * bundle needs.
 * <p/>
 * <p/>
 * This method must complete and return to its caller in a timely manner.
 *
 * @param context The execution context of the bundle being started.
 * @throws Exception If this method throws an exception, this
 *                   bundle is marked as stopped and the Framework will remove this
 *                   bundle's listeners, unregister all services registered by this
 *                   bundle, and release all services used by this bundle.
 */
@Override
public void start(BundleContext context) throws Exception {
    // Initialize OSGi based FunctionLoader
    functionLoader = new BundleFunctionLoader(context);
    functionLoader.start();
    defaultClusterController.setHandlerMapFactory(handlerMapFactory);
    byonClusterController.setHandlerMapFactory(handlerMapFactory);
    // Register services
    clusterControllerFactoryRegistration = context.registerService(ClusterControllerFactory.class.getName(), clusterControllerFactory, null);
    handlerMapFactoryRegistration = context.registerService(DynamicHandlerMapFactory.class.getName(), handlerMapFactory, null);
    // Start tracking
    clusterControllerTracker = new ServiceTracker(context, ClusterController.class.getName(), null) {

        @Override
        public Object addingService(ServiceReference reference) {
            Object service = super.addingService(reference);
            clusterControllerFactory.bind((ClusterController) service);
            return service;
        }

        @Override
        public void removedService(ServiceReference reference, Object service) {
            clusterControllerFactory.unbind((ClusterController) service);
            super.removedService(reference, service);
        }
    };
    clusterControllerTracker.open();
    computeServiceTracker = new ServiceTracker(context, ComputeService.class.getName(), null) {

        @Override
        public Object addingService(ServiceReference reference) {
            Object service = context.getService(reference);
            dynamicComputeCache.bind((ComputeService) service);
            return service;
        }

        @Override
        public void removedService(ServiceReference reference, Object service) {
            dynamicComputeCache.unbind((ComputeService) service);
            super.removedService(reference, service);
        }
    };
    computeServiceTracker.open();
    handlerTracker = new ServiceTracker(context, ClusterActionHandler.class.getName(), null) {

        @Override
        public Object addingService(ServiceReference reference) {
            Object service = context.getService(reference);
            handlerMapFactory.bind((ClusterActionHandler) service);
            return service;
        }

        @Override
        public void removedService(ServiceReference reference, Object service) {
            handlerMapFactory.unbind((ClusterActionHandler) service);
            super.removedService(reference, service);
        }
    };
    handlerTracker.open();
    Properties defaultClusterControllerProperties = new Properties();
    defaultClusterControllerProperties.setProperty("name", "default");
    defaultClusterControllerRegistration = context.registerService(ClusterController.class.getName(), defaultClusterController, defaultClusterControllerProperties);
    Properties byonClusterControllerProperties = new Properties();
    byonClusterControllerProperties.setProperty("name", "byon");
    byonClusterControllerRegistration = context.registerService(ClusterController.class.getName(), byonClusterController, byonClusterControllerProperties);
}

Example 4 with ClusterActionHandler

use of org.apache.whirr.service.ClusterActionHandler in project whirr by apache.

the class ByonClusterController method destroyCluster.

/**
 * Destroys a whirr BYON cluster.
 * Because this is a BYON cluster it also stops the services.
 */
@Override
public void destroyCluster(ClusterSpec clusterSpec) throws IOException, InterruptedException {
    // for BYON only
    stopServices(clusterSpec);
    LoadingCache<String, ClusterActionHandler> handlerMap = handlerMapFactory.create();
    ClusterAction destroyer = new ByonClusterAction(DESTROY_ACTION, getCompute(), handlerMap);
    destroyer.execute(clusterSpec, null);
}

Example 5 with ClusterActionHandler

use of org.apache.whirr.service.ClusterActionHandler in project whirr by apache.

the class ByonClusterController method bootstrapCluster.

/**
 * Provisions the hardware for a BYON cluster.
 */
@Override
public Cluster bootstrapCluster(ClusterSpec clusterSpec) throws IOException, InterruptedException {
    LoadingCache<String, ClusterActionHandler> handlerMap = handlerMapFactory.create();
    ClusterAction bootstrapper = new ByonClusterAction(BOOTSTRAP_ACTION, getCompute(), handlerMap);
    Cluster cluster = bootstrapper.execute(clusterSpec, null);
    getClusterStateStore(clusterSpec).save(cluster);
    return cluster;
}