Examples with LocalAssignment org.apache.storm.generated.LocalAssignment used on opensource projects

Example 6 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class SlotTest method testRunningToEmpty.

@Test
public void testRunningToEmpty() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        Container cContainer = mock(Container.class);
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
        when(cContainer.readHeartbeat()).thenReturn(chb);
        when(cContainer.areAllProcessesDead()).thenReturn(false, true);
        ILocalizer localizer = mock(ILocalizer.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        ISupervisor iSuper = mock(ISupervisor.class);
        LocalState state = mock(LocalState.class);
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state);
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).kill();
        verify(localizer, never()).requestDownloadBaseTopologyBlobs(null, port);
        assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
        assertEquals(null, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).forceKill();
        assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
        assertEquals(null, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.EMPTY, nextState.state);
        verify(cContainer).cleanUp();
        verify(localizer).releaseSlotFor(cAssignment, port);
        assertEquals(null, nextState.container);
        assertEquals(null, nextState.currentAssignment);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.EMPTY, nextState.state);
        assertEquals(null, nextState.container);
        assertEquals(null, nextState.currentAssignment);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.EMPTY, nextState.state);
        assertEquals(null, nextState.container);
        assertEquals(null, nextState.currentAssignment);
        assertTrue(Time.currentTimeMillis() > 3000);
    }
}

Example 7 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class AsyncLocalizerTest method testRequestDownloadTopologyBlobs.

@Test
public void testRequestDownloadTopologyBlobs() throws Exception {
    final String topoId = "TOPO-12345";
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    ExecutorInfo ei = new ExecutorInfo();
    ei.set_task_start(1);
    ei.set_task_end(1);
    la.add_to_executors(ei);
    final String topoName = "TOPO";
    final int port = 8080;
    final String user = "user";
    final String simpleLocalName = "simple.txt";
    final String simpleKey = "simple";
    final String stormLocal = "/tmp/storm-local/";
    final File userDir = new File(stormLocal, user);
    final String stormRoot = stormLocal + topoId + "/";
    final String localizerRoot = "/tmp/storm-localizer/";
    final String simpleLocalFile = localizerRoot + user + "/simple";
    final String simpleCurrentLocalFile = localizerRoot + user + "/simple.current";
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
    Map<String, Object> simple = new HashMap<>();
    simple.put("localname", simpleLocalName);
    simple.put("uncompress", false);
    topoBlobMap.put(simpleKey, simple);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.STORM_LOCAL_DIR, stormLocal);
    Localizer localizer = mock(Localizer.class);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    ConfigUtils mockedCU = mock(ConfigUtils.class);
    Utils mockedU = mock(Utils.class);
    Map<String, Object> topoConf = new HashMap<>(conf);
    topoConf.put(Config.TOPOLOGY_BLOBSTORE_MAP, topoBlobMap);
    topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
    topoConf.put(Config.TOPOLOGY_NAME, topoName);
    List<LocalizedResource> localizedList = new ArrayList<>();
    LocalizedResource simpleLocal = new LocalizedResource(simpleKey, simpleLocalFile, false);
    localizedList.add(simpleLocal);
    AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
    ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
    Utils origUtils = Utils.setInstance(mockedU);
    try {
        when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
        when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
        when(mockedCU.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(st);
        when(localizer.getLocalUserFileCacheDir(user)).thenReturn(userDir);
        when(localizer.getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir))).thenReturn(localizedList);
        Future<Void> f = al.requestDownloadTopologyBlobs(la, port);
        f.get(20, TimeUnit.SECONDS);
        // We should be done now...
        verify(localizer).getLocalUserFileCacheDir(user);
        verify(ops).fileExists(userDir);
        verify(ops).forceMkdir(userDir);
        verify(localizer).getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir));
        verify(ops).createSymlink(new File(stormRoot, simpleLocalName), new File(simpleCurrentLocalFile));
    } finally {
        al.shutdown();
        ConfigUtils.setInstance(orig);
        Utils.setInstance(origUtils);
    }
}

Example 8 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class SlotTest method testRunWithProfileActions.

@Test
public void testRunWithProfileActions() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        Container cContainer = mock(Container.class);
        //NOT going to timeout for a while
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs() + 100);
        when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb);
        when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
        ILocalizer localizer = mock(ILocalizer.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        ISupervisor iSuper = mock(ISupervisor.class);
        LocalState state = mock(LocalState.class);
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state);
        Set<TopoProfileAction> profileActions = new HashSet<>();
        ProfileRequest request = new ProfileRequest();
        request.set_action(ProfileAction.JPROFILE_STOP);
        NodeInfo info = new NodeInfo();
        info.set_node("localhost");
        info.add_to_port(port);
        request.set_nodeInfo(info);
        //3 seconds from now
        request.set_time_stamp(Time.currentTimeMillis() + 3000);
        TopoProfileAction profile = new TopoProfileAction(cTopoId, request);
        profileActions.add(profile);
        Set<TopoProfileAction> expectedPending = new HashSet<>();
        expectedPending.add(profile);
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withProfileActions(profileActions, Collections.<TopoProfileAction>emptySet());
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, false);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, true);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 4000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 5000);
    }
}

Example 9 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class SlotTest method mkLocalAssignment.

static LocalAssignment mkLocalAssignment(String id, List<ExecutorInfo> exec, WorkerResources resources) {
    LocalAssignment ret = new LocalAssignment();
    ret.set_topology_id(id);
    ret.set_executors(exec);
    if (resources != null) {
        ret.set_resources(resources);
    }
    return ret;
}

Example 10 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class SlotTest method testLaunchContainerFromEmpty.

@Test
public void testLaunchContainerFromEmpty() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String topoId = "NEW";
        List<ExecutorInfo> execList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment newAssignment = mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
        ILocalizer localizer = mock(ILocalizer.class);
        Container container = mock(Container.class);
        LocalState state = mock(LocalState.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container);
        LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
        when(container.readHeartbeat()).thenReturn(hb, hb);
        @SuppressWarnings("unchecked") Future<Void> baseFuture = mock(Future.class);
        when(localizer.requestDownloadBaseTopologyBlobs(newAssignment, port)).thenReturn(baseFuture);
        @SuppressWarnings("unchecked") Future<Void> blobFuture = mock(Future.class);
        when(localizer.requestDownloadTopologyBlobs(newAssignment, port)).thenReturn(blobFuture);
        ISupervisor iSuper = mock(ISupervisor.class);
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state);
        DynamicState dynamicState = new DynamicState(null, null, null).withNewAssignment(newAssignment);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        verify(localizer).requestDownloadBaseTopologyBlobs(newAssignment, port);
        assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
        assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
        assertEquals(newAssignment, nextState.pendingLocalization);
        assertEquals(0, Time.currentTimeMillis());
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
        verify(localizer).requestDownloadTopologyBlobs(newAssignment, port);
        assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
        assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
        assertEquals(newAssignment, nextState.pendingLocalization);
        assertEquals(0, Time.currentTimeMillis());
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
        verify(containerLauncher).launchContainer(port, newAssignment, state);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertEquals(0, Time.currentTimeMillis());
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertEquals(0, Time.currentTimeMillis());
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertTrue(Time.currentTimeMillis() > 2000);
    }
}