Examples with LocalAssignment org.apache.storm.generated.LocalAssignment used on opensource projects

Example 11 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class SlotTest method testReschedule.

@Test
public void testReschedule() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        Container cContainer = mock(Container.class);
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
        when(cContainer.readHeartbeat()).thenReturn(chb);
        when(cContainer.areAllProcessesDead()).thenReturn(false, true);
        String nTopoId = "NEW";
        List<ExecutorInfo> nExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment nAssignment = mkLocalAssignment(nTopoId, nExecList, mkWorkerResources(100.0, 100.0, 100.0));
        ILocalizer localizer = mock(ILocalizer.class);
        Container nContainer = mock(Container.class);
        LocalState state = mock(LocalState.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        when(containerLauncher.launchContainer(port, nAssignment, state)).thenReturn(nContainer);
        LSWorkerHeartbeat nhb = mkWorkerHB(nTopoId, 100, nExecList, Time.currentTimeSecs());
        when(nContainer.readHeartbeat()).thenReturn(nhb, nhb);
        @SuppressWarnings("unchecked") Future<Void> baseFuture = mock(Future.class);
        when(localizer.requestDownloadBaseTopologyBlobs(nAssignment, port)).thenReturn(baseFuture);
        @SuppressWarnings("unchecked") Future<Void> blobFuture = mock(Future.class);
        when(localizer.requestDownloadTopologyBlobs(nAssignment, port)).thenReturn(blobFuture);
        ISupervisor iSuper = mock(ISupervisor.class);
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state);
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).kill();
        verify(localizer).requestDownloadBaseTopologyBlobs(nAssignment, port);
        assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
        assertEquals(nAssignment, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).forceKill();
        assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
        assertEquals(nAssignment, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
        verify(cContainer).cleanUp();
        verify(localizer).releaseSlotFor(cAssignment, port);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
        verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
        verify(localizer).requestDownloadTopologyBlobs(nAssignment, port);
        assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
        assertEquals(nAssignment, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
        verify(containerLauncher).launchContainer(port, nAssignment, state);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 4000);
    }
}

Example 12 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class ReadClusterState method run.

@Override
public synchronized void run() {
    try {
        Runnable syncCallback = new EventManagerPushCallback(this, syncSupEventManager);
        List<String> stormIds = stormClusterState.assignments(syncCallback);
        Map<String, VersionedData<Assignment>> assignmentsSnapshot = getAssignmentsSnapshot(stormClusterState, stormIds, assignmentVersions.get(), syncCallback);
        Map<Integer, LocalAssignment> allAssignments = readAssignments(assignmentsSnapshot);
        if (allAssignments == null) {
            //Something odd happened try again later
            return;
        }
        Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
        HashSet<Integer> assignedPorts = new HashSet<>();
        LOG.debug("Synchronizing supervisor");
        LOG.debug("All assignment: {}", allAssignments);
        LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
        for (Integer port : allAssignments.keySet()) {
            if (iSuper.confirmAssigned(port)) {
                assignedPorts.add(port);
            }
        }
        HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
        allPorts.addAll(slots.keySet());
        Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
        for (Entry<String, List<ProfileRequest>> entry : topoIdToProfilerActions.entrySet()) {
            String topoId = entry.getKey();
            if (entry.getValue() != null) {
                for (ProfileRequest req : entry.getValue()) {
                    NodeInfo ni = req.get_nodeInfo();
                    if (host.equals(ni.get_node())) {
                        Long port = ni.get_port().iterator().next();
                        Set<TopoProfileAction> actions = filtered.get(port.intValue());
                        if (actions == null) {
                            actions = new HashSet<>();
                            filtered.put(port.intValue(), actions);
                        }
                        actions.add(new TopoProfileAction(topoId, req));
                    }
                }
            }
        }
        for (Integer port : allPorts) {
            Slot slot = slots.get(port);
            if (slot == null) {
                slot = mkSlot(port);
                slots.put(port, slot);
                slot.start();
            }
            slot.setNewAssignment(allAssignments.get(port));
            slot.addProfilerActions(filtered.get(port));
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
        throw new RuntimeException(e);
    }
}

Example 13 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class ReadClusterState method readAssignments.

protected Map<Integer, LocalAssignment> readAssignments(Map<String, VersionedData<Assignment>> assignmentsSnapshot) {
    try {
        Map<Integer, LocalAssignment> portLA = new HashMap<>();
        for (Map.Entry<String, VersionedData<Assignment>> assignEntry : assignmentsSnapshot.entrySet()) {
            String topoId = assignEntry.getKey();
            Assignment assignment = assignEntry.getValue().getData();
            Map<Integer, LocalAssignment> portTasks = readMyExecutors(topoId, assignmentId, assignment);
            for (Map.Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
                Integer port = entry.getKey();
                LocalAssignment la = entry.getValue();
                if (!portLA.containsKey(port)) {
                    portLA.put(port, la);
                } else {
                    throw new RuntimeException("Should not have multiple topologies assigned to one port " + port + " " + la + " " + portLA);
                }
            }
        }
        readRetry.set(0);
        return portLA;
    } catch (RuntimeException e) {
        if (readRetry.get() > 2) {
            throw e;
        } else {
            readRetry.addAndGet(1);
        }
        LOG.warn("{} : retrying {} of 3", e.getMessage(), readRetry.get());
        return null;
    }
}

Example 14 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class UpdateBlobs method run.

@Override
public void run() {
    try {
        Map<String, Object> conf = supervisor.getConf();
        Set<String> downloadedStormIds = SupervisorUtils.readDownloadedTopologyIds(conf);
        AtomicReference<Map<Long, LocalAssignment>> newAssignment = supervisor.getCurrAssignment();
        Set<String> assignedStormIds = new HashSet<>();
        for (LocalAssignment localAssignment : newAssignment.get().values()) {
            assignedStormIds.add(localAssignment.get_topology_id());
        }
        for (String stormId : downloadedStormIds) {
            if (assignedStormIds.contains(stormId)) {
                String stormRoot = ConfigUtils.supervisorStormDistRoot(conf, stormId);
                LOG.debug("Checking Blob updates for storm topology id {} With target_dir: {}", stormId, stormRoot);
                updateBlobsForTopology(conf, stormId, supervisor.getLocalizer());
            }
        }
    } catch (Exception e) {
        if (Utils.exceptionCauseIsInstanceOf(TTransportException.class, e)) {
            LOG.error("Network error while updating blobs, will retry again later", e);
        } else if (Utils.exceptionCauseIsInstanceOf(NimbusLeaderNotFoundException.class, e)) {
            LOG.error("Nimbus unavailable to update blobs, will retry again later", e);
        } else {
            throw Utils.wrapInRuntime(e);
        }
    }
}

Example 15 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class ReadClusterState method readMyExecutors.

protected Map<Integer, LocalAssignment> readMyExecutors(String stormId, String assignmentId, Assignment assignment) {
    Map<Integer, LocalAssignment> portTasks = new HashMap<>();
    Map<Long, WorkerResources> slotsResources = new HashMap<>();
    Map<NodeInfo, WorkerResources> nodeInfoWorkerResourcesMap = assignment.get_worker_resources();
    if (nodeInfoWorkerResourcesMap != null) {
        for (Map.Entry<NodeInfo, WorkerResources> entry : nodeInfoWorkerResourcesMap.entrySet()) {
            if (entry.getKey().get_node().equals(assignmentId)) {
                Set<Long> ports = entry.getKey().get_port();
                for (Long port : ports) {
                    slotsResources.put(port, entry.getValue());
                }
            }
        }
    }
    Map<List<Long>, NodeInfo> executorNodePort = assignment.get_executor_node_port();
    if (executorNodePort != null) {
        for (Map.Entry<List<Long>, NodeInfo> entry : executorNodePort.entrySet()) {
            if (entry.getValue().get_node().equals(assignmentId)) {
                for (Long port : entry.getValue().get_port()) {
                    LocalAssignment localAssignment = portTasks.get(port.intValue());
                    if (localAssignment == null) {
                        List<ExecutorInfo> executors = new ArrayList<>();
                        localAssignment = new LocalAssignment(stormId, executors);
                        if (slotsResources.containsKey(port)) {
                            localAssignment.set_resources(slotsResources.get(port));
                        }
                        portTasks.put(port.intValue(), localAssignment);
                    }
                    List<ExecutorInfo> executorInfoList = localAssignment.get_executors();
                    executorInfoList.add(new ExecutorInfo(entry.getKey().get(0).intValue(), entry.getKey().get(entry.getKey().size() - 1).intValue()));
                }
            }
        }
    }
    return portTasks;
}