Examples with LocalAssignment org.apache.storm.generated.LocalAssignment used on opensource projects

Example 1 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class ReadClusterState method run.

@Override
public synchronized void run() {
    try {
        Runnable syncCallback = new EventManagerPushCallback(this, syncSupEventManager);
        List<String> stormIds = stormClusterState.assignments(syncCallback);
        Map<String, VersionedData<Assignment>> assignmentsSnapshot = getAssignmentsSnapshot(stormClusterState, stormIds, assignmentVersions.get(), syncCallback);
        Map<Integer, LocalAssignment> allAssignments = readAssignments(assignmentsSnapshot);
        if (allAssignments == null) {
            //Something odd happened try again later
            return;
        }
        Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
        HashSet<Integer> assignedPorts = new HashSet<>();
        LOG.debug("Synchronizing supervisor");
        LOG.debug("All assignment: {}", allAssignments);
        LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
        for (Integer port : allAssignments.keySet()) {
            if (iSuper.confirmAssigned(port)) {
                assignedPorts.add(port);
            }
        }
        HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
        allPorts.addAll(slots.keySet());
        Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
        for (Entry<String, List<ProfileRequest>> entry : topoIdToProfilerActions.entrySet()) {
            String topoId = entry.getKey();
            if (entry.getValue() != null) {
                for (ProfileRequest req : entry.getValue()) {
                    NodeInfo ni = req.get_nodeInfo();
                    if (host.equals(ni.get_node())) {
                        Long port = ni.get_port().iterator().next();
                        Set<TopoProfileAction> actions = filtered.get(port.intValue());
                        if (actions == null) {
                            actions = new HashSet<>();
                            filtered.put(port.intValue(), actions);
                        }
                        actions.add(new TopoProfileAction(topoId, req));
                    }
                }
            }
        }
        for (Integer port : allPorts) {
            Slot slot = slots.get(port);
            if (slot == null) {
                slot = mkSlot(port);
                slots.put(port, slot);
                slot.start();
            }
            slot.setNewAssignment(allAssignments.get(port));
            slot.addProfilerActions(filtered.get(port));
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
        throw new RuntimeException(e);
    }
}

Example 2 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class ReadClusterState method readAssignments.

protected Map<Integer, LocalAssignment> readAssignments(Map<String, VersionedData<Assignment>> assignmentsSnapshot) {
    try {
        Map<Integer, LocalAssignment> portLA = new HashMap<>();
        for (Map.Entry<String, VersionedData<Assignment>> assignEntry : assignmentsSnapshot.entrySet()) {
            String topoId = assignEntry.getKey();
            Assignment assignment = assignEntry.getValue().getData();
            Map<Integer, LocalAssignment> portTasks = readMyExecutors(topoId, assignmentId, assignment);
            for (Map.Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
                Integer port = entry.getKey();
                LocalAssignment la = entry.getValue();
                if (!portLA.containsKey(port)) {
                    portLA.put(port, la);
                } else {
                    throw new RuntimeException("Should not have multiple topologies assigned to one port " + port + " " + la + " " + portLA);
                }
            }
        }
        readRetry.set(0);
        return portLA;
    } catch (RuntimeException e) {
        if (readRetry.get() > 2) {
            throw e;
        } else {
            readRetry.addAndGet(1);
        }
        LOG.warn("{} : retrying {} of 3", e.getMessage(), readRetry.get());
        return null;
    }
}

Example 3 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class UpdateBlobs method run.

@Override
public void run() {
    try {
        Map<String, Object> conf = supervisor.getConf();
        Set<String> downloadedStormIds = SupervisorUtils.readDownloadedTopologyIds(conf);
        AtomicReference<Map<Long, LocalAssignment>> newAssignment = supervisor.getCurrAssignment();
        Set<String> assignedStormIds = new HashSet<>();
        for (LocalAssignment localAssignment : newAssignment.get().values()) {
            assignedStormIds.add(localAssignment.get_topology_id());
        }
        for (String stormId : downloadedStormIds) {
            if (assignedStormIds.contains(stormId)) {
                String stormRoot = ConfigUtils.supervisorStormDistRoot(conf, stormId);
                LOG.debug("Checking Blob updates for storm topology id {} With target_dir: {}", stormId, stormRoot);
                updateBlobsForTopology(conf, stormId, supervisor.getLocalizer());
            }
        }
    } catch (Exception e) {
        if (Utils.exceptionCauseIsInstanceOf(TTransportException.class, e)) {
            LOG.error("Network error while updating blobs, will retry again later", e);
        } else if (Utils.exceptionCauseIsInstanceOf(NimbusLeaderNotFoundException.class, e)) {
            LOG.error("Nimbus unavailable to update blobs, will retry again later", e);
        } else {
            throw Utils.wrapInRuntime(e);
        }
    }
}

Example 4 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class BasicContainerTest method testLaunch.

@Test
public void testLaunch() throws Exception {
    final String topoId = "test_topology";
    final int port = 8080;
    final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
    final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
    final String workerId = "worker-id";
    final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
    final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
    final File stormcode = new File(distRoot, "stormcode.ser");
    final File stormjar = new File(distRoot, "stormjar.jar");
    final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
    final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
    final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
    final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    superConf.put(Config.STORM_LOG4J2_CONF_DIR, log4jdir);
    superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.slurp(stormcode)).thenReturn(serializedState);
    LocalState ls = mock(LocalState.class);
    checkpoint(() -> {
        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
        mc.launch();
        assertEquals(1, mc.workerCmds.size());
        CommandRun cmd = mc.workerCmds.get(0);
        mc.workerCmds.clear();
        assertListEquals(Arrays.asList("java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-Dlog4j.configurationFile=" + workerConf, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "org.apache.storm.LogWriter", "java", "-server", "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-Dlog4j.configurationFile=" + workerConf, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dtesting=true", "-Djava.library.path=JLP", "-Dstorm.conf.file=", "-Dstorm.options=", "-Djava.io.tmpdir=" + workerTmpDir, "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "org.apache.storm.daemon.worker.Worker", topoId, "SUPERVISOR", String.valueOf(port), workerId), cmd.cmd);
        assertEquals(new File(workerRoot), cmd.pwd);
    }, "storm.home", stormHome, "storm.log.dir", stormLogDir);
}

Example 5 with LocalAssignment

use of org.apache.storm.generated.LocalAssignment in project storm by apache.

the class BasicContainerTest method testRecoveryMiss.

@Test
public void testRecoveryMiss() throws Exception {
    final String topoId = "test_topology";
    final int port = 8080;
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    Map<String, Integer> workerState = new HashMap<String, Integer>();
    workerState.put("somethingelse", port + 1);
    LocalState ls = mock(LocalState.class);
    when(ls.getApprovedWorkers()).thenReturn(workerState);
    try {
        new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(), "SUPERVISOR", port, la, null, ls, null, new HashMap<>(), null, "profile");
        fail("Container recovered worker incorrectly");
    } catch (ContainerRecoveryException e) {
    //Expected
    }
}