Examples with Localizer org.apache.storm.localizer.Localizer used on opensource projects

Example 1 with Localizer

use of org.apache.storm.localizer.Localizer in project storm by apache.

the class AsyncLocalizerTest method testRequestDownloadTopologyBlobs.

@Test
public void testRequestDownloadTopologyBlobs() throws Exception {
    final String topoId = "TOPO-12345";
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    ExecutorInfo ei = new ExecutorInfo();
    ei.set_task_start(1);
    ei.set_task_end(1);
    la.add_to_executors(ei);
    final String topoName = "TOPO";
    final int port = 8080;
    final String user = "user";
    final String simpleLocalName = "simple.txt";
    final String simpleKey = "simple";
    final String stormLocal = "/tmp/storm-local/";
    final File userDir = new File(stormLocal, user);
    final String stormRoot = stormLocal + topoId + "/";
    final String localizerRoot = "/tmp/storm-localizer/";
    final String simpleLocalFile = localizerRoot + user + "/simple";
    final String simpleCurrentLocalFile = localizerRoot + user + "/simple.current";
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
    Map<String, Object> simple = new HashMap<>();
    simple.put("localname", simpleLocalName);
    simple.put("uncompress", false);
    topoBlobMap.put(simpleKey, simple);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.STORM_LOCAL_DIR, stormLocal);
    Localizer localizer = mock(Localizer.class);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    ConfigUtils mockedCU = mock(ConfigUtils.class);
    Utils mockedU = mock(Utils.class);
    Map<String, Object> topoConf = new HashMap<>(conf);
    topoConf.put(Config.TOPOLOGY_BLOBSTORE_MAP, topoBlobMap);
    topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
    topoConf.put(Config.TOPOLOGY_NAME, topoName);
    List<LocalizedResource> localizedList = new ArrayList<>();
    LocalizedResource simpleLocal = new LocalizedResource(simpleKey, simpleLocalFile, false);
    localizedList.add(simpleLocal);
    AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
    ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
    Utils origUtils = Utils.setInstance(mockedU);
    try {
        when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
        when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
        when(mockedCU.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(st);
        when(localizer.getLocalUserFileCacheDir(user)).thenReturn(userDir);
        when(localizer.getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir))).thenReturn(localizedList);
        Future<Void> f = al.requestDownloadTopologyBlobs(la, port);
        f.get(20, TimeUnit.SECONDS);
        // We should be done now...
        verify(localizer).getLocalUserFileCacheDir(user);
        verify(ops).fileExists(userDir);
        verify(ops).forceMkdir(userDir);
        verify(localizer).getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir));
        verify(ops).createSymlink(new File(stormRoot, simpleLocalName), new File(simpleCurrentLocalFile));
    } finally {
        al.shutdown();
        ConfigUtils.setInstance(orig);
        Utils.setInstance(origUtils);
    }
}

Example 2 with Localizer

use of org.apache.storm.localizer.Localizer in project storm by apache.

the class AsyncLocalizerTest method testRequestDownloadBaseTopologyBlobs.

@Test
public void testRequestDownloadBaseTopologyBlobs() throws Exception {
    final String topoId = "TOPO";
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    ExecutorInfo ei = new ExecutorInfo();
    ei.set_task_start(1);
    ei.set_task_end(1);
    la.add_to_executors(ei);
    final int port = 8080;
    final String jarKey = topoId + "-stormjar.jar";
    final String codeKey = topoId + "-stormcode.ser";
    final String confKey = topoId + "-stormconf.ser";
    final String stormLocal = "/tmp/storm-local/";
    final String stormRoot = stormLocal + topoId + "/";
    final File fStormRoot = new File(stormRoot);
    ClientBlobStore blobStore = mock(ClientBlobStore.class);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.SUPERVISOR_BLOBSTORE, ClientBlobStore.class.getName());
    conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName());
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    conf.put(Config.STORM_LOCAL_DIR, stormLocal);
    Localizer localizer = mock(Localizer.class);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    ConfigUtils mockedCU = mock(ConfigUtils.class);
    Utils mockedU = mock(Utils.class);
    Map<String, Object> topoConf = new HashMap<>(conf);
    AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
    ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
    Utils origUtils = Utils.setInstance(mockedU);
    try {
        when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
        when(mockedCU.supervisorLocalDirImpl(conf)).thenReturn(stormLocal);
        when(mockedU.newInstanceImpl(ClientBlobStore.class)).thenReturn(blobStore);
        when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
        Future<Void> f = al.requestDownloadBaseTopologyBlobs(la, port);
        f.get(20, TimeUnit.SECONDS);
        // We should be done now...
        verify(blobStore).prepare(conf);
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(jarKey), startsWith(stormLocal), eq(blobStore));
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(codeKey), startsWith(stormLocal), eq(blobStore));
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(confKey), startsWith(stormLocal), eq(blobStore));
        verify(blobStore).shutdown();
        //Extracting the dir from the jar
        verify(mockedU).extractDirFromJarImpl(endsWith("stormjar.jar"), eq("resources"), any(File.class));
        verify(ops).moveDirectoryPreferAtomic(any(File.class), eq(fStormRoot));
        verify(ops).setupStormCodeDir(topoConf, fStormRoot);
        verify(ops, never()).deleteIfExists(any(File.class));
    } finally {
        al.shutdown();
        ConfigUtils.setInstance(orig);
        Utils.setInstance(origUtils);
    }
}

Example 3 with Localizer

use of org.apache.storm.localizer.Localizer in project storm by apache.

the class Supervisor method launch.

/**
     * Launch the supervisor
     */
public void launch() throws Exception {
    LOG.info("Starting Supervisor with conf {}", conf);
    String path = ConfigUtils.supervisorTmpDir(conf);
    FileUtils.cleanDirectory(new File(path));
    Localizer localizer = getLocalizer();
    SupervisorHeartbeat hb = new SupervisorHeartbeat(conf, this);
    hb.run();
    // should synchronize supervisor so it doesn't launch anything after being down (optimization)
    Integer heartbeatFrequency = Utils.getInt(conf.get(Config.SUPERVISOR_HEARTBEAT_FREQUENCY_SECS));
    heartbeatTimer.scheduleRecurring(0, heartbeatFrequency, hb);
    this.eventManager = new EventManagerImp(false);
    this.readState = new ReadClusterState(this);
    Set<String> downloadedTopoIds = SupervisorUtils.readDownloadedTopologyIds(conf);
    for (String topoId : downloadedTopoIds) {
        SupervisorUtils.addBlobReferences(localizer, topoId, conf);
    }
    // do this after adding the references so we don't try to clean things being used
    localizer.startCleaner();
    UpdateBlobs updateBlobsThread = new UpdateBlobs(this);
    if ((Boolean) conf.get(Config.SUPERVISOR_ENABLE)) {
        // This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays up
        // to date even if callbacks don't all work exactly right
        eventTimer.scheduleRecurring(0, 10, new EventManagerPushCallback(readState, eventManager));
        // Blob update thread. Starts with 30 seconds delay, every 30 seconds
        blobUpdateTimer.scheduleRecurring(30, 30, new EventManagerPushCallback(updateBlobsThread, eventManager));
        // supervisor health check
        eventTimer.scheduleRecurring(300, 300, new SupervisorHealthCheck(this));
    }
    LOG.info("Starting supervisor with id {} at host {}.", getId(), getHostName());
}