Examples with WorkerTopologyContext org.apache.storm.task.WorkerTopologyContext used on opensource projects

Example 6 with WorkerTopologyContext

use of org.apache.storm.task.WorkerTopologyContext in project storm by apache.

the class WorkerState method runWorkerStartHooks.

public void runWorkerStartHooks() {
    WorkerTopologyContext workerContext = getWorkerTopologyContext();
    for (ByteBuffer hook : topology.get_worker_hooks()) {
        byte[] hookBytes = Utils.toByteArray(hook);
        BaseWorkerHook hookObject = Utils.javaDeserialize(hookBytes, BaseWorkerHook.class);
        hookObject.start(topologyConf, workerContext);
    }
}

Example 7 with WorkerTopologyContext

use of org.apache.storm.task.WorkerTopologyContext in project storm by apache.

the class Executor method mkExecutor.

public static Executor mkExecutor(WorkerState workerState, List<Long> executorId, Map<String, String> credentials) {
    Executor executor;
    WorkerTopologyContext workerTopologyContext = workerState.getWorkerTopologyContext();
    List<Integer> taskIds = StormCommon.executorIdToTasks(executorId);
    String componentId = workerTopologyContext.getComponentId(taskIds.get(0));
    String type = getExecutorType(workerTopologyContext, componentId);
    if (StatsUtil.SPOUT.equals(type)) {
        executor = new SpoutExecutor(workerState, executorId, credentials);
        executor.stats = new SpoutExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()));
    } else {
        executor = new BoltExecutor(workerState, executorId, credentials);
        executor.stats = new BoltExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()));
    }
    Map<Integer, Task> idToTask = new HashMap<>();
    for (Integer taskId : taskIds) {
        try {
            Task task = new Task(executor, taskId);
            executor.sendUnanchored(task, StormCommon.SYSTEM_STREAM_ID, new Values("startup"), executor.getExecutorTransfer());
            idToTask.put(taskId, task);
        } catch (IOException ex) {
            throw Utils.wrapInRuntime(ex);
        }
    }
    executor.idToTask = idToTask;
    return executor;
}

Example 8 with WorkerTopologyContext

use of org.apache.storm.task.WorkerTopologyContext in project storm by apache.

the class ShuffleGroupingTest method testShuffleGroupMultiThreaded.

/**
     * Tests that we round robbin correctly with multiple threads using ShuffleGrouping implementation.
     */
@Test
public void testShuffleGroupMultiThreaded() throws InterruptedException, ExecutionException {
    final int numTasks = 6;
    final int groupingExecutionsPerThread = 30000;
    final int numThreads = 10;
    final CustomStreamGrouping grouper = new ShuffleGrouping();
    // Task Id not used, so just pick a static value
    final int inputTaskId = 100;
    // Define our taskIds - the test expects these to be incrementing by one up from zero
    final List<Integer> availableTaskIds = Lists.newArrayList();
    for (int i = 0; i < numTasks; i++) {
        availableTaskIds.add(i);
    }
    final WorkerTopologyContext context = mock(WorkerTopologyContext.class);
    // Call prepare with our available taskIds
    grouper.prepare(context, null, availableTaskIds);
    List<Callable<int[]>> threadTasks = Lists.newArrayList();
    for (int x = 0; x < numThreads; x++) {
        Callable<int[]> threadTask = new Callable<int[]>() {

            @Override
            public int[] call() throws Exception {
                int[] taskCounts = new int[availableTaskIds.size()];
                for (int i = 1; i <= groupingExecutionsPerThread; i++) {
                    List<Integer> taskIds = grouper.chooseTasks(inputTaskId, Lists.newArrayList());
                    // Validate a single task id return
                    assertNotNull("Not null taskId list returned", taskIds);
                    assertEquals("Single task Id returned", 1, taskIds.size());
                    int taskId = taskIds.get(0);
                    assertTrue("TaskId should exist", taskId >= 0 && taskId < availableTaskIds.size());
                    taskCounts[taskId]++;
                }
                return taskCounts;
            }
        };
        // Add to our collection.
        threadTasks.add(threadTask);
    }
    ExecutorService executor = Executors.newFixedThreadPool(threadTasks.size());
    List<Future<int[]>> taskResults = executor.invokeAll(threadTasks);
    // Wait for all tasks to complete
    int[] taskIdTotals = new int[numTasks];
    for (Future taskResult : taskResults) {
        while (!taskResult.isDone()) {
            Thread.sleep(1000);
        }
        int[] taskDistributions = (int[]) taskResult.get();
        for (int i = 0; i < taskDistributions.length; i++) {
            taskIdTotals[i] += taskDistributions[i];
        }
    }
    for (int i = 0; i < numTasks; i++) {
        assertEquals(numThreads * groupingExecutionsPerThread / numTasks, taskIdTotals[i]);
    }
}