Examples with HadoopExternalCommunication org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.HadoopExternalCommunication used on opensource projects

Example 1 with HadoopExternalCommunication

use of org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.HadoopExternalCommunication in project ignite by apache.

the class HadoopExternalProcessStarter method run.

/**
     * Run the process.
     *
     * @throws Exception If failed.
     */
public void run() throws Exception {
    File outputDir = outputDirectory();
    initializeStreams(outputDir);
    ExecutorService msgExecSvc = Executors.newFixedThreadPool(Integer.getInteger("MSG_THREAD_POOL_SIZE", Runtime.getRuntime().availableProcessors() * 2));
    IgniteLogger log = logger(outputDir);
    HadoopExternalCommunication comm = new HadoopExternalCommunication(args.nodeId, args.childProcId, new JdkMarshaller(), log, msgExecSvc, "external", args.workDir);
    comm.start();
    HadoopProcessDescriptor nodeDesc = new HadoopProcessDescriptor(args.nodeId, args.parentProcId);
    nodeDesc.address(args.addr);
    nodeDesc.tcpPort(args.tcpPort);
    nodeDesc.sharedMemoryPort(args.shmemPort);
    HadoopChildProcessRunner runner = new HadoopChildProcessRunner();
    runner.start(comm, nodeDesc, msgExecSvc, log);
    System.err.println("Started");
    System.err.flush();
    System.setOut(new PrintStream(out));
    System.setErr(new PrintStream(err));
}

Example 2 with HadoopExternalCommunication

use of org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.HadoopExternalCommunication in project ignite by apache.

the class HadoopExternalTaskExecutor method start.

/** {@inheritDoc} */
@Override
public void start(HadoopContext ctx) throws IgniteCheckedException {
    this.ctx = ctx;
    log = ctx.kernalContext().log(HadoopExternalTaskExecutor.class);
    outputBase = U.resolveWorkDirectory(ctx.kernalContext().config().getWorkDirectory(), "hadoop", false);
    pathSep = System.getProperty("path.separator", U.isWindows() ? ";" : ":");
    initJavaCommand();
    comm = new HadoopExternalCommunication(ctx.localNodeId(), UUID.randomUUID(), ctx.kernalContext().config().getMarshaller(), log, ctx.kernalContext().getSystemExecutorService(), ctx.kernalContext().igniteInstanceName(), ctx.kernalContext().config().getWorkDirectory());
    comm.setListener(new MessageListener());
    comm.start();
    nodeDesc = comm.localProcessDescriptor();
    ctx.kernalContext().ports().registerPort(nodeDesc.tcpPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class);
    if (nodeDesc.sharedMemoryPort() != -1)
        ctx.kernalContext().ports().registerPort(nodeDesc.sharedMemoryPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class);
    jobTracker = ctx.jobTracker();
}

Example 3 with HadoopExternalCommunication

use of org.apache.ignite.internal.processors.hadoop.taskexecutor.external.communication.HadoopExternalCommunication in project ignite by apache.

the class HadoopExternalCommunicationSelfTest method checkSimpleMessageSending.

/**
     * @throws Exception If failed.
     */
private void checkSimpleMessageSending(boolean useShmem) throws Exception {
    UUID parentNodeId = UUID.randomUUID();
    Marshaller marsh = new JdkMarshaller();
    IgniteLogger log = log();
    HadoopExternalCommunication[] comms = new HadoopExternalCommunication[4];
    try {
        String name = "grid";
        TestHadoopListener[] lsnrs = new TestHadoopListener[4];
        int msgs = 10;
        for (int i = 0; i < comms.length; i++) {
            comms[i] = new HadoopExternalCommunication(parentNodeId, UUID.randomUUID(), marsh, log, Executors.newFixedThreadPool(1), name + i, U.defaultWorkDirectory());
            if (useShmem)
                comms[i].setSharedMemoryPort(14000);
            lsnrs[i] = new TestHadoopListener(msgs);
            comms[i].setListener(lsnrs[i]);
            comms[i].start();
        }
        for (int r = 0; r < msgs; r++) {
            for (int from = 0; from < comms.length; from++) {
                for (int to = 0; to < comms.length; to++) {
                    if (from == to)
                        continue;
                    comms[from].sendMessage(comms[to].localProcessDescriptor(), new TestMessage(from, to));
                }
            }
        }
        U.sleep(1000);
        for (TestHadoopListener lsnr : lsnrs) {
            lsnr.await(3_000);
            assertEquals(String.valueOf(lsnr.messages()), msgs * (comms.length - 1), lsnr.messages().size());
        }
    } finally {
        for (HadoopExternalCommunication comm : comms) {
            if (comm != null)
                comm.stop();
        }
    }
}