Examples with FragmentManager org.apache.drill.exec.work.fragment.FragmentManager used on opensource projects

Example 1 with FragmentManager

use of org.apache.drill.exec.work.fragment.FragmentManager in project drill by apache.

the class WorkEventBus method removeFragmentManager.

/**
   * Removes fragment manager (for the corresponding the handle) from the work event bus. This method can be called
   * multiple times. The manager will be removed only once (the first call).
   * @param handle the handle to the fragment
   */
public void removeFragmentManager(final FragmentHandle handle) {
    if (logger.isDebugEnabled()) {
        logger.debug("Removing fragment manager: {}", QueryIdHelper.getQueryIdentifier(handle));
    }
    synchronized (this) {
        final FragmentManager manager = managers.get(handle);
        if (manager != null) {
            recentlyFinishedFragments.put(handle, 1);
            managers.remove(handle);
        } else {
            logger.warn("Fragment {} not found in the work bus.", QueryIdHelper.getQueryIdentifier(handle));
        }
    }
}

Example 2 with FragmentManager

use of org.apache.drill.exec.work.fragment.FragmentManager in project drill by apache.

the class TestBitRpc method testConnectionBackpressure.

@Test
public void testConnectionBackpressure(@Injectable WorkerBee bee, @Injectable final WorkEventBus workBus) throws Exception {
    DrillConfig config1 = DrillConfig.create();
    final BootStrapContext c = new BootStrapContext(config1, ClassPathScanner.fromPrescan(config1));
    DrillConfig config2 = DrillConfig.create();
    BootStrapContext c2 = new BootStrapContext(config2, ClassPathScanner.fromPrescan(config2));
    final FragmentContext fcon = new MockUp<FragmentContext>() {

        BufferAllocator getAllocator() {
            return c.getAllocator();
        }
    }.getMockInstance();
    final FragmentManager fman = new MockUp<FragmentManager>() {

        int v = 0;

        @Mock
        boolean handle(IncomingDataBatch batch) throws FragmentSetupException, IOException {
            try {
                v++;
                if (v % 10 == 0) {
                    System.out.println("sleeping.");
                    Thread.sleep(3000);
                }
            } catch (InterruptedException e) {
            }
            RawFragmentBatch rfb = batch.newRawFragmentBatch(c.getAllocator());
            rfb.sendOk();
            rfb.release();
            return true;
        }

        public FragmentContext getFragmentContext() {
            return fcon;
        }
    }.getMockInstance();
    new NonStrictExpectations() {

        {
            workBus.getFragmentManagerIfExists((FragmentHandle) any);
            result = fman;
            workBus.getFragmentManager((FragmentHandle) any);
            result = fman;
        }
    };
    int port = 1234;
    DataConnectionConfig config = new DataConnectionConfig(c.getAllocator(), c, new DataServerRequestHandler(workBus, bee));
    DataServer server = new DataServer(config);
    port = server.bind(port, true);
    DrillbitEndpoint ep = DrillbitEndpoint.newBuilder().setAddress("localhost").setDataPort(port).build();
    DataConnectionManager manager = new DataConnectionManager(ep, config);
    DataTunnel tunnel = new DataTunnel(manager);
    AtomicLong max = new AtomicLong(0);
    for (int i = 0; i < 40; i++) {
        long t1 = System.currentTimeMillis();
        tunnel.sendRecordBatch(new TimingOutcome(max), new FragmentWritableBatch(false, QueryId.getDefaultInstance(), 1, 1, 1, 1, getRandomBatch(c.getAllocator(), 5000)));
        System.out.println(System.currentTimeMillis() - t1);
    // System.out.println("sent.");
    }
    System.out.println(String.format("Max time: %d", max.get()));
    assertTrue(max.get() > 2700);
    Thread.sleep(5000);
}

Example 3 with FragmentManager

use of org.apache.drill.exec.work.fragment.FragmentManager in project drill by apache.

the class ControlMessageHandler method receivingFragmentFinished.

private Ack receivingFragmentFinished(final FinishedReceiver finishedReceiver) {
    final FragmentManager manager = bee.getContext().getWorkBus().getFragmentManagerIfExists(finishedReceiver.getSender());
    FragmentExecutor executor;
    if (manager != null) {
        manager.receivingFragmentFinished(finishedReceiver.getReceiver());
    } else {
        executor = bee.getFragmentRunner(finishedReceiver.getSender());
        if (executor != null) {
            executor.receivingFragmentFinished(finishedReceiver.getReceiver());
        } else {
            logger.warn("Dropping request for early fragment termination for path {} -> {} as path to executor unavailable.", QueryIdHelper.getQueryIdentifier(finishedReceiver.getSender()), QueryIdHelper.getQueryIdentifier(finishedReceiver.getReceiver()));
        }
    }
    return Acks.OK;
}

Example 4 with FragmentManager

use of org.apache.drill.exec.work.fragment.FragmentManager in project drill by axbaretto.

the class DataServerRequestHandler method submit.

private void submit(IncomingDataBatch batch, int minorStart, int minorStopExclusive) throws FragmentSetupException, IOException {
    for (int minor = minorStart; minor < minorStopExclusive; minor++) {
        final FragmentManager manager = workBus.getFragmentManager(getHandle(batch.getHeader(), minor));
        if (manager == null) {
            // A missing manager means the query already terminated. We can simply drop this data.
            continue;
        }
        final boolean canRun = manager.handle(batch);
        if (canRun) {
            // logger.debug("Arriving batch means local batch can run, starting local batch.");
            /*
         * If we've reached the canRun threshold, we'll proceed. This expects manager.handle() to only return a single
         * true. This is guaranteed by the interface.
         */
            bee.startFragmentPendingRemote(manager);
        }
    }
}

Example 5 with FragmentManager

use of org.apache.drill.exec.work.fragment.FragmentManager in project drill by axbaretto.

the class ControlMessageHandler method resumeFragment.

private Ack resumeFragment(final FragmentHandle handle) {
    // resume a pending fragment
    final FragmentManager manager = bee.getContext().getWorkBus().getFragmentManager(handle);
    if (manager != null) {
        manager.unpause();
        return Acks.OK;
    }
    // resume a paused fragment
    final FragmentExecutor runner = bee.getFragmentRunner(handle);
    if (runner != null) {
        runner.unpause();
        return Acks.OK;
    }
    // fragment completed or does not exist
    logger.warn("Dropping request to resume fragment. {} does not exist.", QueryIdHelper.getQueryIdentifier(handle));
    return Acks.OK;
}