Examples with DrillbitEndpoint org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint used on opensource projects

Example 71 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestFragmentChecker method print.

private void print(String fragmentFile, int bitCount, int expectedFragmentCount) throws Exception {
    System.out.println(String.format("=================Building plan fragments for [%s].  Allowing %d total Drillbits.==================", fragmentFile, bitCount));
    PhysicalPlanReader ppr = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
    Fragment fragmentRoot = getRootFragment(ppr, fragmentFile);
    SimpleParallelizer par = new SimpleParallelizer(1000 * 1000, 5, 10, 1.2);
    List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    DrillbitEndpoint localBit = null;
    for (int i = 0; i < bitCount; i++) {
        DrillbitEndpoint b1 = DrillbitEndpoint.newBuilder().setAddress("localhost").setControlPort(1234 + i).build();
        if (i == 0) {
            localBit = b1;
        }
        endpoints.add(b1);
    }
    final QueryContextInformation queryContextInfo = Utilities.createQueryContextInfo("dummySchemaName", "938ea2d9-7cb9-4baf-9414-a5a0b7777e8e");
    QueryWorkUnit qwu = par.getFragments(new OptionList(), localBit, QueryId.getDefaultInstance(), endpoints, ppr, fragmentRoot, UserSession.Builder.newBuilder().withCredentials(UserBitShared.UserCredentials.newBuilder().setUserName("foo").build()).build(), queryContextInfo);
    System.out.println(String.format("=========ROOT FRAGMENT [%d:%d] =========", qwu.getRootFragment().getHandle().getMajorFragmentId(), qwu.getRootFragment().getHandle().getMinorFragmentId()));
    System.out.print(qwu.getRootFragment().getFragmentJson());
    for (PlanFragment f : qwu.getFragments()) {
        System.out.println(String.format("=========Fragment [%d:%d]=====", f.getHandle().getMajorFragmentId(), f.getHandle().getMinorFragmentId()));
        System.out.print(f.getFragmentJson());
    }
    assertEquals(expectedFragmentCount, qwu.getFragments().size() + 1);
}

Example 72 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class ConnectTriesPropertyTestClusterBits method testConnectFailureUsingZKWithOnlyFakeBits.

@Test
public void testConnectFailureUsingZKWithOnlyFakeBits() throws Exception {
    Properties props = new Properties();
    props.setProperty("tries", "3");
    // Test with Cluster Coordinator connection
    RemoteServiceSet localServiceSet = RemoteServiceSet.getLocalServiceSet();
    DrillClient client = new DrillClient(drillConfig, localServiceSet.getCoordinator());
    // Create couple of fake drillbit endpoints and register with cluster coordinator
    DrillbitEndpoint fakeEndPoint1 = DrillbitEndpoint.newBuilder().setAddress("127.0.0.1").setUserPort(5000).build();
    DrillbitEndpoint fakeEndPoint2 = DrillbitEndpoint.newBuilder().setAddress("127.0.0.1").setUserPort(5001).build();
    ClusterCoordinator.RegistrationHandle fakeEndPoint1Handle = localServiceSet.getCoordinator().register(fakeEndPoint1);
    ClusterCoordinator.RegistrationHandle fakeEndPoint2Handle = localServiceSet.getCoordinator().register(fakeEndPoint2);
    try {
        client.connect(props);
        fail();
    } catch (RpcException ex) {
        assertTrue(ex.getCause() instanceof ExecutionException);
        client.close();
    } finally {
        // Remove the fake drillbits from local cluster cordinator
        localServiceSet.getCoordinator().unregister(fakeEndPoint1Handle);
        localServiceSet.getCoordinator().unregister(fakeEndPoint2Handle);
    }
}

Example 73 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestPauseInjection method pauseOnSpecificBit.

@Test
public void pauseOnSpecificBit() {
    final RemoteServiceSet remoteServiceSet = RemoteServiceSet.getLocalServiceSet();
    final ZookeeperHelper zkHelper = new ZookeeperHelper();
    zkHelper.startZookeeper(1);
    // Creating two drillbits
    final Drillbit drillbit1, drillbit2;
    final DrillConfig drillConfig = zkHelper.getConfig();
    try {
        drillbit1 = Drillbit.start(drillConfig, remoteServiceSet);
        drillbit2 = Drillbit.start(drillConfig, remoteServiceSet);
    } catch (final DrillbitStartupException e) {
        throw new RuntimeException("Failed to start two drillbits.", e);
    }
    final DrillbitContext drillbitContext1 = drillbit1.getContext();
    final DrillbitContext drillbitContext2 = drillbit2.getContext();
    final UserSession session = UserSession.Builder.newBuilder().withCredentials(UserCredentials.newBuilder().setUserName("foo").build()).withUserProperties(UserProperties.getDefaultInstance()).withOptionManager(drillbitContext1.getOptionManager()).build();
    final DrillbitEndpoint drillbitEndpoint1 = drillbitContext1.getEndpoint();
    final String controls = Controls.newBuilder().addPauseOnBit(DummyClass.class, DummyClass.PAUSES, drillbitEndpoint1).build();
    ControlsInjectionUtil.setControls(session, controls);
    {
        final long expectedDuration = 1000L;
        final ExtendedLatch trigger = new ExtendedLatch(1);
        final Pointer<Exception> ex = new Pointer<>();
        final QueryContext queryContext = new QueryContext(session, drillbitContext1, QueryId.getDefaultInstance());
        (new ResumingThread(queryContext, trigger, ex, expectedDuration)).start();
        // test that the pause happens
        final DummyClass dummyClass = new DummyClass(queryContext, trigger);
        final long actualDuration = dummyClass.pauses();
        assertTrue(String.format("Test should stop for at least %d milliseconds.", expectedDuration), expectedDuration <= actualDuration);
        assertTrue("No exception should be thrown.", ex.value == null);
        try {
            queryContext.close();
        } catch (final Exception e) {
            fail("Failed to close query context: " + e);
        }
    }
    {
        final ExtendedLatch trigger = new ExtendedLatch(1);
        final QueryContext queryContext = new QueryContext(session, drillbitContext2, QueryId.getDefaultInstance());
        // if the resume did not happen, the test would hang
        final DummyClass dummyClass = new DummyClass(queryContext, trigger);
        dummyClass.pauses();
        try {
            queryContext.close();
        } catch (final Exception e) {
            fail("Failed to close query context: " + e);
        }
    }
}

Example 74 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestAssignment method manyFiles.

@Test
public void manyFiles() throws Exception {
    List<CompleteFileWork> chunks = generateChunks(1000);
    Iterator<DrillbitEndpoint> incomingEndpointsIterator = Iterators.cycle(endpoints);
    List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
    final int width = widthPerNode * numEndPoints;
    for (int i = 0; i < width; i++) {
        incomingEndpoints.add(incomingEndpointsIterator.next());
    }
    ListMultimap<Integer, CompleteFileWork> mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks);
    System.out.println(mappings.keySet().size());
    for (int i = 0; i < width; i++) {
        Assert.assertTrue("no mapping for entry " + i, mappings.get(i) != null && mappings.get(i).size() > 0);
    }
}

Example 75 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestAssignment method createByteMap.

private EndpointByteMap createByteMap() {
    EndpointByteMap endpointByteMap = new EndpointByteMapImpl();
    Set<DrillbitEndpoint> usedEndpoints = Sets.newHashSet();
    while (usedEndpoints.size() < 3) {
        usedEndpoints.add(getRandom(endpoints));
    }
    for (DrillbitEndpoint ep : usedEndpoints) {
        endpointByteMap.add(ep, FILE_SIZE);
    }
    return endpointByteMap;
}