Examples with DrillConfig org.apache.drill.common.config.DrillConfig used on opensource projects

Example 16 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class TestBitBitKerberos method successEncryptionChunkMode.

@Test
public void successEncryptionChunkMode(@Injectable WorkerBee bee, @Injectable final WorkEventBus workBus) throws Exception {
    newConfig = new DrillConfig(config.withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("kerberos"))).withValue(ExecConstants.BIT_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.BIT_AUTHENTICATION_MECHANISM, ConfigValueFactory.fromAnyRef("kerberos")).withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.BIT_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, ConfigValueFactory.fromAnyRef(100000)).withValue(ExecConstants.USE_LOGIN_PRINCIPAL, ConfigValueFactory.fromAnyRef(true)).withValue(BootStrapContext.SERVICE_PRINCIPAL, ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL)).withValue(BootStrapContext.SERVICE_KEYTAB_LOCATION, ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString())), false);
    updateTestCluster(1, newConfig);
    new NonStrictExpectations() {

        {
            workBus.getFragmentManagerIfExists((FragmentHandle) any);
            result = manager;
            workBus.getFragmentManager((FragmentHandle) any);
            result = manager;
        }
    };
    DataConnectionConfig config = new DataConnectionConfig(c1.getAllocator(), c1, new DataServerRequestHandler(workBus, bee));
    DataServer server = new DataServer(config);
    port = server.bind(port, true);
    DrillbitEndpoint ep = DrillbitEndpoint.newBuilder().setAddress("localhost").setDataPort(port).build();
    DataConnectionManager connectionManager = new DataConnectionManager(ep, config);
    DataTunnel tunnel = new DataTunnel(connectionManager);
    AtomicLong max = new AtomicLong(0);
    for (int i = 0; i < 40; i++) {
        long t1 = System.currentTimeMillis();
        tunnel.sendRecordBatch(new TimingOutcome(max), new FragmentWritableBatch(false, QueryId.getDefaultInstance(), 1, 1, 1, 1, getRandomBatch(c1.getAllocator(), 5000)));
        System.out.println(System.currentTimeMillis() - t1);
    }
    System.out.println(String.format("Max time: %d", max.get()));
    assertTrue(max.get() > 2700);
    Thread.sleep(5000);
}

Example 17 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class TestBitRpc method testConnectionBackpressure.

@Test
public void testConnectionBackpressure(@Injectable WorkerBee bee, @Injectable final WorkEventBus workBus) throws Exception {
    DrillConfig config1 = DrillConfig.create();
    final BootStrapContext c = new BootStrapContext(config1, ClassPathScanner.fromPrescan(config1));
    DrillConfig config2 = DrillConfig.create();
    BootStrapContext c2 = new BootStrapContext(config2, ClassPathScanner.fromPrescan(config2));
    final FragmentContext fcon = new MockUp<FragmentContext>() {

        BufferAllocator getAllocator() {
            return c.getAllocator();
        }
    }.getMockInstance();
    final FragmentManager fman = new MockUp<FragmentManager>() {

        int v = 0;

        @Mock
        boolean handle(IncomingDataBatch batch) throws FragmentSetupException, IOException {
            try {
                v++;
                if (v % 10 == 0) {
                    System.out.println("sleeping.");
                    Thread.sleep(3000);
                }
            } catch (InterruptedException e) {
            }
            RawFragmentBatch rfb = batch.newRawFragmentBatch(c.getAllocator());
            rfb.sendOk();
            rfb.release();
            return true;
        }

        public FragmentContext getFragmentContext() {
            return fcon;
        }
    }.getMockInstance();
    new NonStrictExpectations() {

        {
            workBus.getFragmentManagerIfExists((FragmentHandle) any);
            result = fman;
            workBus.getFragmentManager((FragmentHandle) any);
            result = fman;
        }
    };
    int port = 1234;
    DataConnectionConfig config = new DataConnectionConfig(c.getAllocator(), c, new DataServerRequestHandler(workBus, bee));
    DataServer server = new DataServer(config);
    port = server.bind(port, true);
    DrillbitEndpoint ep = DrillbitEndpoint.newBuilder().setAddress("localhost").setDataPort(port).build();
    DataConnectionManager manager = new DataConnectionManager(ep, config);
    DataTunnel tunnel = new DataTunnel(manager);
    AtomicLong max = new AtomicLong(0);
    for (int i = 0; i < 40; i++) {
        long t1 = System.currentTimeMillis();
        tunnel.sendRecordBatch(new TimingOutcome(max), new FragmentWritableBatch(false, QueryId.getDefaultInstance(), 1, 1, 1, 1, getRandomBatch(c.getAllocator(), 5000)));
        System.out.println(System.currentTimeMillis() - t1);
    // System.out.println("sent.");
    }
    System.out.println(String.format("Max time: %d", max.get()));
    assertTrue(max.get() > 2700);
    Thread.sleep(5000);
}

Example 18 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class BuildTimeScan method main.

/**
   * to generate the prescan file during build
   * @param args the root path for the classes where {@link BuildTimeScan#REGISTRY_FILE} is generated
   * @throws Exception
   */
public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new IllegalArgumentException("Usage: java {cp} " + ClassPathScanner.class.getName() + " path/to/scan");
    }
    String basePath = args[0];
    System.out.println("Scanning: " + basePath);
    File registryFile = new File(basePath, REGISTRY_FILE);
    File dir = registryFile.getParentFile();
    if ((!dir.exists() && !dir.mkdirs()) || !dir.isDirectory()) {
        throw new IllegalArgumentException("could not create dir " + dir.getAbsolutePath());
    }
    DrillConfig config = DrillConfig.create();
    // normalize
    if (!basePath.endsWith("/")) {
        basePath = basePath + "/";
    }
    if (!basePath.startsWith("/")) {
        basePath = "/" + basePath;
    }
    URL url = new URL("file:" + basePath);
    Set<URL> markedPaths = ClassPathScanner.getMarkedPaths();
    if (!markedPaths.contains(url)) {
        throw new IllegalArgumentException(url + " not in " + markedPaths);
    }
    List<String> packagePrefixes = ClassPathScanner.getPackagePrefixes(config);
    List<String> baseClasses = ClassPathScanner.getScannedBaseClasses(config);
    List<String> scannedAnnotations = ClassPathScanner.getScannedAnnotations(config);
    ScanResult preScanned = loadExcept(url);
    ScanResult scan = ClassPathScanner.scan(asList(url), packagePrefixes, baseClasses, scannedAnnotations, preScanned);
    save(scan, registryFile);
}

Example 19 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class TestSpoolingBuffer method testMultipleExchangesSingleThread.

@Test
public void testMultipleExchangesSingleThread() throws Exception {
    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    DrillConfig conf = DrillConfig.create("drill-spool-test-module.conf");
    try (Drillbit bit1 = new Drillbit(conf, serviceSet);
        DrillClient client = new DrillClient(conf, serviceSet.getCoordinator())) {
        bit1.run();
        client.connect();
        List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/work/batch/multiple_exchange.json"), Charsets.UTF_8));
        int count = 0;
        for (QueryDataBatch b : results) {
            if (b.getHeader().getRowCount() != 0) {
                count += b.getHeader().getRowCount();
            }
            b.release();
        }
        assertEquals(500024, count);
    }
}

Example 20 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class TestSplitAndTransfer method test.

@Test
public void test() throws Exception {
    final DrillConfig drillConfig = DrillConfig.create();
    final BufferAllocator allocator = RootAllocatorFactory.newRoot(drillConfig);
    final MaterializedField field = MaterializedField.create("field", Types.optional(MinorType.VARCHAR));
    final NullableVarCharVector varCharVector = new NullableVarCharVector(field, allocator);
    varCharVector.allocateNew(10000, 1000);
    final int valueCount = 500;
    final String[] compareArray = new String[valueCount];
    final NullableVarCharVector.Mutator mutator = varCharVector.getMutator();
    for (int i = 0; i < valueCount; i += 3) {
        final String s = String.format("%010d", i);
        mutator.set(i, s.getBytes());
        compareArray[i] = s;
    }
    mutator.setValueCount(valueCount);
    final TransferPair tp = varCharVector.getTransferPair(allocator);
    final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo();
    final Accessor accessor = newVarCharVector.getAccessor();
    final int[][] startLengths = { { 0, 201 }, { 201, 200 }, { 401, 99 } };
    for (final int[] startLength : startLengths) {
        final int start = startLength[0];
        final int length = startLength[1];
        tp.splitAndTransfer(start, length);
        newVarCharVector.getMutator().setValueCount(length);
        for (int i = 0; i < length; i++) {
            final boolean expectedSet = ((start + i) % 3) == 0;
            if (expectedSet) {
                final byte[] expectedValue = compareArray[start + i].getBytes();
                assertFalse(accessor.isNull(i));
                assertArrayEquals(expectedValue, accessor.get(i));
            } else {
                assertTrue(accessor.isNull(i));
            }
        }
        newVarCharVector.clear();
    }
    varCharVector.close();
    allocator.close();
}