Examples with DrillConfig org.apache.drill.common.config.DrillConfig used on opensource projects

Example 21 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class TestParquetPhysicalPlan method testParseParquetPhysicalPlan.

@Test
@Ignore
public void testParseParquetPhysicalPlan() throws Exception {
    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    DrillConfig config = DrillConfig.create();
    try (Drillbit bit1 = new Drillbit(config, serviceSet);
        DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
        bit1.run();
        client.connect();
        List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName), Charsets.UTF_8));
        RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
        int count = 0;
        for (QueryDataBatch b : results) {
            System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
            count += b.getHeader().getRowCount();
            loader.load(b.getHeader().getDef(), b.getData());
            for (VectorWrapper vw : loader) {
                System.out.print(vw.getValueVector().getField().getPath() + ": ");
                ValueVector vv = vw.getValueVector();
                for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
                    Object o = vv.getAccessor().getObject(i);
                    if (o instanceof byte[]) {
                        System.out.print(" [" + new String((byte[]) o) + "]");
                    } else {
                        System.out.print(" [" + vv.getAccessor().getObject(i) + "]");
                    }
                //            break;
                }
                System.out.println();
            }
            loader.clear();
            b.release();
        }
        client.close();
        System.out.println(String.format("Got %d total results", count));
    }
}

Example 22 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class TestParquetPhysicalPlan method testParseParquetPhysicalPlanRemote.

@Test
@Ignore
public void testParseParquetPhysicalPlanRemote() throws Exception {
    DrillConfig config = DrillConfig.create();
    try (DrillClient client = new DrillClient(config)) {
        client.connect();
        ParquetResultsListener listener = new ParquetResultsListener();
        Stopwatch watch = Stopwatch.createStarted();
        client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName), Charsets.UTF_8), listener);
        System.out.println(String.format("Got %d total records in %d seconds", listener.await(), watch.elapsed(TimeUnit.SECONDS)));
        client.close();
    }
}

Example 23 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class ParquetRecordReaderTest method testPerformance.

@Test
@Ignore
public void testPerformance(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Exception {
    final DrillConfig c = DrillConfig.create();
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
    //    new NonStrictExpectations() {
    //      {
    //        context.getAllocator(); result = BufferAllocator.getAllocator(DrillConfig.create());
    //      }
    //    };
    final String fileName = "/tmp/parquet_test_performance.parquet";
    final HashMap<String, FieldInfo> fields = new HashMap<>();
    final ParquetTestProperties props = new ParquetTestProperties(1, 20 * 1000 * 1000, DEFAULT_BYTES_PER_PAGE, fields);
    populateFieldInfoMap(props);
    //generateParquetFile(fileName, props);
    final Configuration dfsConfig = new Configuration();
    final List<Footer> footers = ParquetFileReader.readFooters(dfsConfig, new Path(fileName));
    final Footer f = footers.iterator().next();
    final List<SchemaPath> columns = Lists.newArrayList();
    columns.add(new SchemaPath("_MAP.integer", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.bigInt", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.f", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.d", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.b", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.bin", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.bin2", ExpressionPosition.UNKNOWN));
    int totalRowCount = 0;
    final FileSystem fs = new CachedSingleFileSystem(fileName);
    final BufferAllocator allocator = RootAllocatorFactory.newRoot(c);
    for (int i = 0; i < 25; i++) {
        final ParquetRecordReader rr = new ParquetRecordReader(context, fileName, 0, fs, CodecFactory.createDirectCodecFactory(dfsConfig, new ParquetDirectByteBufferAllocator(allocator), 0), f.getParquetMetadata(), columns, ParquetReaderUtility.DateCorruptionStatus.META_SHOWS_CORRUPTION);
        final TestOutputMutator mutator = new TestOutputMutator(allocator);
        rr.setup(null, mutator);
        final Stopwatch watch = Stopwatch.createStarted();
        int rowCount = 0;
        while ((rowCount = rr.next()) > 0) {
            totalRowCount += rowCount;
        }
        System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
        rr.close();
    }
    allocator.close();
    System.out.println(String.format("Total row count %s", totalRowCount));
}

Example 24 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class StoragePlugins method main.

public static void main(String[] args) throws Exception {
    DrillConfig config = DrillConfig.create();
    ScanResult scanResult = ClassPathScanner.fromPrescan(config);
    LogicalPlanPersistence lpp = new LogicalPlanPersistence(config, scanResult);
    String data = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
    StoragePlugins se = lpp.getMapper().readValue(data, StoragePlugins.class);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    lpp.getMapper().writeValue(System.out, se);
    lpp.getMapper().writeValue(os, se);
    se = lpp.getMapper().readValue(new ByteArrayInputStream(os.toByteArray()), StoragePlugins.class);
    System.out.println(se);
}

Example 25 with DrillConfig

use of org.apache.drill.common.config.DrillConfig in project drill by apache.

the class QueryTestUtil method testRunAndPrint.

/**
   * Execute a SQL query, and print the results.
   *
   * @param drillClient drill client to use
   * @param type type of the query
   * @param queryString query string
   * @return number of rows returned
   * @throws Exception
   */
public static int testRunAndPrint(final DrillClient drillClient, final QueryType type, final String queryString) throws Exception {
    final String query = normalizeQuery(queryString);
    DrillConfig config = drillClient.getConfig();
    AwaitableUserResultsListener resultListener = new AwaitableUserResultsListener(config.getBoolean(TEST_QUERY_PRINTING_SILENT) ? new SilentListener() : new PrintingResultsListener(config, Format.TSV, VectorUtil.DEFAULT_COLUMN_WIDTH));
    drillClient.runQuery(type, query, resultListener);
    return resultListener.await();
}