Search in sources :

Example 1 with CachedSingleFileSystem

use of org.apache.drill.exec.store.CachedSingleFileSystem in project drill by apache.

the class ParquetRecordReaderTest method testPerformance.

@Test
@Ignore
public void testPerformance(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Exception {
    final DrillConfig c = DrillConfig.create();
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
    //    new NonStrictExpectations() {
    //      {
    //        context.getAllocator(); result = BufferAllocator.getAllocator(DrillConfig.create());
    //      }
    //    };
    final String fileName = "/tmp/parquet_test_performance.parquet";
    final HashMap<String, FieldInfo> fields = new HashMap<>();
    final ParquetTestProperties props = new ParquetTestProperties(1, 20 * 1000 * 1000, DEFAULT_BYTES_PER_PAGE, fields);
    populateFieldInfoMap(props);
    //generateParquetFile(fileName, props);
    final Configuration dfsConfig = new Configuration();
    final List<Footer> footers = ParquetFileReader.readFooters(dfsConfig, new Path(fileName));
    final Footer f = footers.iterator().next();
    final List<SchemaPath> columns = Lists.newArrayList();
    columns.add(new SchemaPath("_MAP.integer", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.bigInt", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.f", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.d", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.b", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.bin", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.bin2", ExpressionPosition.UNKNOWN));
    int totalRowCount = 0;
    final FileSystem fs = new CachedSingleFileSystem(fileName);
    final BufferAllocator allocator = RootAllocatorFactory.newRoot(c);
    for (int i = 0; i < 25; i++) {
        final ParquetRecordReader rr = new ParquetRecordReader(context, fileName, 0, fs, CodecFactory.createDirectCodecFactory(dfsConfig, new ParquetDirectByteBufferAllocator(allocator), 0), f.getParquetMetadata(), columns, ParquetReaderUtility.DateCorruptionStatus.META_SHOWS_CORRUPTION);
        final TestOutputMutator mutator = new TestOutputMutator(allocator);
        rr.setup(null, mutator);
        final Stopwatch watch = Stopwatch.createStarted();
        int rowCount = 0;
        while ((rowCount = rr.next()) > 0) {
            totalRowCount += rowCount;
        }
        System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
        rr.close();
    }
    allocator.close();
    System.out.println(String.format("Total row count %s", totalRowCount));
}
Also used : Path(org.apache.hadoop.fs.Path) SchemaPath(org.apache.drill.common.expression.SchemaPath) FragmentContext(org.apache.drill.exec.ops.FragmentContext) Configuration(org.apache.hadoop.conf.Configuration) HashMap(java.util.HashMap) Stopwatch(com.google.common.base.Stopwatch) TestOutputMutator(org.apache.drill.exec.store.TestOutputMutator) BufferAllocator(org.apache.drill.exec.memory.BufferAllocator) ParquetRecordReader(org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader) DrillConfig(org.apache.drill.common.config.DrillConfig) CachedSingleFileSystem(org.apache.drill.exec.store.CachedSingleFileSystem) SchemaPath(org.apache.drill.common.expression.SchemaPath) FileSystem(org.apache.hadoop.fs.FileSystem) CachedSingleFileSystem(org.apache.drill.exec.store.CachedSingleFileSystem) Footer(org.apache.parquet.hadoop.Footer) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) Ignore(org.junit.Ignore) Test(org.junit.Test)

Aggregations

Stopwatch (com.google.common.base.Stopwatch)1 HashMap (java.util.HashMap)1 DrillConfig (org.apache.drill.common.config.DrillConfig)1 SchemaPath (org.apache.drill.common.expression.SchemaPath)1 FunctionImplementationRegistry (org.apache.drill.exec.expr.fn.FunctionImplementationRegistry)1 BufferAllocator (org.apache.drill.exec.memory.BufferAllocator)1 FragmentContext (org.apache.drill.exec.ops.FragmentContext)1 CachedSingleFileSystem (org.apache.drill.exec.store.CachedSingleFileSystem)1 TestOutputMutator (org.apache.drill.exec.store.TestOutputMutator)1 ParquetRecordReader (org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader)1 Configuration (org.apache.hadoop.conf.Configuration)1 FileSystem (org.apache.hadoop.fs.FileSystem)1 Path (org.apache.hadoop.fs.Path)1 Footer (org.apache.parquet.hadoop.Footer)1 Ignore (org.junit.Ignore)1 Test (org.junit.Test)1