Search in sources :

Example 1 with CsvScanPlan

use of io.ordinate.engine.physicalplan.CsvScanPlan in project Mycat2 by MyCATApache.

the class CsvScanTest method baseTest.

// @Test
@SneakyThrows
public void baseTest() {
    Path path = Paths.get("D:\\testcsv.csv");
    // 
    // CsvWriter writer = new CsvWriter(path.toFile(), new CsvWriterSettings());
    // for (int i = 0; i < 800_0000; i++) {
    // writer.writeRow(Arrays.asList(i,i));
    // }
    // writer.close();
    StopWatch stopWatch = new StopWatch();
    ExecuteCompiler executeCompiler = new ExecuteCompiler();
    CsvScanPlan csvScan = new ValuesCsvScanPlan(path.toString(), SchemaBuilder.ofArrowType(ArrowTypes.INT64_TYPE, ArrowTypes.STRING_TYPE).toArrow(), executeCompiler.createRootContext());
    Function column = executeCompiler.column(0, csvScan.schema());
    Function add = executeCompiler.call("+", Arrays.asList(column, column));
    RootContext rootContext = executeCompiler.createRootContext();
    PhysicalPlan projection = executeCompiler.project(csvScan, Arrays.asList(add));
    for (int i = 0; i < 100; i++) {
        stopWatch.reset();
        stopWatch.start();
        Observable<VectorSchemaRoot> execute = projection.execute(rootContext);
        AtomicLong count = new AtomicLong(0);
        execute.blockingLatest().forEach(c -> {
            count.getAndAdd(c.getRowCount());
            c.close();
        });
        stopWatch.stop();
        System.out.println("count:" + count);
        Duration duration = Duration.ofMillis(stopWatch.getTime());
        System.out.println(duration.getSeconds());
        System.out.println(stopWatch.toString());
    }
// 
// Thread.sleep(100000000);
}
Also used : Path(java.nio.file.Path) ValuesCsvScanPlan(io.ordinate.engine.physicalplan.ValuesCsvScanPlan) VectorSchemaRoot(org.apache.arrow.vector.VectorSchemaRoot) PhysicalPlan(io.ordinate.engine.physicalplan.PhysicalPlan) ValuesCsvScanPlan(io.ordinate.engine.physicalplan.ValuesCsvScanPlan) CsvScanPlan(io.ordinate.engine.physicalplan.CsvScanPlan) Duration(java.time.Duration) StopWatch(org.apache.commons.lang3.time.StopWatch) RootContext(io.ordinate.engine.record.RootContext) Function(io.ordinate.engine.function.Function) AtomicLong(java.util.concurrent.atomic.AtomicLong) ExecuteCompiler(io.ordinate.engine.builder.ExecuteCompiler) SneakyThrows(lombok.SneakyThrows)

Aggregations

ExecuteCompiler (io.ordinate.engine.builder.ExecuteCompiler)1 Function (io.ordinate.engine.function.Function)1 CsvScanPlan (io.ordinate.engine.physicalplan.CsvScanPlan)1 PhysicalPlan (io.ordinate.engine.physicalplan.PhysicalPlan)1 ValuesCsvScanPlan (io.ordinate.engine.physicalplan.ValuesCsvScanPlan)1 RootContext (io.ordinate.engine.record.RootContext)1 Path (java.nio.file.Path)1 Duration (java.time.Duration)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 SneakyThrows (lombok.SneakyThrows)1 VectorSchemaRoot (org.apache.arrow.vector.VectorSchemaRoot)1 StopWatch (org.apache.commons.lang3.time.StopWatch)1