Search in sources :

Example 1 with HetuHdfsFileSystemClient

use of io.hetu.core.filesystem.HetuHdfsFileSystemClient in project hetu-core by openlookeng.

the class TestOrderByOperator method testCaptureRestoreWithSpillToHdfsEnabled.

/**
 * This test is supposed to consume 4 pages and produce the output page with sorted ordering.
 * The spilling and capturing('capture1') happened after the first 2 pages added into the operator.
 * The operator is rescheduled after 4 pages added (but before finish() is called).
 *
 * @throws Exception
 */
@Test
public void testCaptureRestoreWithSpillToHdfsEnabled() throws Exception {
    // Initialization
    Path spillPath = Paths.get("/tmp/hetu/snapshot/");
    HetuHdfsFileSystemClient fs = getLocalHdfs();
    when(fileSystemClientManager.getFileSystemClient(any(String.class), any(Path.class))).thenReturn(fs);
    GenericSpillerFactory genericSpillerFactory = createGenericSpillerFactory(spillPath, fileSystemClientManager, true, "hdfs");
    SnapshotConfig snapshotConfig = new SnapshotConfig();
    snapshotConfig.setSpillProfile("hdfs");
    snapshotConfig.setSpillToHdfs(true);
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.initialize();
    List<Page> input1 = rowPagesBuilder(VARCHAR, BIGINT).row("a", 1L).row("b", 2L).pageBreak().row("b", 3L).row("a", 4L).build();
    List<Page> input2 = rowPagesBuilder(VARCHAR, BIGINT).row("c", 4L).row("d", 6L).pageBreak().row("c", 2L).row("d", 3L).build();
    OrderByOperatorFactory operatorFactory = new OrderByOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), ImmutableList.of(0, 1), 10, ImmutableList.of(0, 1), ImmutableList.of(ASC_NULLS_LAST, DESC_NULLS_LAST), new PagesIndex.TestingFactory(false), true, Optional.of(genericSpillerFactory), new OrderingCompiler(), false);
    DriverContext driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
    driverContext.getPipelineContext().getTaskContext().getSnapshotManager().setTotalComponents(1);
    OrderByOperator orderByOperator = (OrderByOperator) operatorFactory.createOperator(driverContext);
    // Step1: add the first 2 pages
    for (Page page : input1) {
        orderByOperator.addInput(page);
    }
    // Step2: spilling happened here
    getFutureValue(orderByOperator.startMemoryRevoke());
    orderByOperator.finishMemoryRevoke();
    // Step3: add a marker page to make 'capture1' happened
    MarkerPage marker = MarkerPage.snapshotPage(1);
    orderByOperator.addInput(marker);
    // Step4: add another 2 pages
    for (Page page : input2) {
        orderByOperator.addInput(page);
    }
    // Step5: assume the task is rescheduled due to failure and everything is re-constructed
    driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
    operatorFactory = new OrderByOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), ImmutableList.of(0, 1), 10, ImmutableList.of(0, 1), ImmutableList.of(ASC_NULLS_LAST, DESC_NULLS_LAST), new PagesIndex.TestingFactory(false), true, Optional.of(genericSpillerFactory), new OrderingCompiler(), false);
    orderByOperator = (OrderByOperator) operatorFactory.createOperator(driverContext);
    // Step6: restore to 'capture1', the spiller should contains the reference of the first 2 pages for now.
    MarkerPage resumeMarker = MarkerPage.resumePage(1);
    orderByOperator.addInput(resumeMarker);
    // Step7: continue to add another 2 pages
    for (Page page : input2) {
        orderByOperator.addInput(page);
    }
    orderByOperator.finish();
    // Compare the results
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT).row("a", 4L).row("a", 1L).row("b", 3L).row("b", 2L).row("c", 4L).row("c", 2L).row("d", 6L).row("d", 3L).build();
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    Page p = orderByOperator.getOutput();
    while (p instanceof MarkerPage) {
        p = orderByOperator.getOutput();
    }
    outputPages.add(p);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), outputPages.build());
    Assert.assertEquals(actual, expected);
}
Also used : Path(java.nio.file.Path) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) ImmutableList(com.google.common.collect.ImmutableList) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) SnapshotConfig(io.prestosql.snapshot.SnapshotConfig) HetuHdfsFileSystemClient(io.hetu.core.filesystem.HetuHdfsFileSystemClient) SnapshotUtils(io.prestosql.snapshot.SnapshotUtils) OrderByOperatorFactory(io.prestosql.operator.OrderByOperator.OrderByOperatorFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) Test(org.testng.annotations.Test)

Example 2 with HetuHdfsFileSystemClient

use of io.hetu.core.filesystem.HetuHdfsFileSystemClient in project hetu-core by openlookeng.

the class TestWindowOperator method testCaptureRestoreWithSpillToHdfsEnabled.

@Test
public void testCaptureRestoreWithSpillToHdfsEnabled() throws Exception {
    // Initialization
    Path spillPath = Paths.get("/tmp/hetu/snapshot/");
    HetuHdfsFileSystemClient fs = getLocalHdfs();
    when(fileSystemClientManager.getFileSystemClient(any(String.class), any(Path.class))).thenReturn(fs);
    GenericSpillerFactory genericSpillerFactory = createGenericSpillerFactory(spillPath, fileSystemClientManager, false, null);
    SnapshotConfig snapshotConfig = new SnapshotConfig();
    snapshotConfig.setSpillProfile("hdfs");
    snapshotConfig.setSpillToHdfs(true);
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.initialize();
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    List<Page> input1 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("b", -1L, -0.1, true).row("a", 2L, 0.3, false).row("a", 4L, 0.2, true).pageBreak().row("b", 5L, 0.4, false).row("a", 6L, 0.1, true).build();
    List<Page> input2 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("c", -1L, -0.1, true).row("d", 2L, 0.3, false).row("c", 4L, 0.2, true).pageBreak().row("d", 5L, 0.4, false).build();
    WindowOperatorFactory operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    DriverContext driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
    WindowOperator windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step1: add the first 2 pages
    for (Page page : input1) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step2: spilling happened here
    getFutureValue(windowOperator.startMemoryRevoke());
    windowOperator.finishMemoryRevoke();
    // Step3: add a marker page to make 'capture1' happened
    MarkerPage marker = MarkerPage.snapshotPage(1);
    windowOperator.addInput(marker);
    windowOperator.getOutput();
    // Step4: add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step5: assume the task is rescheduled due to failure and everything is re-constructed
    driverContext = createDriverContext(8, TEST_SNAPSHOT_SESSION);
    operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step6: restore to 'capture1', the spiller should contains the reference of the first 2 pages for now.
    MarkerPage resumeMarker = MarkerPage.resumePage(1);
    windowOperator.addInput(resumeMarker);
    windowOperator.getOutput();
    // Step7: continue to add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    windowOperator.finish();
    // Compare the results
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT).row("a", 2L, 0.3, false, 1L).row("a", 4L, 0.2, true, 2L).row("a", 6L, 0.1, true, 3L).row("b", -1L, -0.1, true, 1L).row("b", 5L, 0.4, false, 2L).row("c", -1L, -0.1, true, 1L).row("c", 4L, 0.2, true, 2L).row("d", 2L, 0.3, false, 1L).row("d", 5L, 0.4, false, 2L).build();
    Page p = windowOperator.getOutput();
    while (p == null) {
        p = windowOperator.getOutput();
    }
    outputPages.add(p);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), outputPages.build());
    Assert.assertEquals(actual, expected);
}
Also used : Path(java.nio.file.Path) WindowOperatorFactory(io.prestosql.operator.WindowOperator.WindowOperatorFactory) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) ImmutableList(com.google.common.collect.ImmutableList) SortOrder(io.prestosql.spi.block.SortOrder) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) SnapshotConfig(io.prestosql.snapshot.SnapshotConfig) HetuHdfsFileSystemClient(io.hetu.core.filesystem.HetuHdfsFileSystemClient) SnapshotUtils(io.prestosql.snapshot.SnapshotUtils) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) Test(org.testng.annotations.Test)

Example 3 with HetuHdfsFileSystemClient

use of io.hetu.core.filesystem.HetuHdfsFileSystemClient in project hetu-core by openlookeng.

the class TestFileSingleStreamSpiller method getLocalHdfs.

private HetuHdfsFileSystemClient getLocalHdfs() throws IOException {
    Properties properties = new Properties();
    properties.setProperty("fs.client.type", "hdfs");
    properties.setProperty("hdfs.config.resources", "");
    properties.setProperty("hdfs.authentication.type", "NONE");
    return new HetuHdfsFileSystemClient(new HdfsConfig(properties), Paths.get(rootPath));
}
Also used : HetuHdfsFileSystemClient(io.hetu.core.filesystem.HetuHdfsFileSystemClient) HdfsConfig(io.hetu.core.filesystem.HdfsConfig) Properties(java.util.Properties)

Example 4 with HetuHdfsFileSystemClient

use of io.hetu.core.filesystem.HetuHdfsFileSystemClient in project hetu-core by openlookeng.

the class TestFileSingleStreamSpillerFactory method getLocalHdfs.

private HetuHdfsFileSystemClient getLocalHdfs(String rootPath) throws IOException {
    Properties properties = new Properties();
    properties.setProperty("fs.client.type", "hdfs");
    properties.setProperty("hdfs.config.resources", "");
    properties.setProperty("hdfs.authentication.type", "NONE");
    return new HetuHdfsFileSystemClient(new HdfsConfig(properties), Paths.get(rootPath));
}
Also used : HetuHdfsFileSystemClient(io.hetu.core.filesystem.HetuHdfsFileSystemClient) HdfsConfig(io.hetu.core.filesystem.HdfsConfig) Properties(java.util.Properties)

Example 5 with HetuHdfsFileSystemClient

use of io.hetu.core.filesystem.HetuHdfsFileSystemClient in project hetu-core by openlookeng.

the class TestOrderByOperator method getLocalHdfs.

private HetuHdfsFileSystemClient getLocalHdfs() throws IOException {
    Properties properties = new Properties();
    properties.setProperty("fs.client.type", "hdfs");
    properties.setProperty("hdfs.config.resources", "");
    properties.setProperty("hdfs.authentication.type", "NONE");
    return new HetuHdfsFileSystemClient(new HdfsConfig(properties), Paths.get("/tmp/hetu/snapshot/"));
}
Also used : HetuHdfsFileSystemClient(io.hetu.core.filesystem.HetuHdfsFileSystemClient) HdfsConfig(io.hetu.core.filesystem.HdfsConfig) Properties(java.util.Properties)

Aggregations

HetuHdfsFileSystemClient (io.hetu.core.filesystem.HetuHdfsFileSystemClient)6 HdfsConfig (io.hetu.core.filesystem.HdfsConfig)4 Properties (java.util.Properties)4 ImmutableList (com.google.common.collect.ImmutableList)2 InMemoryNodeManager (io.prestosql.metadata.InMemoryNodeManager)2 OperatorAssertion.toMaterializedResult (io.prestosql.operator.OperatorAssertion.toMaterializedResult)2 SnapshotConfig (io.prestosql.snapshot.SnapshotConfig)2 SnapshotUtils (io.prestosql.snapshot.SnapshotUtils)2 Page (io.prestosql.spi.Page)2 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)2 MarkerPage (io.prestosql.spi.snapshot.MarkerPage)2 GenericSpillerFactory (io.prestosql.spiller.GenericSpillerFactory)2 OrderingCompiler (io.prestosql.sql.gen.OrderingCompiler)2 MaterializedResult (io.prestosql.testing.MaterializedResult)2 Path (java.nio.file.Path)2 Test (org.testng.annotations.Test)2 OrderByOperatorFactory (io.prestosql.operator.OrderByOperator.OrderByOperatorFactory)1 WindowOperatorFactory (io.prestosql.operator.WindowOperator.WindowOperatorFactory)1 SortOrder (io.prestosql.spi.block.SortOrder)1