Examples with SortOrder io.prestosql.spi.block.SortOrder used on opensource projects

Example 21 with SortOrder

use of io.prestosql.spi.block.SortOrder in project hetu-core by openlookeng.

the class TestWindowOperator method testMultipleOutputPages.

@Test(dataProvider = "spillEnabled")
public void testMultipleOutputPages(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    // make operator produce multiple pages during finish phase
    int numberOfRows = 80_000;
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).addSequencePage(numberOfRows, 0, 0).build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT, DOUBLE), Ints.asList(1, 0), ROW_NUMBER, Ints.asList(), Ints.asList(0), ImmutableList.copyOf(new SortOrder[] { SortOrder.DESC_NULLS_FIRST }), spillEnabled);
    DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
    MaterializedResult.Builder expectedBuilder = resultBuilder(driverContext.getSession(), DOUBLE, BIGINT, BIGINT);
    for (int i = 0; i < numberOfRows; ++i) {
        expectedBuilder.row((double) numberOfRows - i - 1, (long) numberOfRows - i - 1, (long) i + 1);
    }
    MaterializedResult expected = expectedBuilder.build();
    List<Page> pages = toPages(operatorFactory, driverContext, input, revokeMemoryWhenAddingPages);
    assertGreaterThan(pages.size(), 1, "Expected more than one output page");
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), pages);
    assertEquals(actual.getMaterializedRows(), expected.getMaterializedRows());
    assertTrue(spillEnabled == (spillerFactory.getSpillsCount() > 0), format("Spill state mismatch. Expected spill: %s, spill count: %s", spillEnabled, spillerFactory.getSpillsCount()));
}

Example 22 with SortOrder

use of io.prestosql.spi.block.SortOrder in project boostkit-bigdata by kunpengcompute.

the class OrcDeletedRows method isDeleted.

private boolean isDeleted(OrcAcidRowId sourcePageRowId) {
    if (sortedRowsIterator == null) {
        for (WriteIdInfo deleteDeltaInfo : deleteDeltaLocations.getDeleteDeltas()) {
            Path path = createPath(deleteDeltaLocations.getPartitionLocation(), deleteDeltaInfo, sourceFileName);
            try {
                FileSystem fileSystem = hdfsEnvironment.getFileSystem(sessionUser, path, configuration);
                FileStatus fileStatus = hdfsEnvironment.doAs(sessionUser, () -> fileSystem.getFileStatus(path));
                pageSources.add(pageSourceFactory.createPageSource(fileStatus.getPath(), fileStatus.getLen(), fileStatus.getModificationTime()));
            } catch (FileNotFoundException ignored) {
                // source file does not have a delta delete file in this location
                continue;
            } catch (PrestoException e) {
                throw e;
            } catch (OrcCorruptionException e) {
                throw new PrestoException(HiveErrorCode.HIVE_BAD_DATA, format("Failed to read ORC file: %s", path), e);
            } catch (RuntimeException | IOException e) {
                throw new PrestoException(HiveErrorCode.HIVE_CURSOR_ERROR, format("Failed to read ORC file: %s", path), e);
            }
        }
        List<Type> columnTypes = ImmutableList.of(BigintType.BIGINT, IntegerType.INTEGER, BigintType.BIGINT);
        // Last index for rowIdHandle
        List<Integer> sortFields = ImmutableList.of(0, 1, 2);
        List<SortOrder> sortOrders = ImmutableList.of(SortOrder.ASC_NULLS_FIRST, SortOrder.ASC_NULLS_FIRST, SortOrder.ASC_NULLS_FIRST);
        sortedRowsIterator = HiveUtil.getMergeSortedPages(pageSources, columnTypes, sortFields, sortOrders);
    }
    do {
        if (currentPage == null || currentPageOffset >= currentPage.getPositionCount()) {
            currentPage = null;
            currentPageOffset = 0;
            if (sortedRowsIterator.hasNext()) {
                currentPage = sortedRowsIterator.next();
            } else {
                // No more entries in deleted_delta
                return false;
            }
        }
        do {
            deletedRowId.set(currentPage, currentPageOffset);
            if (deletedRowId.compareTo(sourcePageRowId) == 0) {
                // source row is deleted.
                return true;
            } else if (deletedRowId.compareTo(sourcePageRowId) > 0) {
                // So current source row is not deleted.
                return false;
            }
            currentPageOffset++;
        } while (currentPageOffset < currentPage.getPositionCount());
    } while (sortedRowsIterator.hasNext());
    // No more entries;
    return false;
}

Example 23 with SortOrder

use of io.prestosql.spi.block.SortOrder in project boostkit-bigdata by kunpengcompute.

the class TestMergingPageIterator method testMerging.

@Test
public void testMerging() {
    List<Type> types = ImmutableList.of(INTEGER, INTEGER);
    List<Integer> sortIndexes = ImmutableList.of(1);
    List<SortOrder> sortOrders = ImmutableList.of(SortOrder.ASC_NULLS_FIRST);
    List<List<Page>> pageLists = new ArrayList<>();
    PageBuilder pageBuilder = new PageBuilder(types);
    for (int i = 0; i < 10; i++) {
        Iterator<Integer> values = IntStream.range(0, 1000).map(ignored -> ThreadLocalRandom.current().nextInt(100_000)).mapToObj(n -> ((n % 100) == 0) ? null : n).sorted(nullsFirst(naturalOrder())).iterator();
        List<Page> pages = new ArrayList<>();
        for (int j = 0; j < 10; j++) {
            for (int k = 0; k < 100; k++) {
                Integer n = values.next();
                pageBuilder.declarePosition();
                if (n == null) {
                    pageBuilder.getBlockBuilder(0).appendNull();
                    pageBuilder.getBlockBuilder(1).appendNull();
                } else {
                    INTEGER.writeLong(pageBuilder.getBlockBuilder(0), n);
                    INTEGER.writeLong(pageBuilder.getBlockBuilder(1), n * 22L);
                }
            }
            pages.add(pageBuilder.build());
            pageBuilder.reset();
        }
        pageLists.add(pages);
        assertFalse(values.hasNext());
    }
    List<Iterator<Page>> pages = pageLists.stream().map(List::iterator).collect(toList());
    Iterator<Page> iterator = new MergingPageIterator(pages, types, sortIndexes, sortOrders);
    List<Long> values = new ArrayList<>();
    while (iterator.hasNext()) {
        Page page = iterator.next();
        for (int i = 0; i < page.getPositionCount(); i++) {
            if (page.getBlock(0).isNull(i)) {
                assertTrue(page.getBlock(1).isNull(i));
                values.add(null);
            } else {
                long x = INTEGER.getLong(page.getBlock(0), i);
                long y = INTEGER.getLong(page.getBlock(1), i);
                assertEquals(y, x * 22);
                values.add(x);
            }
        }
    }
    assertThat(values).isSortedAccordingTo(nullsFirst(naturalOrder()));
}

Example 24 with SortOrder

use of io.prestosql.spi.block.SortOrder in project hetu-core by openlookeng.

the class TestWindowOperator method testMultipleOutputPagesSnapshot.

@Test(dataProvider = "spillEnabledSnapshot")
public void testMultipleOutputPagesSnapshot(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    // make operator produce multiple pages during finish phase
    int numberOfRows = 80_000;
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).addSequencePage(numberOfRows, 0, 0).build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT, DOUBLE), Ints.asList(1, 0), ROW_NUMBER, Ints.asList(), Ints.asList(0), ImmutableList.copyOf(new SortOrder[] { SortOrder.DESC_NULLS_FIRST }), spillEnabled);
    DriverContext driverContext = createDriverContext(memoryLimit);
    MaterializedResult.Builder expectedBuilder = resultBuilder(driverContext.getSession(), DOUBLE, BIGINT, BIGINT);
    for (int i = 0; i < numberOfRows; ++i) {
        expectedBuilder.row((double) numberOfRows - i - 1, (long) numberOfRows - i - 1, (long) i + 1);
    }
    MaterializedResult expected = expectedBuilder.build();
    List<Page> pages = toPagesWithRestoreToNewOperator(operatorFactory, driverContext, () -> createDriverContext(memoryLimit), input, revokeMemoryWhenAddingPages);
    assertGreaterThan(pages.size(), 1, "Expected more than one output page");
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), pages);
    assertEquals(actual.getMaterializedRows(), expected.getMaterializedRows());
    assertTrue(false == (spillerFactory.getSpillsCount() > 0), format("Spill state mismatch. Expected spill: %s, spill count: %s", false, spillerFactory.getSpillsCount()));
}

Example 25 with SortOrder

use of io.prestosql.spi.block.SortOrder in project hetu-core by openlookeng.

the class TestWindowOperator method testRowNumberArbitrary.

@Test
public void testRowNumberArbitrary() {
    List<Page> input = rowPagesBuilder(BIGINT).row(1L).row(3L).row(5L).row(7L).pageBreak().row(2L).row(4L).row(6L).row(8L).build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT), Ints.asList(0), ROW_NUMBER, Ints.asList(), Ints.asList(), ImmutableList.copyOf(new SortOrder[] {}), false);
    DriverContext driverContext = createDriverContext();
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, BIGINT).row(1L, 1L).row(3L, 2L).row(5L, 3L).row(7L, 4L).row(2L, 5L).row(4L, 6L).row(6L, 7L).row(8L, 8L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected);
}