Examples with VARCHAR com.facebook.presto.common.type.VarcharType.VARCHAR used on opensource projects

Example 6 with VARCHAR

use of com.facebook.presto.common.type.VarcharType.VARCHAR in project presto by prestodb.

the class TestSpatialJoinOperator method testYield.

@Test
public void testYield() {
    // create a filter function that yields for every probe match
    // verify we will yield #match times totally
    TaskContext taskContext = createTaskContext();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // force a yield for every match
    AtomicInteger filterFunctionCalls = new AtomicInteger();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> {
        filterFunctionCalls.incrementAndGet();
        driverContext.getYieldSignal().forceYieldForTesting();
        return true;
    }));
    RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)).row(POLYGON_A, "A").pageBreak().row(POLYGON_B, "B");
    PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.of(filterFunction), buildPages);
    // 10 points in polygon A (x0...x9)
    // 10 points in polygons A and B (y0...y9)
    // 10 points in polygon B (z0...z9)
    // 40 total matches
    RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR));
    for (int i = 0; i < 10; i++) {
        probePages.row(stPoint(1 + 0.1 * i, 1 + 0.1 * i), "x" + i);
    }
    for (int i = 0; i < 10; i++) {
        probePages.row(stPoint(4.5 + 0.01 * i, 4.5 + 0.01 * i), "y" + i);
    }
    for (int i = 0; i < 10; i++) {
        probePages.row(stPoint(6 + 0.1 * i, 6 + 0.1 * i), "z" + i);
    }
    List<Page> probeInput = probePages.build();
    OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
    Operator operator = joinOperatorFactory.createOperator(driverContext);
    assertTrue(operator.needsInput());
    operator.addInput(probeInput.get(0));
    operator.finish();
    // we will yield 40 times due to filterFunction
    for (int i = 0; i < 40; i++) {
        driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
        assertNull(operator.getOutput());
        assertEquals(filterFunctionCalls.get(), i + 1, "Expected join to stop processing (yield) after calling filter function once");
        driverContext.getYieldSignal().reset();
    }
    // delayed yield is not going to prevent operator from producing a page now (yield won't be forced because filter function won't be called anymore)
    driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
    Page output = operator.getOutput();
    assertNotNull(output);
    // make sure we have 40 matches
    assertEquals(output.getPositionCount(), 40);
}

Example 7 with VARCHAR

use of com.facebook.presto.common.type.VarcharType.VARCHAR in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testPageSourceLazyLoad.

@Test
public void testPageSourceLazyLoad() {
    Block inputBlock = BlockAssertions.createLongSequenceBlock(0, 100);
    // If column 1 is loaded, test will fail
    Page input = new Page(100, inputBlock, new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    DriverContext driverContext = newDriverContext();
    List<RowExpression> projections = ImmutableList.of(field(0, VARCHAR));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key");
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new PageProjectionWithOutputs(new LazyPagePageProjection(), new int[] { 0 })));
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> new SinglePagePageSource(input), cursorProcessor, () -> pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), ImmutableList.of(new Page(inputBlock)));
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), toPages(operator));
    assertEquals(actual.getRowCount(), expected.getRowCount());
    assertEquals(actual, expected);
}

Example 8 with VARCHAR

use of com.facebook.presto.common.type.VarcharType.VARCHAR in project presto by prestodb.

the class HiveMetadata method getPropertiesSystemTable.

private Optional<SystemTable> getPropertiesSystemTable(ConnectorSession session, SchemaTableName tableName, SchemaTableName sourceTableName) {
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Optional<Table> table = metastore.getTable(metastoreContext, sourceTableName.getSchemaName(), sourceTableName.getTableName());
    if (!table.isPresent() || table.get().getTableType().equals(VIRTUAL_VIEW)) {
        throw new TableNotFoundException(tableName);
    }
    Map<String, String> sortedTableParameters = ImmutableSortedMap.copyOf(table.get().getParameters());
    List<ColumnMetadata> columns = sortedTableParameters.keySet().stream().map(key -> new ColumnMetadata(key, VARCHAR)).collect(toImmutableList());
    List<Type> types = columns.stream().map(ColumnMetadata::getType).collect(toImmutableList());
    Iterable<List<Object>> propertyValues = ImmutableList.of(ImmutableList.copyOf(sortedTableParameters.values()));
    return Optional.of(createSystemTable(new ConnectorTableMetadata(sourceTableName, columns), constraint -> new InMemoryRecordSet(types, propertyValues).cursor()));
}

Example 9 with VARCHAR

use of com.facebook.presto.common.type.VarcharType.VARCHAR in project presto by prestodb.

the class TestSelectiveOrcReader method testVarchars.

@Test
public void testVarchars() throws Exception {
    Random random = new Random(0);
    tester.testRoundTripTypes(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR), ImmutableList.of(newArrayList("abc", "def", null, "hij", "klm"), newArrayList(null, null, null, null, null), newArrayList("abc", "def", null, null, null)), toSubfieldFilters(ImmutableMap.of(0, stringIn(true, "abc", "def"), 1, stringIn(true, "10", "11"), 2, stringIn(true, "def", "abc"))));
    // direct and dictionary
    tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")), NUM_ROWS)), stringIn(false, "apple", "apple pie"));
    // direct and dictionary materialized
    tester.testRoundTrip(VARCHAR, intsBetween(0, NUM_ROWS).stream().map(Object::toString).collect(toList()), stringIn(false, "10", "11"), stringIn(true, "10", "11"), stringBetween(false, "14", "10"));
    // direct & dictionary with filters
    tester.testRoundTripTypes(ImmutableList.of(VARCHAR, VARCHAR), ImmutableList.of(intsBetween(0, NUM_ROWS).stream().map(Object::toString).collect(toList()), newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS))), toSubfieldFilters(ImmutableMap.of(0, stringBetween(true, "16", "10"), 1, stringBetween(false, "B", "A"))));
    // stripe dictionary
    tester.testRoundTrip(VARCHAR, newArrayList(concat(ImmutableList.of("a"), nCopies(9999, "123"), ImmutableList.of("b"), nCopies(9999, "123"))));
    // empty sequence
    tester.testRoundTrip(VARCHAR, nCopies(NUM_ROWS, ""), stringEquals(false, ""));
    // copy of AbstractOrcTester::testDwrfInvalidCheckpointsForRowGroupDictionary
    List<Integer> values = newArrayList(limit(cycle(concat(ImmutableList.of(1), nCopies(9999, 123), ImmutableList.of(2), nCopies(9999, 123), ImmutableList.of(3), nCopies(9999, 123), nCopies(1_000_000, null))), 200_000));
    tester.assertRoundTrip(VARCHAR, newArrayList(values).stream().map(value -> value == null ? null : String.valueOf(value)).collect(toList()));
    // copy of AbstractOrcTester::testDwrfInvalidCheckpointsForStripeDictionary
    tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 200_000)).stream().map(Object::toString).collect(toList()));
    // presentStream is null in some row groups & dictionary materialized
    Function<Integer, String> randomStrings = i -> String.valueOf(random.nextInt(NUM_ROWS));
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, VARCHAR), ImmutableList.of(createList(NUM_ROWS, i -> random.nextInt(NUM_ROWS)), newArrayList(createList(NUM_ROWS, randomStrings))), toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(10_000, 15_000, true))));
    // dataStream is null and lengths are 0
    tester.testRoundTrip(VARCHAR, newArrayList("", null), toSubfieldFilters(stringNotEquals(true, "")));
    tester.testRoundTrip(VARCHAR, newArrayList("", ""), toSubfieldFilters(stringLessThan(true, "")));
}

Example 10 with VARCHAR

use of com.facebook.presto.common.type.VarcharType.VARCHAR in project presto by prestodb.

the class TestSelectiveOrcReader method testArrays.

@Test
public void testArrays() throws Exception {
    Random random = new Random(0);
    // non-null arrays of varying sizes; some arrays may be empty
    tester.testRoundTrip(arrayType(INTEGER), createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random)), IS_NULL, IS_NOT_NULL);
    BigintRange negative = BigintRange.of(Integer.MIN_VALUE, 0, false);
    BigintRange nonNegative = BigintRange.of(0, Integer.MAX_VALUE, false);
    // arrays of strings
    tester.testRoundTrip(arrayType(VARCHAR), createList(1000, i -> randomStrings(5 + random.nextInt(5), random)), ImmutableList.of(toSubfieldFilter("c[1]", IS_NULL), toSubfieldFilter("c[1]", stringIn(true, "a", "b", "c", "d"))));
    tester.testRoundTrip(arrayType(VARCHAR), createList(10, i -> randomStringsWithNulls(5 + random.nextInt(5), random)), ImmutableList.of(toSubfieldFilter("c[1]", IS_NULL), toSubfieldFilter("c[1]", stringIn(true, "a", "b", "c", "d"))));
    // non-empty non-null arrays of varying sizes
    tester.testRoundTrip(arrayType(INTEGER), createList(NUM_ROWS, i -> randomIntegers(5 + random.nextInt(5), random)), ImmutableList.of(toSubfieldFilter(IS_NULL), toSubfieldFilter(IS_NOT_NULL), // c[1] >= 0
    toSubfieldFilter("c[1]", nonNegative), // c[2] >= 0 AND c[4] >= 0
    ImmutableMap.of(new Subfield("c[2]"), nonNegative, new Subfield("c[4]"), nonNegative)));
    // non-null arrays of varying sizes; some arrays may be empty
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(INTEGER)), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random))), toSubfieldFilters(ImmutableMap.of(0, nonNegative), ImmutableMap.of(0, nonNegative, 1, IS_NULL), ImmutableMap.of(0, nonNegative, 1, IS_NOT_NULL)));
    // non-empty non-null arrays of varying sizes
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(INTEGER)), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> randomIntegers(5 + random.nextInt(5), random))), ImmutableList.of(// c[1] >= 0
    ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, toSubfieldFilter("c[1]", nonNegative)), // c[3] >= 0
    ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, toSubfieldFilter("c[3]", nonNegative)), // c[2] >= 0 AND c[4] <= 0
    ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, ImmutableMap.of(new Subfield("c[2]"), nonNegative, new Subfield("c[4]"), negative))));
    // nested arrays
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(arrayType(INTEGER))), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> createList(random.nextInt(10), index -> randomIntegers(random.nextInt(5), random)))), toSubfieldFilters(ImmutableMap.of(0, nonNegative), ImmutableMap.of(1, IS_NULL), ImmutableMap.of(1, IS_NOT_NULL), ImmutableMap.of(0, nonNegative, 1, IS_NULL)));
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(arrayType(INTEGER))), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> createList(3 + random.nextInt(10), index -> randomIntegers(3 + random.nextInt(5), random)))), ImmutableList.of(// c[1] IS NULL
    ImmutableMap.of(1, ImmutableMap.of(new Subfield("c[1]"), IS_NULL)), // c[2] IS NOT NULL AND c[2][3] >= 0
    ImmutableMap.of(1, ImmutableMap.of(new Subfield("c[2]"), IS_NOT_NULL, new Subfield("c[2][3]"), nonNegative)), ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, ImmutableMap.of(new Subfield("c[1]"), IS_NULL))));
}