Examples with Symbol io.prestosql.spi.plan.Symbol used on opensource projects

Example 71 with Symbol

use of io.prestosql.spi.plan.Symbol in project hetu-core by openlookeng.

the class TestDynamicFilterServiceWithBloomFilter method testRegisterAndMergeDynamicFilters.

@Test
public void testRegisterAndMergeDynamicFilters() throws InterruptedException {
    registerDf(filterId, session, PARTITIONED, dynamicFilterService);
    // Test getDynamicFilterSupplier
    VariableReferenceExpression mockExpression = mock(VariableReferenceExpression.class);
    when(mockExpression.getName()).thenReturn("name");
    ColumnHandle mockColumnHandle = mock(ColumnHandle.class);
    Supplier<List<Set<DynamicFilter>>> dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(session.getQueryId(), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    assertTrue(dynamicFilterSupplier.get().isEmpty(), "should return empty dynamic filter set when dynamic filters are not available");
    mockLocalDynamicFilter("task1.0", filterId, session.getQueryId().toString(), Arrays.asList("1", "2", "3", "4"));
    mockLocalDynamicFilter("task1.1", filterId, session.getQueryId().toString(), Arrays.asList("5", "6", "7", "8"));
    Thread.sleep(3000);
    BloomFilter bf = fetchDynamicFilter(filterId, session.getQueryId().toString());
    for (int i = 1; i < 9; i++) {
        assertTrue(bf.test((String.valueOf(i).getBytes(StandardCharsets.UTF_8))));
    }
    assertFalse(bf.test("10".getBytes(StandardCharsets.UTF_8)));
    // Test getDynamicFilterSupplier
    dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(session.getQueryId(), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    List<Set<DynamicFilter>> dynamicFilters = dynamicFilterSupplier.get();
    assertFalse(dynamicFilters == null, "dynamic filters should be ready");
    assertEquals(dynamicFilters.size(), 1, "there should be 1 dynamic filter in supplier");
    DynamicFilter dynamicFilter = dynamicFilters.get(0).iterator().next();
    for (int i = 1; i < 9; i++) {
        assertTrue(dynamicFilter.contains(String.valueOf(i)));
    }
    assertFalse(dynamicFilter.contains("10"));
    dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(new QueryId("invalid"), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    assertTrue(dynamicFilterSupplier.get().isEmpty(), "should return empty dynamic filter set for invalid or non-existing queryId");
    String queryId = session.getQueryId().getId();
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.PARTIALPREFIX, filterId, queryId)).size(), 2);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.TASKSPREFIX, filterId, queryId)).size(), 2);
    dynamicFilterService.clearDynamicFiltersForQuery(queryId);
    Thread.sleep(1000);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.PARTIALPREFIX, filterId, queryId)).size(), 0);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.TASKSPREFIX, filterId, queryId)).size(), 0);
}

Example 72 with Symbol

use of io.prestosql.spi.plan.Symbol in project hetu-core by openlookeng.

the class TestDynamicFilterServiceWithHashSet method testRegisterAndMergeDynamicFiltersHashSet.

@Test
public void testRegisterAndMergeDynamicFiltersHashSet() throws InterruptedException {
    setUpHashSet();
    filterId = "df2";
    registerDf(filterId, session, PARTITIONED, dynamicFilterService);
    // Test getDynamicFilterSupplier
    VariableReferenceExpression mockExpression = mock(VariableReferenceExpression.class);
    when(mockExpression.getName()).thenReturn("name");
    ColumnHandle mockColumnHandle = mock(ColumnHandle.class);
    Supplier<List<Set<DynamicFilter>>> dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(session.getQueryId(), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    assertTrue(dynamicFilterSupplier.get().isEmpty(), "should return empty dynamic filter set when dynamic filters are not available");
    mockLocalDynamicFilterHashSet("task1.0", filterId, session.getQueryId().toString(), Arrays.asList("11", "12", "13", "14"));
    mockLocalDynamicFilterHashSet("task1.1", filterId, session.getQueryId().toString(), Arrays.asList("15", "16", "17", "18"));
    Thread.sleep(2000);
    Set hs = fetchDynamicFilterHashSet(filterId, session.getQueryId().toString());
    for (int i = 11; i < 19; i++) {
        Assert.assertEquals(true, hs.contains(i + ""));
    }
    Assert.assertEquals(false, hs.contains("10"));
    // Test getDynamicFilterSupplier
    dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(session.getQueryId(), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    List<Set<DynamicFilter>> dynamicFilters = dynamicFilterSupplier.get();
    assertFalse(dynamicFilters == null, "dynamic filters should be ready");
    assertEquals(dynamicFilters.size(), 1, "there should be 1 dynamic filter in supplier");
    HashSetDynamicFilter hsDF = ((HashSetDynamicFilter) dynamicFilters.get(0).toArray()[0]);
    assertEquals(hs, hsDF.getSetValues(), "dynamic filter in supplier should be the same as the one merged");
    dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(new QueryId("invalid"), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    assertTrue(dynamicFilterSupplier.get().isEmpty(), "should return empty dynamic filter set for invalid or non-existing queryId");
    String queryId = session.getQueryId().getId();
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.PARTIALPREFIX, filterId, queryId)).size(), 2);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.TASKSPREFIX, filterId, queryId)).size(), 2);
    dynamicFilterService.clearDynamicFiltersForQuery(queryId);
    Thread.sleep(1000);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.PARTIALPREFIX, filterId, queryId)).size(), 0);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.TASKSPREFIX, filterId, queryId)).size(), 0);
}

Example 73 with Symbol

use of io.prestosql.spi.plan.Symbol in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method createOperatorFactory.

private DynamicFilterSourceOperatorFactory createOperatorFactory(DynamicFilter.Type dfType, FeaturesConfig.DynamicFilterDataType dataType, int partitionCount, Channel... buildChannels) {
    NodeInfo nodeInfo = new NodeInfo("test");
    Multimap<String, Symbol> probeSymbols = MultimapBuilder.treeKeys().arrayListValues().build();
    Map<String, Integer> buildChannelMap = new HashMap<>();
    Arrays.stream(buildChannels).map(channel -> buildChannelMap.put(channel.getFilterId(), channel.getIndex()));
    Arrays.stream(buildChannels).map(channel -> probeSymbols.put(channel.getFilterId(), new Symbol(String.valueOf(channel.getIndex()))));
    TaskId taskId = new TaskId("test0.0");
    LocalDynamicFilter localDynamicFilter = new LocalDynamicFilter(probeSymbols, buildChannelMap, partitionCount, dfType, dataType, 0.1D, taskId, stateStoreProvider);
    return new DynamicFilterSourceOperatorFactory(0, new PlanNodeId("PLAN_NODE_ID"), localDynamicFilter.getValueConsumer(), Arrays.stream(buildChannels).collect(toList()), getDynamicFilteringMaxPerDriverValueCount(TEST_SESSION), getDynamicFilteringMaxPerDriverSize(TEST_SESSION));
}

Example 74 with Symbol

use of io.prestosql.spi.plan.Symbol in project hetu-core by openlookeng.

the class PlanSymbolAllocator method newSymbol.

@Override
public Symbol newSymbol(String nameHint, Type type, String suffix) {
    requireNonNull(nameHint, "name is null");
    requireNonNull(type, "type is null");
    // TODO: workaround for the fact that QualifiedName lowercases parts
    String nameHintLower = nameHint.toLowerCase(ENGLISH);
    // don't strip the tail if the only _ is the first character
    int index = nameHintLower.lastIndexOf("_");
    if (index > 0) {
        String tail = nameHintLower.substring(index + 1);
        // only strip if tail is numeric or _ is the last character
        if (Ints.tryParse(tail) != null || index == nameHintLower.length() - 1) {
            nameHintLower = nameHintLower.substring(0, index);
        }
    }
    String unique = nameHintLower;
    if (suffix != null) {
        unique = unique + "$" + suffix;
    }
    String attempt = unique;
    while (symbols.containsKey(new Symbol(attempt))) {
        attempt = unique + "_" + nextId();
    }
    Symbol symbol = new Symbol(attempt);
    symbols.put(symbol, type);
    return symbol;
}

Example 75 with Symbol

use of io.prestosql.spi.plan.Symbol in project hetu-core by openlookeng.

the class QueryPlanner method planUpdateRowAsInsert.

public UpdateDeleteRelationPlan planUpdateRowAsInsert(Update node) {
    Table table = node.getTable();
    RelationType descriptor = analysis.getOutputDescriptor(table);
    TableHandle handle = analysis.getTableHandle(table);
    ColumnHandle rowIdHandle = analysis.getRowIdHandle(table);
    ColumnMetadata rowIdColumnMetadata = metadata.getColumnMetadata(session, handle, rowIdHandle);
    // add table columns
    ImmutableList.Builder<Symbol> outputSymbols = ImmutableList.builder();
    ImmutableMap.Builder<Symbol, ColumnHandle> columnsBuilder = ImmutableMap.builder();
    ImmutableList.Builder<Field> fields = ImmutableList.builder();
    for (Field field : descriptor.getAllFields()) {
        Symbol symbol = planSymbolAllocator.newSymbol(field);
        outputSymbols.add(symbol);
        columnsBuilder.put(symbol, analysis.getColumn(field));
        fields.add(field);
    }
    // create table scan
    ImmutableMap<Symbol, ColumnHandle> columns = columnsBuilder.build();
    PlanNode tableScan = TableScanNode.newInstance(idAllocator.getNextId(), handle, outputSymbols.build(), columns, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, true);
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields.build())).build();
    RelationPlan relationPlan = new RelationPlan(tableScan, scope, outputSymbols.build());
    TranslationMap translations = new TranslationMap(relationPlan, analysis, lambdaDeclarationToSymbolMap);
    translations.setFieldMappings(relationPlan.getFieldMappings());
    PlanBuilder builder = new PlanBuilder(translations, relationPlan.getRoot());
    Optional<RowExpression> predicate = Optional.empty();
    if (node.getWhere().isPresent()) {
        builder = filter(builder, node.getWhere().get(), node);
        if (builder.getRoot() instanceof FilterNode) {
            predicate = Optional.of(((FilterNode) builder.getRoot()).getPredicate());
        }
    }
    List<AssignmentItem> assignmentItems = node.getAssignmentItems();
    Analysis.Update update = analysis.getUpdate().get();
    Assignments.Builder assignments = Assignments.builder();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, update.getTarget());
    Symbol orderBySymbol = null;
    for (Map.Entry<Symbol, ColumnHandle> entry : columns.entrySet()) {
        ColumnMetadata column;
        ColumnHandle columnHandle = entry.getValue();
        Symbol input = entry.getKey();
        if (columnHandle.getColumnName().equals(rowIdHandle.getColumnName())) {
            column = rowIdColumnMetadata;
        } else {
            column = tableMetadata.getColumn(columnHandle.getColumnName());
        }
        if (column != rowIdColumnMetadata && column.isHidden()) {
            continue;
        }
        Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        Type tableType = column.getType();
        Type queryType = planSymbolAllocator.getTypes().get(input);
        List<AssignmentItem> assignment = assignmentItems.stream().filter(item -> item.getName().equals(QualifiedName.of(column.getName()))).collect(Collectors.toList());
        if (!assignment.isEmpty()) {
            Expression expression = assignment.get(0).getValue();
            Expression cast;
            if (expression instanceof Identifier) {
                // assigning by column reference
                Optional<Symbol> first = columns.entrySet().stream().filter(e -> e.getValue().getColumnName().equals(((Identifier) expression).getValue())).map(Entry::getKey).findFirst();
                Symbol source = (first.orElseThrow(() -> new IllegalArgumentException("Unable to find column " + ((Identifier) expression).getValue())));
                cast = new Cast(toSymbolReference(source), tableType.getTypeSignature().toString());
            } else {
                cast = new Cast(expression, tableType.getTypeSignature().toString());
            }
            assignments.put(output, castToRowExpression(cast));
        } else if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
            assignments.put(output, castToRowExpression(toSymbolReference(input)));
        } else {
            Expression cast = new Cast(toSymbolReference(input), tableType.getTypeSignature().toString());
            assignments.put(output, castToRowExpression(cast));
        }
        if (column == rowIdColumnMetadata) {
            orderBySymbol = output;
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), builder.getRoot(), assignments.build());
    PlanBuilder planBuilder = new PlanBuilder(translations, projectNode);
    SortOrder sortOrder = SortOrder.ASC_NULLS_LAST;
    Symbol sortSymbol = orderBySymbol;
    Map<Symbol, SortOrder> sortOrderMap = ImmutableMap.<Symbol, SortOrder>builder().put(sortSymbol, sortOrder).build();
    OrderingScheme orderingScheme = new OrderingScheme(ImmutableList.of(sortSymbol), sortOrderMap);
    builder = sort(planBuilder, Optional.of(orderingScheme));
    ImmutableList.Builder<Field> projectFields = ImmutableList.builder();
    projectFields.addAll(fields.build().stream().filter(x -> !x.isHidden()).collect(toImmutableList()));
    scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(projectFields.build())).build();
    RelationPlan plan = new RelationPlan(builder.getRoot(), scope, projectNode.getOutputSymbols());
    List<String> visibleTableColumnNames = tableMetadata.getColumns().stream().filter(c -> !c.isHidden()).map(ColumnMetadata::getName).collect(Collectors.toList());
    visibleTableColumnNames.add(rowIdColumnMetadata.getName());
    return new UpdateDeleteRelationPlan(plan, visibleTableColumnNames, columns, predicate);
}