Examples with GeneratedReference io.crate.metadata.GeneratedReference used on opensource projects

Example 11 with GeneratedReference

use of io.crate.metadata.GeneratedReference in project crate by crate.

the class CopyStatementPlanner method planCopyFrom.

public Plan planCopyFrom(CopyFromAnalyzedStatement analysis, Planner.Context context) {
    /**
         * copy from has two "modes":
         *
         * 1: non-partitioned tables or partitioned tables with partition ident --> import into single es index
         *    -> collect raw source and import as is
         *
         * 2: partitioned table without partition ident
         *    -> collect document and partition by values
         *    -> exclude partitioned by columns from document
         *    -> insert into es index (partition determined by partition by value)
         */
    DocTableInfo table = analysis.table();
    int clusteredByPrimaryKeyIdx = table.primaryKey().indexOf(analysis.table().clusteredBy());
    List<String> partitionedByNames;
    String partitionIdent = null;
    List<BytesRef> partitionValues;
    if (analysis.partitionIdent() == null) {
        if (table.isPartitioned()) {
            partitionedByNames = Lists.newArrayList(Lists.transform(table.partitionedBy(), ColumnIdent::fqn));
        } else {
            partitionedByNames = Collections.emptyList();
        }
        partitionValues = ImmutableList.of();
    } else {
        assert table.isPartitioned() : "table must be partitioned if partitionIdent is set";
        // partitionIdent is present -> possible to index raw source into concrete es index
        partitionValues = PartitionName.decodeIdent(analysis.partitionIdent());
        partitionIdent = analysis.partitionIdent();
        partitionedByNames = Collections.emptyList();
    }
    SourceIndexWriterProjection sourceIndexWriterProjection = new SourceIndexWriterProjection(table.ident(), partitionIdent, table.getReference(DocSysColumns.RAW), table.primaryKey(), table.partitionedBy(), partitionValues, table.clusteredBy(), clusteredByPrimaryKeyIdx, analysis.settings(), null, partitionedByNames.size() > 0 ? partitionedByNames.toArray(new String[partitionedByNames.size()]) : null, // autoCreateIndices
    table.isPartitioned());
    List<Projection> projections = Collections.<Projection>singletonList(sourceIndexWriterProjection);
    partitionedByNames.removeAll(Lists.transform(table.primaryKey(), ColumnIdent::fqn));
    int referencesSize = table.primaryKey().size() + partitionedByNames.size() + 1;
    referencesSize = clusteredByPrimaryKeyIdx == -1 ? referencesSize + 1 : referencesSize;
    List<Symbol> toCollect = new ArrayList<>(referencesSize);
    // add primaryKey columns
    for (ColumnIdent primaryKey : table.primaryKey()) {
        toCollect.add(table.getReference(primaryKey));
    }
    // add partitioned columns (if not part of primaryKey)
    Set<Reference> referencedReferences = new HashSet<>();
    for (String partitionedColumn : partitionedByNames) {
        Reference reference = table.getReference(ColumnIdent.fromPath(partitionedColumn));
        Symbol symbol;
        if (reference instanceof GeneratedReference) {
            symbol = ((GeneratedReference) reference).generatedExpression();
            referencedReferences.addAll(((GeneratedReference) reference).referencedReferences());
        } else {
            symbol = reference;
        }
        toCollect.add(symbol);
    }
    // add clusteredBy column (if not part of primaryKey)
    if (clusteredByPrimaryKeyIdx == -1 && table.clusteredBy() != null && !DocSysColumns.ID.equals(table.clusteredBy())) {
        toCollect.add(table.getReference(table.clusteredBy()));
    }
    // add _raw or _doc
    if (table.isPartitioned() && analysis.partitionIdent() == null) {
        toCollect.add(table.getReference(DocSysColumns.DOC));
    } else {
        toCollect.add(table.getReference(DocSysColumns.RAW));
    }
    // add columns referenced by generated columns which are used as partitioned by column
    for (Reference reference : referencedReferences) {
        if (!toCollect.contains(reference)) {
            toCollect.add(reference);
        }
    }
    DiscoveryNodes allNodes = clusterService.state().nodes();
    FileUriCollectPhase collectPhase = new FileUriCollectPhase(context.jobId(), context.nextExecutionPhaseId(), "copyFrom", getExecutionNodes(allNodes, analysis.settings().getAsInt("num_readers", allNodes.getSize()), analysis.nodePredicate()), analysis.uri(), toCollect, projections, analysis.settings().get("compression", null), analysis.settings().getAsBoolean("shared", null));
    Collect collect = new Collect(collectPhase, TopN.NO_LIMIT, 0, 1, 1, null);
    return Merge.ensureOnHandler(collect, context, Collections.singletonList(MergeCountProjection.INSTANCE));
}

Example 12 with GeneratedReference

use of io.crate.metadata.GeneratedReference in project crate by crate.

the class TransportShardUpsertAction method getNotUsedNonGeneratedColumns.

public static Collection<ColumnIdent> getNotUsedNonGeneratedColumns(Reference[] targetColumns, DocTableInfo tableInfo) {
    Set<String> targetColumnsSet = new HashSet<>();
    Collection<ColumnIdent> columnsNotUsed = new ArrayList<>();
    if (targetColumns != null) {
        for (Reference targetColumn : targetColumns) {
            targetColumnsSet.add(targetColumn.column().fqn());
        }
    }
    for (Reference reference : tableInfo.columns()) {
        if (!reference.isNullable() && !(reference instanceof GeneratedReference || reference.defaultExpression() != null)) {
            if (!targetColumnsSet.contains(reference.column().fqn())) {
                columnsNotUsed.add(reference.column());
            }
        }
    }
    return columnsNotUsed;
}

Example 13 with GeneratedReference

use of io.crate.metadata.GeneratedReference in project crate by crate.

the class UserDefinedFunctionService method validateFunctionIsNotInUseByGeneratedColumn.

void validateFunctionIsNotInUseByGeneratedColumn(String schema, String functionName, UserDefinedFunctionsMetadata functionsMetadata, ClusterState currentState) {
    // The iteration of schemas/tables must happen on the node context WITHOUT the UDF already removed.
    // Otherwise the lazy table factories will already fail while evaluating generated functionsMetadata.
    // To avoid that, a copy of the node context with the removed UDF function is used on concrete expression evaluation.
    var nodeCtxWithRemovedFunction = new NodeContext(nodeCtx.functions().copyOf());
    updateImplementations(schema, functionsMetadata.functionsMetadata().stream(), nodeCtxWithRemovedFunction);
    var metadata = currentState.metadata();
    var indices = Stream.of(metadata.getConcreteAllIndices()).filter(NO_BLOB_NOR_DANGLING).map(IndexParts::new).filter(indexParts -> !indexParts.isPartitioned()).collect(Collectors.toList());
    var templates = metadata.getTemplates().keysIt();
    while (templates.hasNext()) {
        var indexParts = new IndexParts(templates.next());
        if (indexParts.isPartitioned()) {
            indices.add(indexParts);
        }
    }
    var indexNameExpressionResolver = new IndexNameExpressionResolver();
    for (var indexParts : indices) {
        var tableInfo = new DocTableInfoBuilder(nodeCtx, indexParts.toRelationName(), currentState, indexNameExpressionResolver).build();
        TableReferenceResolver tableReferenceResolver = new TableReferenceResolver(tableInfo.columns(), tableInfo.ident());
        CoordinatorTxnCtx coordinatorTxnCtx = CoordinatorTxnCtx.systemTransactionContext();
        ExpressionAnalyzer exprAnalyzer = new ExpressionAnalyzer(coordinatorTxnCtx, nodeCtxWithRemovedFunction, ParamTypeHints.EMPTY, tableReferenceResolver, null);
        for (var ref : tableInfo.columns()) {
            if (ref instanceof GeneratedReference) {
                var genRef = (GeneratedReference) ref;
                Expression expression = SqlParser.createExpression(genRef.formattedGeneratedExpression());
                try {
                    exprAnalyzer.convert(expression, new ExpressionAnalysisContext(coordinatorTxnCtx.sessionContext()));
                } catch (UnsupportedOperationException e) {
                    throw new IllegalArgumentException("Cannot drop function '" + functionName + "', it is still in use by '" + tableInfo + "." + genRef + "'");
                }
            }
        }
    }
}

Example 14 with GeneratedReference

use of io.crate.metadata.GeneratedReference in project crate by crate.

the class InsertAnalyzer method ensureClusteredByPresentOrNotRequired.

private static void ensureClusteredByPresentOrNotRequired(List<Reference> targetColumnRefs, DocTableInfo tableInfo) {
    ColumnIdent clusteredBy = tableInfo.clusteredBy();
    if (clusteredBy == null || clusteredBy.equals(DocSysColumns.ID)) {
        return;
    }
    Reference clusteredByRef = tableInfo.getReference(clusteredBy);
    if (clusteredByRef.defaultExpression() != null) {
        return;
    }
    // target columns are always top level columns;
    // so we can compare against clusteredBy-root column in case clustered by is nested
    // In insert-from-query cases we cannot peek into object values to ensure the value is present
    // and need to rely on later runtime failures
    ColumnIdent clusteredByRoot = clusteredBy.getRoot();
    List<ColumnIdent> targetColumns = Lists2.mapLazy(targetColumnRefs, Reference::column);
    if (targetColumns.contains(clusteredByRoot)) {
        return;
    }
    if (clusteredByRef instanceof GeneratedReference) {
        GeneratedReference generatedClusteredBy = (GeneratedReference) clusteredByRef;
        var topLevelDependencies = Lists2.mapLazy(generatedClusteredBy.referencedReferences(), x -> x.column().getRoot());
        if (targetColumns.containsAll(topLevelDependencies)) {
            return;
        }
    }
    throw new IllegalArgumentException("Column `" + clusteredBy + "` is required but is missing from the insert statement");
}

Example 15 with GeneratedReference

use of io.crate.metadata.GeneratedReference in project crate by crate.

the class DocIndexMetadataTest method testArrayAsGeneratedColumn.

@Test
public void testArrayAsGeneratedColumn() throws Exception {
    DocIndexMetadata md = getDocIndexMetadataFromStatement("create table t1 (x as ([10, 20]))");
    GeneratedReference generatedReference = md.generatedColumnReferences().get(0);
    assertThat(generatedReference.valueType(), is(new ArrayType<>(DataTypes.INTEGER)));
}