use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.
the class KsqlResource method describe.
private SourceDescription describe(String name, boolean extended) throws KsqlException {
StructuredDataSource dataSource = ksqlEngine.getMetaStore().getSource(name);
if (dataSource == null) {
throw new KsqlException(String.format("Could not find STREAM/TABLE '%s' in the Metastore", name));
}
List<PersistentQueryMetadata> queries = ksqlEngine.getPersistentQueries().values().stream().filter(meta -> ((KsqlStructuredDataOutputNode) meta.getOutputNode()).getKafkaTopicName().equals(dataSource.getKsqlTopic().getTopicName())).collect(Collectors.toList());
return new SourceDescription(dataSource, extended, dataSource.getKsqlTopic().getKsqlTopicSerDe().getSerDe().name(), "", "", getReadQueryIds(queries), getWriteQueryIds(queries), ksqlEngine.getTopicClient());
}
use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.
the class AstBuilder method getResultDatasource.
private StructuredDataSource getResultDatasource(Select select, Table into) {
SchemaBuilder dataSource = SchemaBuilder.struct().name(into.toString());
for (SelectItem selectItem : select.getSelectItems()) {
if (selectItem instanceof SingleColumn) {
SingleColumn singleColumn = (SingleColumn) selectItem;
String fieldName = singleColumn.getAlias().get();
dataSource = dataSource.field(fieldName, Schema.BOOLEAN_SCHEMA);
}
}
KsqlTopic ksqlTopic = new KsqlTopic(into.getName().toString(), into.getName().toString(), null);
StructuredDataSource resultStream = new KsqlStream("AstBuilder-Into", into.getName().toString(), dataSource.schema(), dataSource.fields().get(0), null, ksqlTopic);
return resultStream;
}
use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.
the class AstBuilder method getSelectStartItems.
private List<SelectItem> getSelectStartItems(final SelectItem selectItem, final Relation from) {
List<SelectItem> selectItems = new ArrayList<>();
AllColumns allColumns = (AllColumns) selectItem;
if (from instanceof Join) {
Join join = (Join) from;
AliasedRelation left = (AliasedRelation) join.getLeft();
StructuredDataSource leftDataSource = dataSourceExtractor.getMetaStore().getSource(left.getRelation().toString());
if (leftDataSource == null) {
throw new InvalidColumnReferenceException(left.getRelation().toString() + " does not exist.");
}
AliasedRelation right = (AliasedRelation) join.getRight();
StructuredDataSource rightDataSource = dataSourceExtractor.getMetaStore().getSource(right.getRelation().toString());
if (rightDataSource == null) {
throw new InvalidColumnReferenceException(right.getRelation().toString() + " does not exist.");
}
for (Field field : leftDataSource.getSchema().fields()) {
QualifiedNameReference qualifiedNameReference = new QualifiedNameReference(allColumns.getLocation().get(), QualifiedName.of(left.getAlias() + "." + field.name()));
SingleColumn newSelectItem = new SingleColumn(qualifiedNameReference, left.getAlias() + "_" + field.name());
selectItems.add(newSelectItem);
}
for (Field field : rightDataSource.getSchema().fields()) {
QualifiedNameReference qualifiedNameReference = new QualifiedNameReference(allColumns.getLocation().get(), QualifiedName.of(right.getAlias() + "." + field.name()));
SingleColumn newSelectItem = new SingleColumn(qualifiedNameReference, right.getAlias() + "_" + field.name());
selectItems.add(newSelectItem);
}
} else {
AliasedRelation fromRel = (AliasedRelation) from;
StructuredDataSource fromDataSource = dataSourceExtractor.getMetaStore().getSource(((Table) fromRel.getRelation()).getName().getSuffix());
if (fromDataSource == null) {
throw new InvalidColumnReferenceException(((Table) fromRel.getRelation()).getName().getSuffix() + " does not exist.");
}
for (Field field : fromDataSource.getSchema().fields()) {
QualifiedNameReference qualifiedNameReference = new QualifiedNameReference(allColumns.getLocation().get(), QualifiedName.of(fromDataSource.getName() + "." + field.name()));
SingleColumn newSelectItem = new SingleColumn(qualifiedNameReference, field.name());
selectItems.add(newSelectItem);
}
}
return selectItems;
}
use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.
the class DataSourceExtractor method visitAliasedRelation.
@Override
public Node visitAliasedRelation(final SqlBaseParser.AliasedRelationContext context) {
Table table = (Table) visit(context.relationPrimary());
String alias = null;
if (context.children.size() == 1) {
alias = table.getName().getSuffix().toUpperCase();
} else if (context.children.size() == 2) {
alias = context.children.get(1).getText().toUpperCase();
}
if (!isJoin) {
this.fromAlias = alias;
StructuredDataSource fromDataSource = metaStore.getSource(table.getName().getSuffix());
if (fromDataSource == null) {
throw new KsqlException(table.getName().getSuffix() + " does not exist.");
}
return null;
}
// TODO: Figure out if the call to toUpperCase() here is really necessary
return new AliasedRelation(getLocation(context), table, alias.toUpperCase(), getColumnAliases(context.columnAliases()));
}
use of io.confluent.ksql.metastore.StructuredDataSource in project ksql by confluentinc.
the class Analyzer method visitJoin.
@Override
protected Node visitJoin(final Join node, final AnalysisContext context) {
AliasedRelation left = (AliasedRelation) process(node.getLeft(), context);
AliasedRelation right = (AliasedRelation) process(node.getRight(), context);
String leftSideName = ((Table) left.getRelation()).getName().getSuffix();
StructuredDataSource leftDataSource = metaStore.getSource(leftSideName);
if (leftDataSource == null) {
throw new KsqlException(format("Resource %s does not exist.", leftSideName));
}
leftDataSource = timestampColumn(left, leftDataSource);
String rightSideName = ((Table) right.getRelation()).getName().getSuffix();
StructuredDataSource rightDataSource = metaStore.getSource(rightSideName);
if (rightDataSource == null) {
throw new KsqlException(format("Resource %s does not exist.", rightSideName));
}
rightDataSource = timestampColumn(right, rightDataSource);
String leftAlias = left.getAlias();
String rightAlias = right.getAlias();
JoinNode.Type joinType = getJoinType(node);
if (!node.getCriteria().isPresent()) {
throw new KsqlException(String.format("%s Join criteria is not set.", node.getLocation().isPresent() ? node.getLocation().get().toString() : ""));
}
JoinOn joinOn = (JoinOn) (node.getCriteria().get());
ComparisonExpression comparisonExpression = (ComparisonExpression) joinOn.getExpression();
Pair<String, String> leftSide = fetchKeyFieldName(comparisonExpression, leftAlias, leftDataSource.getSchema());
Pair<String, String> rightSide = fetchKeyFieldName(comparisonExpression, rightAlias, rightDataSource.getSchema());
String leftKeyFieldName = leftSide.getRight();
String rightKeyFieldName = rightSide.getRight();
if (comparisonExpression.getType() != ComparisonExpression.Type.EQUAL) {
throw new KsqlException("Only equality join criteria is supported.");
}
StructuredDataSourceNode leftSourceKafkaTopicNode = new StructuredDataSourceNode(new PlanNodeId("KafkaTopic_Left"), leftDataSource, leftDataSource.getSchema());
StructuredDataSourceNode rightSourceKafkaTopicNode = new StructuredDataSourceNode(new PlanNodeId("KafkaTopic_Right"), rightDataSource, rightDataSource.getSchema());
JoinNode joinNode = new JoinNode(new PlanNodeId("Join"), joinType, leftSourceKafkaTopicNode, rightSourceKafkaTopicNode, leftKeyFieldName, rightKeyFieldName, leftAlias, rightAlias);
analysis.setJoin(joinNode);
return null;
}
Aggregations