Examples with LogicalSchema io.confluent.ksql.schema.ksql.LogicalSchema used on opensource projects

Example 76 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldHandleHavingClause.

@Test
public void shouldHandleHavingClause() {
    // Note: HAVING clause are handled centrally by KsqlMaterialization. This logic will have been
    // installed as part of building the below statement:
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) AS COUNT FROM " + USER_TABLE + " GROUP BY USERID" + " HAVING SUM(REGISTERTIME) > 2;");
    final LogicalSchema schema = schema("COUNT", SqlTypes.BIGINT);
    final int matches = (int) USER_DATA_PROVIDER.data().values().stream().filter(row -> ((Long) row.get(0)) > 2).count();
    final Map<String, GenericRow> rows = waitForUniqueUserRows(matches, STRING_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    final MaterializedTable table = materialization.nonWindowed();
    rows.forEach((rowKey, value) -> {
        // Rows passing the HAVING clause:
        final GenericKey key = genericKey(rowKey);
        final List<Row> rowList = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION)));
        assertThat(rowList.size(), is(1));
        assertThat(rowList.get(0).schema(), is(schema));
        assertThat(rowList.get(0).key(), is(key));
        assertThat(rowList.get(0).value(), is(value));
    });
    USER_DATA_PROVIDER.data().entries().stream().filter(e -> !rows.containsKey(e.getKey().get(0))).forEach(e -> {
        // Rows filtered by the HAVING clause:
        final List<Row> rowList = withRetry(() -> Lists.newArrayList(table.get(e.getKey(), PARTITION)));
        assertThat(rowList.isEmpty(), is(true));
    });
}

Example 77 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldQueryMaterializedTableForTumblingWindowed.

@Test
public void shouldQueryMaterializedTableForTumblingWindowed() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) AS COUNT FROM " + USER_STREAM + " WINDOW TUMBLING (SIZE " + WINDOW_SIZE.getSeconds() + " SECONDS)" + " GROUP BY USERID;");
    final LogicalSchema schema = schema("COUNT", SqlTypes.BIGINT);
    final Map<Windowed<String>, GenericRow> rows = waitForUniqueUserRows(TIME_WINDOWED_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    assertThat(materialization.windowType(), is(Optional.of(WindowType.TUMBLING)));
    final MaterializedWindowedTable table = materialization.windowed();
    rows.forEach((k, v) -> {
        final Window w = Window.of(k.window().startTime(), k.window().endTime());
        final GenericKey key = genericKey(k.key());
        final List<WindowedRow> resultAtWindowStart = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.singleton(w.start()), Range.all())));
        assertThat("at exact window start", resultAtWindowStart, hasSize(1));
        assertThat(resultAtWindowStart.get(0).schema(), is(schema));
        assertThat(resultAtWindowStart.get(0).window(), is(Optional.of(w)));
        assertThat(resultAtWindowStart.get(0).key(), is(key));
        assertThat(resultAtWindowStart.get(0).value(), is(v));
        final List<WindowedRow> resultAtWindowEnd = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.all(), Range.singleton(w.end()))));
        assertThat("at exact window end", resultAtWindowEnd, hasSize(1));
        final List<WindowedRow> resultFromRange = withRetry(() -> withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().minusMillis(1), w.start().plusMillis(1)), Range.all()))));
        assertThat("range including window start", resultFromRange, is(resultAtWindowStart));
        final List<WindowedRow> resultPast = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().plusMillis(1), w.start().plusMillis(1)), Range.all())));
        assertThat("past start", resultPast, is(empty()));
    });
}

Example 78 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class JsonFormatTest method produceInitData.

private static void produceInitData() {
    TEST_HARNESS.produceRows(inputTopic, ORDER_DATA_PROVIDER, KAFKA, JSON);
    final LogicalSchema messageSchema = LogicalSchema.builder().keyColumn(SystemColumns.ROWKEY_NAME, SqlTypes.STRING).valueColumn(ColumnName.of("MESSAGE"), SqlTypes.STRING).build();
    final GenericKey messageKey = genericKey("1");
    final GenericRow messageRow = genericRow("{\"log\":{\"@timestamp\":\"2017-05-30T16:44:22.175Z\",\"@version\":\"1\"," + "\"caasVersion\":\"0.0.2\",\"cloud\":\"aws\",\"logs\":[{\"entry\":\"first\"}],\"clusterId\":\"cp99\",\"clusterName\":\"kafka\",\"cpComponentId\":\"kafka\",\"host\":\"kafka-1-wwl0p\",\"k8sId\":\"k8s13\",\"k8sName\":\"perf\",\"level\":\"ERROR\",\"logger\":\"kafka.server.ReplicaFetcherThread\",\"message\":\"Found invalid messages during fetch for partition [foo512,172] offset 0 error Record is corrupt (stored crc = 1321230880, computed crc = 1139143803)\",\"networkId\":\"vpc-d8c7a9bf\",\"region\":\"us-west-2\",\"serverId\":\"1\",\"skuId\":\"sku5\",\"source\":\"kafka\",\"tenantId\":\"t47\",\"tenantName\":\"perf-test\",\"thread\":\"ReplicaFetcherThread-0-2\",\"zone\":\"us-west-2a\"},\"stream\":\"stdout\",\"time\":2017}");
    final Map<GenericKey, GenericRow> records = new HashMap<>();
    records.put(messageKey, messageRow);
    final PhysicalSchema schema = PhysicalSchema.from(messageSchema, SerdeFeatures.of(), SerdeFeatures.of());
    TEST_HARNESS.produceRows(messageLogTopic, records.entrySet(), schema, KAFKA, JSON);
}

Example 79 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class ProjectOperatorTest method shouldProjectOnlyWindowStartWindowed.

@Test
public void shouldProjectOnlyWindowStartWindowed() {
    // Given:
    final LogicalSchema schema = LogicalSchema.builder().valueColumn(SystemColumns.WINDOWSTART_NAME, SqlTypes.BIGINT).build();
    when(logicalNode.getAddAdditionalColumnsToIntermediateSchema()).thenReturn(true);
    when(logicalNode.getSchema()).thenReturn(schema);
    when(logicalNode.getCompiledSelectExpressions()).thenReturn(Collections.emptyList());
    final ProjectOperator projectOperator = new ProjectOperator(logger, logicalNode, selectValueMapperFactorySupplier);
    projectOperator.addChild(child);
    final QueryRowImpl windowedRow = QueryRowImpl.of(WINDOWED_INTERMEDIATE_SCHEMA_WITH_PSEUDO, A_KEY, Optional.of(A_WINDOW), GenericRow.genericRow("a", "b", A_ROWTIME, "k", A_WINDOW.start().toEpochMilli(), A_WINDOW.end().toEpochMilli()), A_ROWTIME);
    when(child.next()).thenReturn(windowedRow);
    when(selectValueMapperFactorySupplier.create(any(), any())).thenReturn(selectValueMapper);
    when(selectValueMapper.getTransformer(logger)).thenReturn(transformer);
    when(transformer.transform(A_KEY, windowedRow.value(), new PullProcessingContext(12335L))).thenReturn(GenericRow.genericRow(A_WINDOW.start().toEpochMilli()));
    projectOperator.open();
    // When:
    QueryRow result = (QueryRow) projectOperator.next();
    // Then:
    assertThat(result.value().values(), is(ImmutableList.of(A_WINDOW.start().toEpochMilli())));
}

Example 80 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class ProjectOperatorTest method shouldProjectKeyAndValueNonWindowed.

@Test
public void shouldProjectKeyAndValueNonWindowed() {
    // Given:
    final LogicalSchema schema = LogicalSchema.builder().keyColumn(ColumnName.of("k0"), SqlTypes.STRING).valueColumn(ColumnName.of("v1"), SqlTypes.STRING).build();
    when(logicalNode.getAddAdditionalColumnsToIntermediateSchema()).thenReturn(true);
    when(logicalNode.getSchema()).thenReturn(schema);
    when(logicalNode.getCompiledSelectExpressions()).thenReturn(Collections.emptyList());
    final ProjectOperator projectOperator = new ProjectOperator(logger, logicalNode, selectValueMapperFactorySupplier);
    projectOperator.addChild(child);
    final QueryRowImpl row = QueryRowImpl.of(INTERMEDIATE_SCHEMA_WITH_PSEUDO, A_KEY, Optional.empty(), GenericRow.genericRow("a", "b", A_ROWTIME, "k"), A_ROWTIME);
    when(child.next()).thenReturn(row);
    when(selectValueMapperFactorySupplier.create(any(), any())).thenReturn(selectValueMapper);
    when(selectValueMapper.getTransformer(logger)).thenReturn(transformer);
    when(transformer.transform(A_KEY, row.value(), new PullProcessingContext(12335L))).thenReturn(GenericRow.genericRow("k", "b"));
    projectOperator.open();
    // When:
    QueryRow result = (QueryRow) projectOperator.next();
    // Then:
    final List<Object> expected = new ArrayList<>(row.key().values());
    expected.add(row.value().values().get(1));
    assertThat(result.value().values(), is(expected));
}