Examples with SamzaSqlRelRecord org.apache.samza.sql.SamzaSqlRelRecord used on opensource projects

Example 26 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class TestBuildOutputRecordUdf method testSinglePair.

@Test
public void testSinglePair() {
    BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf();
    SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("key", "value");
    SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("key"), Arrays.asList("value"));
    Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames());
    Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues());
}

Example 27 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class TestBuildOutputRecordUdf method testNestedRecord.

@Test
public void testNestedRecord() {
    BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf();
    SamzaSqlRelRecord nestedSamzaSqlRelRecord = new SamzaSqlRelRecord(Arrays.asList("k3"), Arrays.asList("v3"));
    SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", nestedSamzaSqlRelRecord);
    SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", nestedSamzaSqlRelRecord));
    Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames());
    Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues());
}

Example 28 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class TestBuildOutputRecordUdf method testMultiPairs.

@Test
public void testMultiPairs() {
    BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf();
    SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", "v2");
    SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", "v2"));
    Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames());
    Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues());
}

Example 29 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class JoinTranslator method getTable.

private Table getTable(JoinInputNode tableNode, TranslatorContext context) {
    SqlIOConfig sourceTableConfig = resolveSQlIOForTable(tableNode.getRelNode(), context.getExecutionContext().getSamzaSqlApplicationConfig().getInputSystemStreamConfigBySource());
    if (sourceTableConfig == null || !sourceTableConfig.getTableDescriptor().isPresent()) {
        String errMsg = "Failed to resolve table source in join operation: node=" + tableNode.getRelNode();
        log.error(errMsg);
        throw new SamzaException(errMsg);
    }
    Table<KV<SamzaSqlRelRecord, SamzaSqlRelMessage>> table = context.getStreamAppDescriptor().getTable(sourceTableConfig.getTableDescriptor().get());
    if (tableNode.isRemoteTable()) {
        return table;
    }
    // If local table, load the table.
    // Load the local table with the fields in the join condition as composite key and relational message as the value.
    // Send the messages from the input stream denoted as 'table' to the created table store.
    MessageStream<SamzaSqlRelMessage> relOutputStream = context.getMessageStream(tableNode.getRelNode().getId());
    SamzaSqlRelRecordSerdeFactory.SamzaSqlRelRecordSerde keySerde = (SamzaSqlRelRecordSerdeFactory.SamzaSqlRelRecordSerde) new SamzaSqlRelRecordSerdeFactory().getSerde(null, null);
    SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde valueSerde = (SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde) new SamzaSqlRelMessageSerdeFactory().getSerde(null, null);
    List<Integer> tableKeyIds = tableNode.getKeyIds();
    // Let's always repartition by the join fields as key before sending the key and value to the table.
    // We need to repartition the stream denoted as table to ensure that both the stream and table that are joined
    // have the same partitioning scheme with the same partition key and number. Please note that bootstrap semantic is
    // not propagated to the intermediate streams. Please refer SAMZA-1613 for more details on this. Subsequently, the
    // results are consistent only after the local table is caught up.
    relOutputStream.partitionBy(m -> createSamzaSqlCompositeKey(m, tableKeyIds), m -> m, KVSerde.of(keySerde, valueSerde), intermediateStreamPrefix + "table_" + logicalOpId).sendTo(table);
    return table;
}

Example 30 with SamzaSqlRelRecord

use of org.apache.samza.sql.SamzaSqlRelRecord in project samza by apache.

the class TestSamzaSqlRelMessage method testCompositeKeyCreation.

@Test
public void testCompositeKeyCreation() {
    List<String> keyPartNames = Arrays.asList("kfield1", "kfield2");
    SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata(0L, 0L));
    SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Collections.singletonList(0));
    Assert.assertEquals(relRecord1.getFieldNames().size(), 1);
    Assert.assertEquals(relRecord1.getFieldNames().get(0), "field1");
    Assert.assertEquals(relRecord1.getFieldValues().get(0), "value1");
    SamzaSqlRelRecord relRecord2 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1, 0)));
    Assert.assertEquals(relRecord2.getFieldNames().size(), 2);
    Assert.assertEquals(relRecord2.getFieldNames().get(0), "kfield2");
    Assert.assertEquals(relRecord2.getFieldValues().get(0), "value2");
    Assert.assertEquals(relRecord2.getFieldNames().get(1), "kfield1");
    Assert.assertEquals(relRecord2.getFieldValues().get(1), "value1");
}