Examples with TSnowflakeOutputProperties org.talend.components.snowflake.tsnowflakeoutput.TSnowflakeOutputProperties used on opensource projects

Example 11 with TSnowflakeOutputProperties

use of org.talend.components.snowflake.tsnowflakeoutput.TSnowflakeOutputProperties in project components by Talend.

the class SnowflakeWritersTestIT method testTableNamesOutput.

@Test
public void testTableNamesOutput() throws Throwable {
    TSnowflakeOutputProperties props = (TSnowflakeOutputProperties) getComponentService().getComponentProperties(TSnowflakeOutputDefinition.COMPONENT_NAME);
    setupProps(props.getConnectionProperties());
    ComponentTestUtils.checkSerialize(props, errorCollector);
    checkAndSetupTable(props);
}

Example 12 with TSnowflakeOutputProperties

use of org.talend.components.snowflake.tsnowflakeoutput.TSnowflakeOutputProperties in project components by Talend.

the class SnowflakeWritersTestIT method testOutputFeedback.

@Test
public void testOutputFeedback() throws Throwable {
    TSnowflakeOutputProperties props = (TSnowflakeOutputProperties) getComponentService().getComponentProperties(TSnowflakeOutputDefinition.COMPONENT_NAME);
    setupProps(props.getConnectionProperties());
    setupTableWithStaticValues(props);
    props.outputAction.setStoredValue(TSnowflakeOutputProperties.OutputAction.INSERT);
    DefaultComponentRuntimeContainerImpl container = new DefaultComponentRuntimeContainerImpl();
    // Initialize the Sink, WriteOperation and Writer
    SnowflakeSink sfSink = new SnowflakeSink();
    sfSink.initialize(container, props);
    sfSink.validate(container);
    SnowflakeWriteOperation sfWriteOp = sfSink.createWriteOperation();
    sfWriteOp.initialize(container);
    SnowflakeWriter sfWriter = sfSink.createWriteOperation().createWriter(container);
    sfWriter.open("uid1");
    List<IndexedRecord> rows = makeRows(2);
    IndexedRecord r = rows.get(0);
    r.put(0, "badId");
    r.put(2, "badBoolean");
    r.put(4, "badDate");
    r.put(5, "badTime");
    r.put(6, "badTimestamp");
    sfWriter.write(r);
    sfWriter.write(rows.get(1));
    Result wr1 = sfWriter.close();
    // The rejected writes would come in here
    Iterable<IndexedRecord> rejected = sfWriter.getRejectedWrites();
    Iterator<IndexedRecord> it = rejected.iterator();
    IndexedRecord rej;
    rej = it.next();
    // row
    assertEquals("1", rej.get(1));
    // character
    assertEquals("1", rej.get(3));
    assertThat((String) rej.get(4), containsString("Numeric value 'badId'"));
    // byte offset
    assertEquals("0", rej.get(5));
    // line
    assertEquals("1", rej.get(6));
    // code
    assertEquals("100038", rej.get(8));
    rej = it.next();
    // row
    assertEquals("1", rej.get(1));
    // character
    assertEquals("13", rej.get(3));
    assertThat((String) rej.get(4), containsString("Boolean value 'badBoolean'"));
    // byte offset
    assertEquals("12", rej.get(5));
    // line
    assertEquals("1", rej.get(6));
    // code
    assertEquals("100037", rej.get(8));
    rej = it.next();
    // row
    assertEquals("1", rej.get(1));
    // character
    assertEquals("32", rej.get(3));
    assertThat((String) rej.get(4), containsString("Date 'badDate'"));
    // byte offset
    assertEquals("31", rej.get(5));
    // line
    assertEquals("1", rej.get(6));
    // code
    assertEquals("100040", rej.get(8));
    rej = it.next();
    // row
    assertEquals("1", rej.get(1));
    // character
    assertEquals("40", rej.get(3));
    assertThat((String) rej.get(4), containsString("Time 'badTime'"));
    // byte offset
    assertEquals("39", rej.get(5));
    // line
    assertEquals("1", rej.get(6));
    // code
    assertEquals("100108", rej.get(8));
    rej = it.next();
    // row
    assertEquals("1", rej.get(1));
    // character
    assertEquals("48", rej.get(3));
    assertThat((String) rej.get(4), containsString("Timestamp 'badTimestamp'"));
    // byte offset
    assertEquals("47", rej.get(5));
    // line
    assertEquals("1", rej.get(6));
    // code
    assertEquals("100035", rej.get(8));
    assertFalse(it.hasNext());
    assertEquals(1, wr1.getSuccessCount());
    assertEquals(1, wr1.getRejectCount());
    assertEquals(2, wr1.getTotalCount());
    sfWriteOp.finalize(Arrays.asList(wr1), container);
}

Example 13 with TSnowflakeOutputProperties

use of org.talend.components.snowflake.tsnowflakeoutput.TSnowflakeOutputProperties in project components by Talend.

the class SnowflakeWritersTestIT method testSchemaSerialized.

@Test
public void testSchemaSerialized() throws Throwable {
    TSnowflakeOutputProperties outputProps = (TSnowflakeOutputProperties) getComponentService().getComponentProperties(TSnowflakeOutputDefinition.COMPONENT_NAME);
    Schema reject = SchemaBuilder.record("Reject").fields().name("A").type().stringType().noDefault().name("B").type().stringType().noDefault().endRecord();
    Schema main = SchemaBuilder.record("Main").fields().name("C").type().stringType().noDefault().name("D").type().stringType().noDefault().endRecord();
    assertEquals(1, outputProps.getAvailableConnectors(null, true).size());
    for (Connector connector : outputProps.getAvailableConnectors(null, true)) {
        if (connector.getName().equals(Connector.MAIN_NAME)) {
            outputProps.setConnectedSchema(connector, main, true);
        } else {
            outputProps.setConnectedSchema(connector, reject, true);
        }
    }
    String serialized = outputProps.toSerialized();
    TSnowflakeOutputProperties afterSerialized = org.talend.daikon.properties.Properties.Helper.fromSerializedPersistent(serialized, TSnowflakeOutputProperties.class).object;
    assertEquals(1, afterSerialized.getAvailableConnectors(null, true).size());
    for (Connector connector : afterSerialized.getAvailableConnectors(null, true)) {
        if (connector.getName().equals(Connector.MAIN_NAME)) {
            Schema main2 = afterSerialized.getSchema(connector, true);
            assertEquals(main.toString(), main2.toString());
        } else {
            Schema reject2 = afterSerialized.getSchema(connector, true);
            assertEquals(reject.toString(), reject2.toString());
        }
    }
}

Example 14 with TSnowflakeOutputProperties

use of org.talend.components.snowflake.tsnowflakeoutput.TSnowflakeOutputProperties in project components by Talend.

the class SnowflakeWritersTestIT method testOutputBadTable.

@Test
public void testOutputBadTable() throws Throwable {
    TSnowflakeOutputProperties outputProps = (TSnowflakeOutputProperties) getComponentService().getComponentProperties(TSnowflakeOutputDefinition.COMPONENT_NAME);
    setupProps(outputProps.connection);
    setupTableWithStaticValues(outputProps);
    SnowflakeTableProperties tableProps = outputProps.table;
    Form f = tableProps.getForm(Form.REFERENCE);
    tableProps.tableName.setValue("BADONE");
    tableProps = (SnowflakeTableProperties) PropertiesTestUtils.checkAndAfter(getComponentService(), f, tableProps.tableName.getName(), tableProps);
    LOGGER.info(String.valueOf(tableProps.getValidationResult()));
    assertEquals(ValidationResult.Result.ERROR, tableProps.getValidationResult().getStatus());
    assertThat(tableProps.getValidationResult().getMessage(), containsString("BADONE"));
}

Example 15 with TSnowflakeOutputProperties

use of org.talend.components.snowflake.tsnowflakeoutput.TSnowflakeOutputProperties in project components by Talend.

the class SnowflakeWritersTestIT method testOutputUpsert.

@Test
public void testOutputUpsert() throws Throwable {
    TSnowflakeOutputProperties props = (TSnowflakeOutputProperties) populateOutput(100);
    handleRows(makeRows(50), props, TSnowflakeOutputProperties.OutputAction.DELETE);
    assertEquals(50, readRows(props).size());
    Form f = props.getForm(MAIN);
    props = (TSnowflakeOutputProperties) PropertiesTestUtils.checkAndBeforePresent(getComponentService(), f, props.upsertKeyColumn.getName(), props);
    LOGGER.debug(props.upsertKeyColumn.getPossibleValues().toString());
    assertEquals(NUM_COLUMNS, props.upsertKeyColumn.getPossibleValues().size());
    props.upsertKeyColumn.setStoredValue("ID");
    handleRows(makeRows(100), props, TSnowflakeOutputProperties.OutputAction.UPSERT);
    assertEquals(100, readRows(props).size());
}