Examples with DataSetOptions com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions used on opensource projects

Example 16 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class AbstractJdbcDataSetProviderTest method write.

/**
 * Verify writing to a JDBC table.
 */
@Test
public void write() throws Exception {
    // Mock data set
    final DataFrame dataFrame = Mockito.mock(DataFrame.class);
    final SQLContext sqlContext = Mockito.mock(SQLContext.class);
    Mockito.when(sqlContext.conf()).thenReturn(new SQLConf());
    Mockito.when(dataFrame.sqlContext()).thenReturn(sqlContext);
    final StructField field1 = DataTypes.createStructField("col1", DataTypes.IntegerType, true);
    final StructField field2 = DataTypes.createStructField("col2", DataTypes.StringType, true);
    Mockito.when(dataFrame.schema()).thenReturn(DataTypes.createStructType(Arrays.asList(field1, field2)));
    // Mock options
    final DataSetOptions options = new DataSetOptions();
    options.setOption("dbtable", "mytable");
    options.setOption("url", "jdbc:h2:mem:spark");
    // Test writing
    try (final Connection conn = DriverManager.getConnection("jdbc:h2:mem:spark")) {
        final MockJdbcDataSetProvider provider = new MockJdbcDataSetProvider();
        provider.write(Mockito.mock(KyloCatalogClient.class), options, dataFrame);
        try (final Statement stmt = conn.createStatement();
            final ResultSet rs = stmt.executeQuery("SHOW COLUMNS FROM mytable")) {
            Assert.assertTrue("Expected 2 rows; found 0 rows", rs.next());
            Assert.assertEquals("COL1", rs.getString(1));
            Assert.assertEquals("INTEGER(10)", rs.getString(2));
            Assert.assertTrue("Expected 2 rows; found 1 row", rs.next());
            Assert.assertEquals("COL2", rs.getString(1));
            Assert.assertEquals("CLOB(2147483647)", rs.getString(2));
            Assert.assertFalse("Expected 2 rows; fonud 3 rows", rs.next());
        }
    }
}

Example 17 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class AbstractJdbcDataSetProviderTest method writeWithoutDbtable.

/**
 * Verify exception if dbtable is not defined.
 */
@Test(expected = MissingOptionException.class)
public void writeWithoutDbtable() {
    final DataSetOptions options = new DataSetOptions();
    options.setOption("url", "jdbc:h2:mem:");
    final MockJdbcDataSetProvider provider = new MockJdbcDataSetProvider();
    provider.write(Mockito.mock(KyloCatalogClient.class), options, Mockito.mock(DataFrame.class));
}

Example 18 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class AbstractHiveDataSetProviderTest method write.

/**
 * Verify writing to a Hive table.
 */
@Test
public void write() {
    // Mock writer
    writer = createDataFrameWriter(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) {
            final InsertIntoTable plan = invocation.getArgumentAt(0, InsertIntoTable.class);
            final UnresolvedRelation relation = (UnresolvedRelation) plan.table();
            Assert.assertEquals("default", relation.tableIdentifier().database().get());
            Assert.assertEquals("mytable", relation.tableIdentifier().table());
            return null;
        }
    });
    // Create options
    final DataSetOptions options = new DataSetOptions();
    options.addJar("http://example.com/serde.jar");
    options.setOption(KyloCatalogConstants.PATH_OPTION, "default.mytable");
    // Test writing
    final MockHiveDataSetProvider provider = new MockHiveDataSetProvider();
    provider.write(Mockito.mock(KyloCatalogClient.class), options, Mockito.mock(DataFrame.class));
    Mockito.verify(sqlContext).sql("ADD JAR http://example.com/serde.jar");
    Mockito.verify(sqlContext).executePlan(Mockito.any(LogicalPlan.class));
}

Example 19 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class AbstractHiveDataSetProviderTest method read.

/**
 * Verify reading from a Hive table.
 */
@Test
public void read() {
    // Create options
    final DataSetOptions options = new DataSetOptions();
    options.addJar("http://example.com/serde.jar");
    options.setOption(KyloCatalogConstants.PATH_OPTION, "default.mytable");
    // Test reading
    dataFrame = Mockito.mock(DataFrame.class);
    final MockHiveDataSetProvider provider = new MockHiveDataSetProvider();
    final DataFrame result = provider.read(Mockito.mock(KyloCatalogClient.class), options);
    Assert.assertEquals(dataFrame, result);
    Mockito.verify(sqlContext).sql("ADD JAR http://example.com/serde.jar");
}

Example 20 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class AbstractHiveDataSetProviderTest method readWithoutPath.

/**
 * Verify exception if path not defined.
 */
@Test(expected = MissingOptionException.class)
public void readWithoutPath() {
    final MockHiveDataSetProvider provider = new MockHiveDataSetProvider();
    provider.read(Mockito.mock(KyloCatalogClient.class), new DataSetOptions());
}