Examples with DataSetOptions com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions used on opensource projects

Example 11 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method isFileFormat.

/**
 * Verify determining if a format reads files.
 */
@Test
@SuppressWarnings("unchecked")
public void isFileFormat() {
    // Mock delegate
    final SparkDataSetDelegate<DataFrame> delegate = Mockito.mock(SparkDataSetDelegate.class);
    Mockito.when(delegate.isFileFormat(Mockito.any(Class.class))).then(new Answer<Boolean>() {

        @Nonnull
        @Override
        public Boolean answer(@Nonnull final InvocationOnMock invocation) {
            final Class<?> clazz = invocation.getArgumentAt(0, Class.class);
            return HadoopFsRelationProvider.class.isAssignableFrom(clazz);
        }
    });
    // Test invalid format
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("invalid");
    final KyloCatalogClient<DataFrame> client = Mockito.mock(KyloCatalogClient.class);
    SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, client, delegate);
    Assert.assertFalse("Expected 'invalid' to not be a file format", context.isFileFormat());
    // Test non-file format
    options.setFormat("jdbc");
    context = new SparkDataSetContext<>(options, client, delegate);
    Assert.assertFalse("Expected 'jdbc' to not be a file format", context.isFileFormat());
    // Test short name
    options.setFormat("text");
    context = new SparkDataSetContext<>(options, client, delegate);
    Assert.assertTrue("Expected 'text' to be a file format", context.isFileFormat());
    // Test class name
    options.setFormat("org.apache.spark.sql.execution.datasources.text.DefaultSource");
    context = new SparkDataSetContext<>(options, client, delegate);
    Assert.assertTrue("Expected 'org.apache.spark.sql.execution.datasources.text.DefaultSource' to be a file format", context.isFileFormat());
    // Test package name
    options.setFormat("org.apache.spark.sql.execution.datasources.text");
    context = new SparkDataSetContext<>(options, client, delegate);
    Assert.assertTrue("Expected 'org.apache.spark.sql.execution.datasources.text' to be a file format", context.isFileFormat());
}

Example 12 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method getOptionsFileFormat.

/**
 * Verify retrieving options for file data sources.
 */
@Test
@SuppressWarnings("unchecked")
public void getOptionsFileFormat() {
    // Mock options
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("text");
    options.setOption(KyloCatalogConstants.PATH_OPTION, "/mock/path/file.txt");
    // Mock delegate
    final SparkDataSetDelegate<DataFrame> delegate = Mockito.mock(SparkDataSetDelegate.class);
    Mockito.when(delegate.isFileFormat(Mockito.any(Class.class))).thenReturn(true);
    // Test getting path
    final SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, Mockito.mock(KyloCatalogClient.class), delegate);
    Assert.assertTrue("Expected path option to be empty", context.getOption(KyloCatalogConstants.PATH_OPTION).isEmpty());
}

Example 13 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method getOptions.

/**
 * Verify retrieving options.
 */
@Test
@SuppressWarnings("unchecked")
public void getOptions() {
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("text");
    options.setOption(KyloCatalogConstants.PATH_OPTION, "/mock/path/file.txt");
    final SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, Mockito.mock(KyloCatalogClient.class), Mockito.mock(SparkDataSetDelegate.class));
    Assert.assertEquals("/mock/path/file.txt", context.getOption(KyloCatalogConstants.PATH_OPTION).get());
}

Example 14 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method getPathsFileFormatEmpty.

/**
 * Verify retrieving {@code null} paths.
 */
@Test
@SuppressWarnings("unchecked")
public void getPathsFileFormatEmpty() {
    // Mock options
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("text");
    // Mock delegate
    final SparkDataSetDelegate<DataFrame> delegate = Mockito.mock(SparkDataSetDelegate.class);
    Mockito.when(delegate.isFileFormat(Mockito.any(Class.class))).thenReturn(true);
    // Test getting paths
    final SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, Mockito.mock(KyloCatalogClient.class), delegate);
    Assert.assertNull(context.getPaths());
}

Example 15 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class AbstractJdbcDataSetProviderTest method read.

/**
 * Verify reading from a JDBC table.
 */
@Test
public void read() {
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("jdbc");
    // Test reading
    final MockJdbcDataSetProvider provider = new MockJdbcDataSetProvider();
    provider.read(Mockito.mock(KyloCatalogClient.class), options);
    Mockito.verify(reader).format(JdbcRelationProvider.class.getName());
}