Examples with DataSetOptions com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions used on opensource projects

Example 6 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class AbstractJdbcDataSetProviderTest method writeWithoutUrl.

/**
 * Verify exception if url is not defined.
 */
@Test(expected = MissingOptionException.class)
public void writeWithoutUrl() {
    final DataSetOptions options = new DataSetOptions();
    options.setOption("dbtable", "mytable");
    final MockJdbcDataSetProvider provider = new MockJdbcDataSetProvider();
    provider.write(Mockito.mock(KyloCatalogClient.class), options, Mockito.mock(DataFrame.class));
}

Example 7 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method getPathsHighWaterMarkEmpty.

/**
 * Verify retrieving High Water Mark paths when no files match.
 */
@Test
@SuppressWarnings("unchecked")
public void getPathsHighWaterMarkEmpty() throws IOException {
    final long currentTime = System.currentTimeMillis();
    // Mock client
    final KyloCatalogClient client = Mockito.mock(KyloCatalogClient.class);
    Mockito.when(client.getHighWaterMarks()).thenReturn(Collections.singletonMap("water.mark", Long.toString(currentTime)));
    // Mock file
    final File file = tempFolder.newFile("file.txt");
    Assert.assertTrue(file.setLastModified(currentTime - 1000));
    // Mock options
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("mock");
    options.setOption(HighWaterMarkInputFormat.HIGH_WATER_MARK, "water.mark");
    options.setPaths(Collections.singletonList(file.getAbsolutePath()));
    // Mock delegate
    final SparkDataSetDelegate<DataFrame> delegate = Mockito.mock(SparkDataSetDelegate.class);
    Mockito.when(delegate.getHadoopConfiguration(Mockito.any(KyloCatalogClient.class))).thenReturn(new Configuration(false));
    Mockito.when(delegate.isFileFormat(Mockito.any(Class.class))).thenReturn(true);
    // Test resolving paths
    final SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, client, delegate);
    final List<String> paths = context.getPaths();
    Assert.assertNotNull("Expected paths to be non-null", paths);
    Assert.assertEquals("file:/dev/null", paths.get(0));
    Assert.assertEquals(1, paths.size());
    Mockito.verify(client).setHighWaterMarks(Collections.singletonMap("water.mark", Long.toString(currentTime)));
}

Example 8 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method getPathsHighWaterMark.

/**
 * Verify retrieving High Water Mark paths.
 */
@Test
@SuppressWarnings("unchecked")
public void getPathsHighWaterMark() throws IOException {
    final long currentTime = System.currentTimeMillis();
    // Mock client
    final KyloCatalogClient client = Mockito.mock(KyloCatalogClient.class);
    Mockito.when(client.getHighWaterMarks()).thenReturn(Collections.singletonMap("water.mark", Long.toString(currentTime - 60000)));
    // Mock files
    final List<String> inputPaths = new ArrayList<>();
    final File file1 = tempFolder.newFile("file1");
    Assert.assertTrue(file1.setLastModified(currentTime - 60000));
    inputPaths.add(file1.getAbsolutePath());
    final File file2 = tempFolder.newFile("file2");
    Assert.assertTrue(file2.setLastModified(currentTime - 30000));
    inputPaths.add(file2.getAbsolutePath());
    final File file3 = tempFolder.newFile("file3");
    Assert.assertTrue(file3.setLastModified(currentTime));
    inputPaths.add(file3.getAbsolutePath());
    // Mock options
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("mock");
    options.setOption(HighWaterMarkInputFormat.HIGH_WATER_MARK, "water.mark");
    options.setOption(HighWaterMarkInputFormat.MAX_FILE_AGE, "300000");
    options.setOption(HighWaterMarkInputFormat.MIN_FILE_AGE, "15000");
    options.setPaths(inputPaths);
    // Mock delegate
    final SparkDataSetDelegate<DataFrame> delegate = Mockito.mock(SparkDataSetDelegate.class);
    Mockito.when(delegate.getHadoopConfiguration(Mockito.any(KyloCatalogClient.class))).thenReturn(new Configuration(false));
    Mockito.when(delegate.isFileFormat(Mockito.any(Class.class))).thenReturn(true);
    // Test resolving paths
    final SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, client, delegate);
    final List<String> paths = context.getPaths();
    Assert.assertNotNull("Expected paths to be non-null", paths);
    Assert.assertEquals(file2.toURI().toString(), paths.get(0));
    Assert.assertEquals(1, paths.size());
    Mockito.verify(client).setHighWaterMarks(Collections.singletonMap("water.mark", Long.toString(file2.lastModified())));
}

Example 9 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method getPaths.

/**
 * Verify retrieving paths.
 */
@Test
@SuppressWarnings("unchecked")
public void getPaths() {
    // Mock options
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("mock");
    options.setOption(KyloCatalogConstants.PATH_OPTION, "/mock/path/file1.txt");
    options.setPaths(Collections.singletonList("/mock/path/file2.txt"));
    // Test getting path for non-file-format data sources
    final SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, Mockito.mock(KyloCatalogClient.class), Mockito.mock(SparkDataSetDelegate.class));
    Assert.assertEquals("/mock/path/file1.txt", context.getOption("path").get());
    Assert.assertEquals(Collections.singletonList("/mock/path/file2.txt"), context.getPaths());
}

Example 10 with DataSetOptions

use of com.thinkbiganalytics.kylo.catalog.spi.DataSetOptions in project kylo by Teradata.

the class SparkDataSetContextTest method getPathsFileFormat.

/**
 * Verify retrieving file format paths.
 */
@Test
@SuppressWarnings("unchecked")
public void getPathsFileFormat() {
    // Mock options
    final DataSetOptions options = new DataSetOptions();
    options.setFormat("text");
    options.setOption(KyloCatalogConstants.PATH_OPTION, "/mock/path/file1.txt");
    options.setPaths(Collections.singletonList("/mock/path/file2.txt"));
    // Mock delegate
    final SparkDataSetDelegate<DataFrame> delegate = Mockito.mock(SparkDataSetDelegate.class);
    Mockito.when(delegate.isFileFormat(Mockito.any(Class.class))).thenReturn(true);
    // Test retrieving paths
    final SparkDataSetContext<DataFrame> context = new SparkDataSetContext<>(options, Mockito.mock(KyloCatalogClient.class), delegate);
    final List<String> paths = context.getPaths();
    Assert.assertTrue("Expected path option to be empty", context.getOption("path").isEmpty());
    Assert.assertNotNull("Expected paths to be non-null", paths);
    Assert.assertEquals("/mock/path/file1.txt", paths.get(0));
    Assert.assertEquals("/mock/path/file2.txt", paths.get(1));
    Assert.assertEquals(2, paths.size());
}