Examples with TransformService com.thinkbiganalytics.spark.service.TransformService used on opensource projects

Example 6 with TransformService

use of com.thinkbiganalytics.spark.service.TransformService in project kylo by Teradata.

the class LivyWranglerConfig method transformService.

/**
 * Gets the transform service.
 *
 * @param transformScriptClass      the transform script class
 * @param engine                    the Spark script engine
 * @param sparkContextService       the Spark context service
 * @param tracker                   the transform job tracker
 * @param datasourceProviderFactory the data source provider factory
 * @param profiler                  the profiler
 * @return the transform service
 */
@Bean
public TransformService transformService(final Class<? extends TransformScript> transformScriptClass, final SparkScriptEngine engine, final SparkContextService sparkContextService, final JobTrackerService tracker, final DatasourceProviderFactory datasourceProviderFactory, final Profiler profiler, final DataValidator validator, final FileSystem fileSystem, final DataSetConverterService converterService, final KyloCatalogClientBuilder kyloCatalogClientBuilder, final CatalogDataSetProviderFactory catalogDataSetProviderFactory) {
    final TransformService service = new TransformService(transformScriptClass, engine, sparkContextService, tracker, converterService, kyloCatalogClientBuilder);
    service.setDatasourceProviderFactory(datasourceProviderFactory);
    service.setFileSystem(fileSystem);
    service.setProfiler(profiler);
    service.setValidator(validator);
    service.setCatalogDataSetProviderFactory(catalogDataSetProviderFactory);
    return service;
}

Example 7 with TransformService

use of com.thinkbiganalytics.spark.service.TransformService in project kylo by Teradata.

the class SparkShellTransformControllerTest method create.

/**
 * Verify requesting a Spark transformation.
 */
@Test
public void create() throws Exception {
    // Mock transform objects
    TransformRequest transformRequest = new TransformRequest();
    transformRequest.setScript("sqlContext.sql(\"SELECT * FROM invalid\")");
    TransformResponse transformResponse = new TransformResponse();
    transformResponse.setProgress(0.0);
    transformResponse.setStatus(TransformResponse.Status.PENDING);
    transformResponse.setTable("results");
    TransformService transformService = Mockito.mock(TransformService.class);
    Mockito.when(transformService.execute(transformRequest)).thenReturn(transformResponse);
    // Test transforming
    SparkShellTransformController controller = new SparkShellTransformController();
    controller.transformService = transformService;
    Response response = controller.create(transformRequest);
    Assert.assertEquals(Response.Status.OK, response.getStatusInfo());
    Assert.assertEquals(transformResponse, response.getEntity());
}

Example 8 with TransformService

use of com.thinkbiganalytics.spark.service.TransformService in project kylo by Teradata.

the class AbstractTransformControllerTest method getTable.

/**
 * Verify requesting a transformation status.
 */
@Test
public void getTable() throws Exception {
    // Mock transform objects
    TransformJob pendingJob = Mockito.mock(TransformJob.class);
    Mockito.when(pendingJob.getGroupId()).thenReturn("PendingJob");
    Mockito.when(pendingJob.progress()).thenReturn(0.5);
    TransformJob successJob = Mockito.mock(TransformJob.class);
    TransformResponse successResponse = new TransformResponse();
    Mockito.when(successJob.get()).thenReturn(successResponse);
    Mockito.when(successJob.isDone()).thenReturn(true);
    TransformService transformService = Mockito.mock(TransformService.class);
    Mockito.when(transformService.getTransformJob("PendingJob")).thenReturn(pendingJob);
    Mockito.when(transformService.getTransformJob("SuccessJob")).thenReturn(successJob);
    // Test with pending job
    AbstractTransformController controller = new AbstractTransformController() {
    };
    controller.transformService = transformService;
    Response response = controller.getTable("PendingJob");
    Assert.assertEquals(Response.Status.OK, response.getStatusInfo());
    TransformResponse transformResponse = (TransformResponse) response.getEntity();
    Assert.assertEquals(0.5, transformResponse.getProgress(), 0.001);
    Assert.assertEquals(TransformResponse.Status.PENDING, transformResponse.getStatus());
    Assert.assertEquals("PendingJob", transformResponse.getTable());
    // Test with success job
    response = controller.getTable("SuccessJob");
    Assert.assertEquals(successResponse, response.getEntity());
    Assert.assertEquals(Response.Status.OK, response.getStatusInfo());
}