Examples with PushDownOptimizationMeta org.pentaho.di.trans.dataservice.optimization.PushDownOptimizationMeta used on opensource projects

Example 11 with PushDownOptimizationMeta

use of org.pentaho.di.trans.dataservice.optimization.PushDownOptimizationMeta in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testExecuteQueryNoResults.

@Test
public void testExecuteQueryNoResults() throws Exception {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
    StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).build();
    ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
    verify(serviceTrans).setContainerObjectId(objectIds.capture());
    when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(genTrans).setContainerObjectId(objectIds.capture());
    when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(serviceTrans).setMetaStore(metastore);
    verify(genTrans).setMetaStore(metastore);
    RowProducer sqlTransRowProducer = mock(RowProducer.class);
    when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    // Start Execution
    executor.executeQuery(new DataOutputStream(outputStream));
    // Check header was written
    assertThat(outputStream.size(), greaterThan(0));
    outputStream.reset();
    InOrder genTransStartup = inOrder(genTrans, resultStep);
    InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
    ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
    ArgumentCaptor<TransListener> transListenerCaptor = ArgumentCaptor.forClass(TransListener.class);
    genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
    genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
    genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
    genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
    RowListener clientRowListener = listenerArgumentCaptor.getValue();
    genTransStartup.verify(genTrans).startThreads();
    serviceTransStartup.verify(optimization).activate(executor);
    serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
    serviceTransStartup.verify(serviceTrans).startThreads();
    // Verify linkage
    RowListener serviceRowListener = listenerArgumentCaptor.getValue();
    assertNotNull(serviceRowListener);
    // Push row from service to sql Trans
    RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
    doReturn(true).when(serviceTrans).isRunning();
    resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
    verify(serviceTrans).stopAll();
    // Verify Service Trans finished
    ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addStepListener(serviceStepListener.capture());
    serviceStepListener.getValue().stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
    verify(sqlTransRowProducer).finished();
    // finish transformation, so that the listener runs
    transListenerCaptor.getValue().transFinished(genTrans);
    InOrder writeRows = inOrder(rowMeta);
    ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
    writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
    DataOutputStream dataOutputStream = streamCaptor.getValue();
    writeRows.verify(rowMeta, times(0)).writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
    writeRows.verifyNoMoreInteractions();
    executor.waitUntilFinished();
    verify(serviceTrans).waitUntilFinished();
    verify(genTrans).waitUntilFinished();
}

Example 12 with PushDownOptimizationMeta

use of org.pentaho.di.trans.dataservice.optimization.PushDownOptimizationMeta in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testExecuteQuery.

@Test
public void testExecuteQuery() throws Exception {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
    StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).build();
    ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
    verify(serviceTrans).setContainerObjectId(objectIds.capture());
    when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(genTrans).setContainerObjectId(objectIds.capture());
    when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
    verify(serviceTrans).setMetaStore(metastore);
    verify(genTrans).setMetaStore(metastore);
    RowProducer sqlTransRowProducer = mock(RowProducer.class);
    when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    // Start Execution
    executor.executeQuery(new DataOutputStream(outputStream));
    // Check header was written
    assertThat(outputStream.size(), greaterThan(0));
    outputStream.reset();
    InOrder genTransStartup = inOrder(genTrans, resultStep);
    InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
    ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
    ArgumentCaptor<TransListener> transListenerCaptor = ArgumentCaptor.forClass(TransListener.class);
    genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
    genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
    genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
    genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
    RowListener clientRowListener = listenerArgumentCaptor.getValue();
    genTransStartup.verify(genTrans).startThreads();
    serviceTransStartup.verify(optimization).activate(executor);
    serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
    serviceTransStartup.verify(serviceTrans).startThreads();
    // Verify linkage
    RowListener serviceRowListener = listenerArgumentCaptor.getValue();
    assertNotNull(serviceRowListener);
    // Push row from service to sql Trans
    RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
    Object[] data;
    for (int i = 0; i < 50; i++) {
        data = new Object[] { i };
        Object[] dataClone = { i };
        when(rowMeta.cloneRow(data)).thenReturn(dataClone);
        serviceRowListener.rowWrittenEvent(rowMeta, data);
        verify(sqlTransRowProducer).putRowWait(same(rowMeta), and(eq(dataClone), not(same(data))), any(Long.class), any(TimeUnit.class));
        verify(rowMeta).cloneRow(data);
    }
    doReturn(true).when(serviceTrans).isRunning();
    resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
    verify(serviceTrans).stopAll();
    // Verify Service Trans finished
    ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addStepListener(serviceStepListener.capture());
    serviceStepListener.getValue().stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
    verify(sqlTransRowProducer).finished();
    // Push row from service to sql Trans
    for (int i = 0; i < 50; i++) {
        Object[] row = { i };
        clientRowListener.rowWrittenEvent(rowMeta, row);
    }
    transListenerCaptor.getValue().transFinished(genTrans);
    InOrder writeRows = inOrder(rowMeta);
    ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
    writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
    DataOutputStream dataOutputStream = streamCaptor.getValue();
    writeRows.verify(rowMeta, times(50)).writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
    writeRows.verifyNoMoreInteractions();
    executor.waitUntilFinished();
    verify(serviceTrans).waitUntilFinished();
    verify(genTrans).waitUntilFinished();
}

Example 13 with PushDownOptimizationMeta

use of org.pentaho.di.trans.dataservice.optimization.PushDownOptimizationMeta in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testWaitUntilFinishedStreaming.

@Test
public void testWaitUntilFinishedStreaming() throws Exception {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    when(sqlTransGenerator.getSql()).thenReturn(sql);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    dataService.setStreaming(true);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).rowLimit(1).timeLimit(10000).windowMode(IDataServiceClientService.StreamingMode.ROW_BASED).windowSize(1).windowEvery(0).windowLimit(0).build();
    when(genTrans.isFinishedOrStopped()).thenAnswer(new Answer<Boolean>() {

        int numberThreadSleeps = 3;

        @Override
        public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
            return numberThreadSleeps-- <= 0;
        }
    });
    executor.waitUntilFinished();
    verify(genTrans, times(4)).isFinishedOrStopped();
    when(genTrans.isFinishedOrStopped()).thenReturn(true);
    executor.waitUntilFinished();
    verify(genTrans, times(5)).isFinishedOrStopped();
}

Example 14 with PushDownOptimizationMeta

use of org.pentaho.di.trans.dataservice.optimization.PushDownOptimizationMeta in project pdi-dataservice-server-plugin by pentaho.

the class DataServiceExecutorTest method testStopStreaming.

@Test
public void testStopStreaming() throws KettleException {
    SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
    when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
    when(sqlTransGenerator.getSql()).thenReturn(sql);
    PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
    when(optimization.isEnabled()).thenReturn(true);
    dataService.getPushDownOptimizationMeta().add(optimization);
    dataService.setStreaming(true);
    IMetaStore metastore = mock(IMetaStore.class);
    DataServiceExecutor executor = new DataServiceExecutor.Builder(sql, dataService, context).serviceTrans(serviceTrans).sqlTransGenerator(sqlTransGenerator).genTrans(genTrans).metastore(metastore).windowMode(IDataServiceClientService.StreamingMode.ROW_BASED).windowSize(1).windowEvery(0).windowLimit(0).build();
    executor.stop();
    verify(serviceTrans, times(0)).stopAll();
    verify(genTrans, times(0)).stopAll();
}

Example 15 with PushDownOptimizationMeta

use of org.pentaho.di.trans.dataservice.optimization.PushDownOptimizationMeta in project pdi-dataservice-server-plugin by pentaho.

the class AutoParameterGenerationServiceTest method testApplyWithExisting.

@Test
public void testApplyWithExisting() throws Exception {
    Map<String, Set<List<StepFieldOperations>>> operationPaths;
    List<StepFieldOperations> inputOperations = ImmutableList.of(new StepFieldOperations("Input", "field", null));
    operationPaths = ImmutableMap.of("Input 1", Collections.singleton(inputOperations));
    when(lineageClient.getOperationPaths(same(transMeta), eq(SERVICE_STEP), eq(ImmutableList.copyOf(SERVICE_FIELDS)))).thenReturn(operationPaths);
    StepMeta input1Meta = mock(StepMeta.class);
    when(transMeta.findStep("Input")).thenReturn(input1Meta);
    when(serviceProvider.supportsStep(input1Meta)).thenReturn(true);
    assertThat(service.parametrizedSteps(dataService), empty());
    assertThat(service.apply(dataService), hasSize(1));
    PushDownOptimizationMeta optimizationMeta = new PushDownOptimizationMeta();
    optimizationMeta.setName("Existing PDO");
    optimizationMeta.setStepName("Input");
    optimizationMeta.setType(mock(ParameterGeneration.class));
    dataService.setPushDownOptimizationMeta(Lists.newArrayList(optimizationMeta));
    assertThat(service.parametrizedSteps(dataService), contains("Input"));
    assertThat(service.apply(dataService), empty());
}