Examples with DataServiceExecutor org.pentaho.di.trans.dataservice.DataServiceExecutor used on opensource projects

Example 16 with DataServiceExecutor

use of org.pentaho.di.trans.dataservice.DataServiceExecutor in project pdi-dataservice-server-plugin by pentaho.

the class ExecutorQueryServiceTest method testQueryBuildsWithMetastore.

@Test
public void testQueryBuildsWithMetastore() throws Exception {
    final DataServiceFactory factory = mock(DataServiceFactory.class);
    final DataServiceContext context = mock(DataServiceContext.class);
    final DataServiceResolver dataServiceResolver = mock(DataServiceResolver.class);
    final DataServiceExecutor dataServiceExecutor = mock(DataServiceExecutor.class);
    final Trans serviceTrans = mock(Trans.class);
    final Trans genTrans = mock(Trans.class);
    when(context.getMetaStoreUtil()).thenReturn(factory);
    DataServiceExecutor.Builder builder = mock(DataServiceExecutor.Builder.class);
    final IMetaStore metastore = mock(IMetaStore.class);
    final MetastoreLocator metastoreLocator = mock(MetastoreLocator.class);
    when(metastoreLocator.getMetastore()).thenReturn(metastore);
    SQL sql = new SQL("select field from table");
    HashMap<String, String> parameters = new HashMap<>();
    int rowLimit = 5432;
    ExecutorQueryService executorQueryService = new ExecutorQueryService(dataServiceResolver, metastoreLocator);
    when(dataServiceResolver.createBuilder(argThat(matchesSql(sql)))).thenReturn(builder);
    when(factory.getMetaStore()).thenReturn(metastore);
    when(builder.rowLimit(rowLimit)).thenReturn(builder);
    when(builder.parameters(parameters)).thenReturn(builder);
    when(builder.metastore(metastore)).thenReturn(builder);
    when(builder.windowMode(null)).thenReturn(builder);
    when(builder.windowSize(0)).thenReturn(builder);
    when(builder.windowEvery(0)).thenReturn(builder);
    when(builder.windowLimit(0)).thenReturn(builder);
    when(builder.build()).thenReturn(dataServiceExecutor);
    when(dataServiceExecutor.getServiceTrans()).thenReturn(serviceTrans);
    when(dataServiceExecutor.getGenTrans()).thenReturn(genTrans);
    Query result = executorQueryService.prepareQuery(sql.getSqlString(), rowLimit, parameters);
    assertEquals(ImmutableList.of(serviceTrans, genTrans), result.getTransList());
    verify(builder).rowLimit(rowLimit);
    verify(builder).parameters(parameters);
    verify(builder).metastore(metastore);
}

Example 17 with DataServiceExecutor

use of org.pentaho.di.trans.dataservice.DataServiceExecutor in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method testObserve.

@Test
public void testObserve() throws Exception {
    DataServiceExecutor executor = dataServiceExecutor(BASE_QUERY);
    ServiceObserver observer = new ServiceObserver(executor);
    assertThat(observer.install(), sameInstance((ListenableFuture<CachedService>) observer));
    assertThat(executor.getListenerMap().get(DataServiceExecutor.ExecutionPoint.READY), hasItem(observer));
    observer.run();
    ArgumentCaptor<RowListener> rowListener = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> stepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addRowListener(rowListener.capture());
    verify(serviceStep).addStepListener(stepListener.capture());
    assertThat(observer.isDone(), is(false));
    for (RowMetaAndData metaAndData : testData) {
        rowListener.getValue().rowWrittenEvent(metaAndData.getRowMeta(), metaAndData.getData());
    }
    stepListener.getValue().stepFinished(serviceTrans, mock(StepMeta.class), serviceStep);
    assertThat(observer.isDone(), is(true));
    assertThat(observer.get().getRowMetaAndData(), equalTo((Iterable<RowMetaAndData>) testData));
    assertThat(observer.get().isComplete(), is(true));
}

Example 18 with DataServiceExecutor

use of org.pentaho.di.trans.dataservice.DataServiceExecutor in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method testReplayPartialCache.

@Test
public void testReplayPartialCache() throws Exception {
    DataServiceExecutor executor = dataServiceExecutor(BASE_QUERY + " LIMIT 20");
    CachedService cachedService = CachedService.complete(testData);
    RowProducer rowProducer = genTrans.addRowProducer(INJECTOR_STEP, 0);
    // Activate cachedServiceLoader
    Executor mockExecutor = mock(Executor.class);
    final CachedServiceLoader cachedServiceLoader = new CachedServiceLoader(cachedService, mockExecutor);
    ListenableFuture<Integer> replay = cachedServiceLoader.replay(executor);
    ArgumentCaptor<Runnable> replayRunnable = ArgumentCaptor.forClass(Runnable.class);
    verify(mockExecutor).execute(replayRunnable.capture());
    stepMetaDataCombi.step = inputStep;
    stepMetaDataCombi.meta = inputStepMetaInterface;
    stepMetaDataCombi.data = inputStepDataInterface;
    List<StepMetaDataCombi> stepMetaDataCombis = new ArrayList<>();
    stepMetaDataCombis.add(stepMetaDataCombi);
    when(serviceTrans.getSteps()).thenReturn(stepMetaDataCombis);
    // Simulate executing data service
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.READY);
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.START);
    // Verify that serviceTrans never started, genTrans is accepting rows
    verify(serviceTrans).stopAll();
    verify(inputStep).setOutputDone();
    verify(inputStep).dispose(inputStepMetaInterface, inputStepDataInterface);
    verify(inputStep).markStop();
    verify(serviceTrans, never()).startThreads();
    verify(genTrans).startThreads();
    final AtomicInteger rowsProduced = new AtomicInteger(0);
    when(rowProducer.putRowWait(any(RowMetaInterface.class), any(Object[].class), anyInt(), any(TimeUnit.class))).then(new Answer<Boolean>() {

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            rowsProduced.getAndIncrement();
            return true;
        }
    });
    when(genTrans.isRunning()).then(new Answer<Boolean>() {

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            return rowsProduced.get() < 20;
        }
    });
    // Run cache loader (would be asynchronous)
    replayRunnable.getValue().run();
    verify(rowProducer).finished();
    assertThat(replay.get(1, TimeUnit.SECONDS), equalTo(20));
    assertThat(rowsProduced.get(), equalTo(20));
    for (RowMetaAndData metaAndData : Iterables.limit(testData, 20)) {
        Object[] data = metaAndData.getData();
        verify(rowProducer).putRowWait(eq(metaAndData.getRowMeta()), and(eq(data), AdditionalMatchers.not(same(data))), anyInt(), any(TimeUnit.class));
    }
}

Example 19 with DataServiceExecutor

use of org.pentaho.di.trans.dataservice.DataServiceExecutor in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method testObservePartial.

@Test
public void testObservePartial() throws Exception {
    DataServiceExecutor executor = dataServiceExecutor(BASE_QUERY + " LIMIT 20");
    ServiceObserver observer = new ServiceObserver(executor);
    observer.install();
    assertThat(executor.getListenerMap().get(DataServiceExecutor.ExecutionPoint.READY), hasItem(observer));
    observer.run();
    ArgumentCaptor<RowListener> rowListener = ArgumentCaptor.forClass(RowListener.class);
    ArgumentCaptor<StepListener> stepListener = ArgumentCaptor.forClass(StepListener.class);
    verify(serviceStep).addRowListener(rowListener.capture());
    verify(serviceStep).addStepListener(stepListener.capture());
    assertThat(observer.isDone(), is(false));
    for (RowMetaAndData metaAndData : FluentIterable.from(testData)) {
        rowListener.getValue().rowWrittenEvent(metaAndData.getRowMeta(), metaAndData.getData());
    }
    when(serviceStep.isStopped()).thenReturn(true);
    stepListener.getValue().stepFinished(serviceTrans, mock(StepMeta.class), serviceStep);
    assertThat(observer.isDone(), is(true));
    CachedService cachedService = observer.get(5, TimeUnit.SECONDS);
    assertThat(cachedService.isComplete(), is(false));
    assertThat(cachedService.getRowMetaAndData(), equalTo(testData));
}

Example 20 with DataServiceExecutor

use of org.pentaho.di.trans.dataservice.DataServiceExecutor in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method testAnswersOverflowDETQuery.

@Test
public void testAnswersOverflowDETQuery() throws Exception {
    String query = "SELECT * FROM " + SERVICE_NAME;
    DataServiceExecutor executor = dataServiceExecutor(query);
    when(sqlTransGenerator.getServiceRowLimit()).thenReturn(10000);
    CachedService partial = partial(executor);
    String aggregateQuery = "SELECT count(*) FROM " + SERVICE_NAME;
    assertThat(partial.answersQuery(dataServiceExecutor(aggregateQuery)), is(true));
}