Examples with RowMetaAndData org.pentaho.di.core.RowMetaAndData used on opensource projects

Example 1 with RowMetaAndData

use of org.pentaho.di.core.RowMetaAndData in project pentaho-kettle by pentaho.

the class DefaultCache method storeRowInCache.

@Override
public void storeRowInCache(DatabaseLookupMeta meta, RowMetaInterface lookupMeta, Object[] lookupRow, Object[] add) {
    RowMetaAndData rowMetaAndData = new RowMetaAndData(lookupMeta, lookupRow);
    // DEinspanjer 2009-02-01 XXX: I want to write a test case to prove this point before checking in.
    // /* Don't insert a row with a duplicate key into the cache. It doesn't seem
    // * to serve a useful purpose and can potentially cause the step to return
    // * different values over the life of the transformation (if the source DB rows change)
    // * Additionally, if using the load all data feature, re-inserting would reverse the order
    // * specified in the step.
    // */
    // if (!data.look.containsKey(rowMetaAndData)) {
    // data.look.put(rowMetaAndData, new TimedRow(add));
    // }
    map.put(rowMetaAndData, new TimedRow(add));
    // method would throw out entries if the previous cache size wasn't big enough.
    if (!meta.isLoadingAllDataInCache() && meta.getCacheSize() > 0 && map.size() > meta.getCacheSize()) {
        List<RowMetaAndData> keys = new ArrayList<RowMetaAndData>(map.keySet());
        List<Date> samples = new ArrayList<Date>();
        int incr = keys.size() / 10;
        if (incr == 0) {
            incr = 1;
        }
        for (int k = 0; k < keys.size(); k += incr) {
            RowMetaAndData key = keys.get(k);
            TimedRow timedRow = map.get(key);
            samples.add(timedRow.getLogDate());
        }
        Collections.sort(samples);
        if (samples.size() > 1) {
            Date smallest = samples.get(1);
            // Everything below the smallest date goes away...
            for (RowMetaAndData key : keys) {
                TimedRow timedRow = map.get(key);
                if (timedRow.getLogDate().compareTo(smallest) < 0) {
                    map.remove(key);
                }
            }
        }
    }
}

Example 2 with RowMetaAndData

use of org.pentaho.di.core.RowMetaAndData in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method testReplayFullCache.

@Test
public void testReplayFullCache() throws Exception {
    DataServiceExecutor executor = dataServiceExecutor(BASE_QUERY);
    CachedService cachedService = CachedService.complete(testData);
    RowProducer rowProducer = genTrans.addRowProducer(INJECTOR_STEP, 0);
    // Activate cachedServiceLoader
    Executor mockExecutor = mock(Executor.class);
    final CachedServiceLoader cachedServiceLoader = new CachedServiceLoader(cachedService, mockExecutor);
    ListenableFuture<Integer> replay = cachedServiceLoader.replay(executor);
    ArgumentCaptor<Runnable> replayRunnable = ArgumentCaptor.forClass(Runnable.class);
    verify(mockExecutor).execute(replayRunnable.capture());
    stepMetaDataCombi.step = inputStep;
    stepMetaDataCombi.meta = inputStepMetaInterface;
    stepMetaDataCombi.data = inputStepDataInterface;
    List<StepMetaDataCombi> stepMetaDataCombis = new ArrayList<>();
    stepMetaDataCombis.add(stepMetaDataCombi);
    when(serviceTrans.getSteps()).thenReturn(stepMetaDataCombis);
    // Simulate executing data service
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.READY);
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.START);
    // Verify that serviceTrans never started, genTrans is accepting rows
    verify(serviceTrans).stopAll();
    verify(inputStep).setOutputDone();
    verify(inputStep).dispose(inputStepMetaInterface, inputStepDataInterface);
    verify(inputStep).markStop();
    verify(serviceTrans, never()).startThreads();
    verify(genTrans).startThreads();
    when(rowProducer.putRowWait(any(RowMetaInterface.class), any(Object[].class), anyInt(), any(TimeUnit.class))).then(new Answer<Boolean>() {

        int calls = 0;

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            // Simulate full row set on tenth call
            return ++calls != 10;
        }
    });
    when(genTrans.isRunning()).thenReturn(true);
    // Run cache loader (would be asynchronous)
    replayRunnable.getValue().run();
    assertThat(replay.get(1, TimeUnit.SECONDS), equalTo(testData.size()));
    InOrder rowsProduced = inOrder(rowProducer);
    for (int i = 0; i < testData.size(); i++) {
        RowMetaAndData metaAndData = testData.get(i);
        // Tenth row was called twice, since row set was full
        Object[] data = metaAndData.getData();
        rowsProduced.verify(rowProducer, times(i == 9 ? 2 : 1)).putRowWait(eq(metaAndData.getRowMeta()), and(eq(data), AdditionalMatchers.not(same(data))), anyInt(), any(TimeUnit.class));
    }
    rowsProduced.verify(rowProducer).finished();
    rowsProduced.verifyNoMoreInteractions();
}

Example 3 with RowMetaAndData

use of org.pentaho.di.core.RowMetaAndData in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method setUp.

@Before
public void setUp() throws Exception {
    rowMeta = new RowMeta();
    rowMeta.addValueMeta(new ValueMetaString("ID"));
    rowMeta.addValueMeta(new ValueMetaInteger("A"));
    rowMeta.addValueMeta(new ValueMetaInteger("B"));
    testData = Lists.newArrayListWithExpectedSize(100);
    for (long i = 0; i < 100; i++) {
        testData.add(new RowMetaAndData(rowMeta, String.valueOf(i), i % 13, i % 17));
    }
    when(serviceTrans.findRunThread(SERVICE_STEP)).thenReturn(serviceStep);
    when(serviceStep.getTrans()).thenReturn(serviceTrans);
    when(sqlTransGenerator.getInjectorStepName()).thenReturn(INJECTOR_STEP);
    transMeta = serviceTrans.getTransMeta();
    when(transMeta.getCacheVersion()).thenReturn(1);
    dataServiceMeta = new DataServiceMeta(transMeta);
    dataServiceMeta.setName(SERVICE_NAME);
    dataServiceMeta.setStepname(SERVICE_STEP);
    when(transMeta.getStepFields(SERVICE_STEP)).thenReturn(rowMeta);
}

Example 4 with RowMetaAndData

use of org.pentaho.di.core.RowMetaAndData in project pdi-dataservice-server-plugin by pentaho.

the class ServiceObserverTest method testRowIterator.

@Test
public void testRowIterator() throws Exception {
    CountDownLatch delayRowsLatch = new CountDownLatch(1);
    CountDownLatch firstRowLatch = new CountDownLatch(1);
    ServiceObserver serviceObserver = new ServiceObserver(executor) {

        @Override
        public void run() {
            rowMetaAndData.add(new RowMetaAndData());
            firstRowLatch.countDown();
            try {
                delayRowsLatch.await();
                rowMetaAndData.add(new RowMetaAndData());
                latch.countDown();
                rowMetaAndData.add(new RowMetaAndData());
                latch.countDown();
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            isRunning = false;
        }
    };
    Executors.newSingleThreadExecutor().submit(serviceObserver);
    firstRowLatch.await();
    Iterator<RowMetaAndData> rows = serviceObserver.rows();
    assertTrue(rows.hasNext());
    rows.next();
    Executors.newSingleThreadScheduledExecutor().schedule(delayRowsLatch::countDown, 1100, TimeUnit.MILLISECONDS);
    assertTrue(rows.hasNext());
    rows.next();
    assertTrue(rows.hasNext());
    rows.next();
    assertFalse(rows.hasNext());
    delayRowsLatch.countDown();
}

Example 5 with RowMetaAndData

use of org.pentaho.di.core.RowMetaAndData in project pentaho-kettle by pentaho.

the class KettleDatabaseRepository method insertStepDatabase.

public synchronized void insertStepDatabase(ObjectId id_transformation, ObjectId id_step, ObjectId id_database) throws KettleException {
    // First check if the relationship is already there.
    // There is no need to store it twice!
    RowMetaAndData check = getStepDatabase(id_step);
    if (check.getInteger(0) == null) {
        RowMetaAndData table = new RowMetaAndData();
        table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_STEP_DATABASE_ID_TRANSFORMATION), id_transformation);
        table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_STEP_DATABASE_ID_STEP), id_step);
        table.addValue(new ValueMetaInteger(KettleDatabaseRepository.FIELD_STEP_DATABASE_ID_DATABASE), id_database);
        connectionDelegate.insertTableRow(KettleDatabaseRepository.TABLE_R_STEP_DATABASE, table);
    }
}