Examples with StepMetaDataCombi org.pentaho.di.trans.step.StepMetaDataCombi used on opensource projects

Example 26 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pdi-dataservice-server-plugin by pentaho.

the class CachedServiceTest method testReplayFullCache.

@Test
public void testReplayFullCache() throws Exception {
    DataServiceExecutor executor = dataServiceExecutor(BASE_QUERY);
    CachedService cachedService = CachedService.complete(testData);
    RowProducer rowProducer = genTrans.addRowProducer(INJECTOR_STEP, 0);
    // Activate cachedServiceLoader
    Executor mockExecutor = mock(Executor.class);
    final CachedServiceLoader cachedServiceLoader = new CachedServiceLoader(cachedService, mockExecutor);
    ListenableFuture<Integer> replay = cachedServiceLoader.replay(executor);
    ArgumentCaptor<Runnable> replayRunnable = ArgumentCaptor.forClass(Runnable.class);
    verify(mockExecutor).execute(replayRunnable.capture());
    stepMetaDataCombi.step = inputStep;
    stepMetaDataCombi.meta = inputStepMetaInterface;
    stepMetaDataCombi.data = inputStepDataInterface;
    List<StepMetaDataCombi> stepMetaDataCombis = new ArrayList<>();
    stepMetaDataCombis.add(stepMetaDataCombi);
    when(serviceTrans.getSteps()).thenReturn(stepMetaDataCombis);
    // Simulate executing data service
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.READY);
    executor.executeListeners(DataServiceExecutor.ExecutionPoint.START);
    // Verify that serviceTrans never started, genTrans is accepting rows
    verify(serviceTrans).stopAll();
    verify(inputStep).setOutputDone();
    verify(inputStep).dispose(inputStepMetaInterface, inputStepDataInterface);
    verify(inputStep).markStop();
    verify(serviceTrans, never()).startThreads();
    verify(genTrans).startThreads();
    when(rowProducer.putRowWait(any(RowMetaInterface.class), any(Object[].class), anyInt(), any(TimeUnit.class))).then(new Answer<Boolean>() {

        int calls = 0;

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            // Simulate full row set on tenth call
            return ++calls != 10;
        }
    });
    when(genTrans.isRunning()).thenReturn(true);
    // Run cache loader (would be asynchronous)
    replayRunnable.getValue().run();
    assertThat(replay.get(1, TimeUnit.SECONDS), equalTo(testData.size()));
    InOrder rowsProduced = inOrder(rowProducer);
    for (int i = 0; i < testData.size(); i++) {
        RowMetaAndData metaAndData = testData.get(i);
        // Tenth row was called twice, since row set was full
        Object[] data = metaAndData.getData();
        rowsProduced.verify(rowProducer, times(i == 9 ? 2 : 1)).putRowWait(eq(metaAndData.getRowMeta()), and(eq(data), AdditionalMatchers.not(same(data))), anyInt(), any(TimeUnit.class));
    }
    rowsProduced.verify(rowProducer).finished();
    rowsProduced.verifyNoMoreInteractions();
}

Example 27 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-metaverse by pentaho.

the class StepExternalResourceConsumerListener method callExtensionPoint.

/**
 * This method is called by the Kettle code when a step is about to start
 *
 * @param log    the logging channel to log debugging information to
 * @param object The subject object that is passed to the plugin code
 * @throws org.pentaho.di.core.exception.KettleException In case the plugin decides that an error has occurred
 *                                                       and the parent process should stop.
 */
@Override
public void callExtensionPoint(LogChannelInterface log, Object object) throws KettleException {
    if (stepConsumerProvider == null) {
        stepConsumerProvider = (IStepExternalResourceConsumerProvider) MetaverseBeanUtil.getInstance().get("IStepExternalResourceConsumerProvider");
    }
    StepMetaDataCombi stepCombi = (StepMetaDataCombi) object;
    if (stepCombi != null) {
        StepMetaInterface meta = stepCombi.meta;
        StepInterface step = stepCombi.step;
        if (meta != null) {
            Class<?> metaClass = meta.getClass();
            if (BaseStepMeta.class.isAssignableFrom(metaClass)) {
                if (stepConsumerProvider != null) {
                    // Put the class into a collection and get the consumers that can process this class
                    Set<Class<?>> metaClassSet = new HashSet<Class<?>>(1);
                    metaClassSet.add(metaClass);
                    List<IStepExternalResourceConsumer> stepConsumers = stepConsumerProvider.getExternalResourceConsumers(metaClassSet);
                    if (stepConsumers != null) {
                        for (IStepExternalResourceConsumer stepConsumer : stepConsumers) {
                            // We might know enough at this point, so call the consumer
                            Collection<IExternalResourceInfo> resources = stepConsumer.getResourcesFromMeta(meta);
                            addExternalResources(resources, step);
                            // Add a RowListener if the step is data-driven
                            if (stepConsumer.isDataDriven(meta)) {
                                stepCombi.step.addRowListener(new StepExternalConsumerRowListener(stepConsumer, step));
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 28 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class Validator method init.

public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (ValidatorMeta) smi;
    data = (ValidatorData) sdi;
    if (super.init(smi, sdi)) {
        // initialize steps by names
        List<StepMeta> steps = new ArrayList<>();
        List<StepMetaDataCombi> transSteps = getTrans().getSteps();
        if (transSteps != null) {
            for (StepMetaDataCombi s : transSteps) {
                steps.add(s.stepMeta);
            }
        }
        meta.searchInfoAndTargetSteps(steps);
        // initialize arrays of validation data
        data.constantsMeta = new ValueMetaInterface[meta.getValidations().size()];
        data.minimumValueAsString = new String[meta.getValidations().size()];
        data.maximumValueAsString = new String[meta.getValidations().size()];
        data.fieldsMinimumLengthAsInt = new int[meta.getValidations().size()];
        data.fieldsMaximumLengthAsInt = new int[meta.getValidations().size()];
        data.minimumValue = new Object[meta.getValidations().size()];
        data.maximumValue = new Object[meta.getValidations().size()];
        data.listValues = new Object[meta.getValidations().size()][];
        data.errorCode = new String[meta.getValidations().size()];
        data.errorDescription = new String[meta.getValidations().size()];
        data.conversionMask = new String[meta.getValidations().size()];
        data.decimalSymbol = new String[meta.getValidations().size()];
        data.groupingSymbol = new String[meta.getValidations().size()];
        data.maximumLength = new String[meta.getValidations().size()];
        data.minimumLength = new String[meta.getValidations().size()];
        data.startString = new String[meta.getValidations().size()];
        data.endString = new String[meta.getValidations().size()];
        data.startStringNotAllowed = new String[meta.getValidations().size()];
        data.endStringNotAllowed = new String[meta.getValidations().size()];
        data.regularExpression = new String[meta.getValidations().size()];
        data.regularExpressionNotAllowed = new String[meta.getValidations().size()];
        data.patternExpected = new Pattern[meta.getValidations().size()];
        data.patternDisallowed = new Pattern[meta.getValidations().size()];
        for (int i = 0; i < meta.getValidations().size(); i++) {
            Validation field = meta.getValidations().get(i);
            try {
                data.constantsMeta[i] = createValueMeta(field.getFieldName(), field.getDataType());
                data.constantsMeta[i].setConversionMask(field.getConversionMask());
                data.constantsMeta[i].setDecimalSymbol(field.getDecimalSymbol());
                data.constantsMeta[i].setGroupingSymbol(field.getGroupingSymbol());
                data.errorCode[i] = environmentSubstitute(Const.NVL(field.getErrorCode(), ""));
                data.errorDescription[i] = environmentSubstitute(Const.NVL(field.getErrorDescription(), ""));
                data.conversionMask[i] = environmentSubstitute(Const.NVL(field.getConversionMask(), ""));
                data.decimalSymbol[i] = environmentSubstitute(Const.NVL(field.getDecimalSymbol(), ""));
                data.groupingSymbol[i] = environmentSubstitute(Const.NVL(field.getGroupingSymbol(), ""));
                data.maximumLength[i] = environmentSubstitute(Const.NVL(field.getMaximumLength(), ""));
                data.minimumLength[i] = environmentSubstitute(Const.NVL(field.getMinimumLength(), ""));
                data.maximumValueAsString[i] = environmentSubstitute(Const.NVL(field.getMaximumValue(), ""));
                data.minimumValueAsString[i] = environmentSubstitute(Const.NVL(field.getMinimumValue(), ""));
                data.startString[i] = environmentSubstitute(Const.NVL(field.getStartString(), ""));
                data.endString[i] = environmentSubstitute(Const.NVL(field.getEndString(), ""));
                data.startStringNotAllowed[i] = environmentSubstitute(Const.NVL(field.getStartStringNotAllowed(), ""));
                data.endStringNotAllowed[i] = environmentSubstitute(Const.NVL(field.getEndStringNotAllowed(), ""));
                data.regularExpression[i] = environmentSubstitute(Const.NVL(field.getRegularExpression(), ""));
                data.regularExpressionNotAllowed[i] = environmentSubstitute(Const.NVL(field.getRegularExpressionNotAllowed(), ""));
                ValueMetaInterface stringMeta = cloneValueMeta(data.constantsMeta[i], ValueMetaInterface.TYPE_STRING);
                data.minimumValue[i] = Utils.isEmpty(data.minimumValueAsString[i]) ? null : data.constantsMeta[i].convertData(stringMeta, data.minimumValueAsString[i]);
                data.maximumValue[i] = Utils.isEmpty(data.maximumValueAsString[i]) ? null : data.constantsMeta[i].convertData(stringMeta, data.maximumValueAsString[i]);
                try {
                    data.fieldsMinimumLengthAsInt[i] = Integer.valueOf(Const.NVL(data.minimumLength[i], "-1"));
                } catch (NumberFormatException nfe) {
                    throw new KettleValueException("Caught a number format exception converting minimum length with value " + data.minimumLength[i] + " to an int.", nfe);
                }
                try {
                    data.fieldsMaximumLengthAsInt[i] = Integer.valueOf(Const.NVL(data.maximumLength[i], "-1"));
                } catch (NumberFormatException nfe) {
                    throw new KettleValueException("Caught a number format exception converting minimum length with value " + data.maximumLength[i] + " to an int.", nfe);
                }
                int listSize = field.getAllowedValues() != null ? field.getAllowedValues().length : 0;
                data.listValues[i] = new Object[listSize];
                for (int s = 0; s < listSize; s++) {
                    data.listValues[i][s] = Utils.isEmpty(field.getAllowedValues()[s]) ? null : data.constantsMeta[i].convertData(stringMeta, environmentSubstitute(field.getAllowedValues()[s]));
                }
            } catch (KettleException e) {
                if (field.getDataType() == ValueMetaInterface.TYPE_NONE) {
                    logError(BaseMessages.getString(PKG, "Validator.Exception.SpecifyDataType"), e);
                } else {
                    logError(BaseMessages.getString(PKG, "Validator.Exception.DataConversionErrorEncountered"), e);
                }
                return false;
            }
            if (!Utils.isEmpty(data.regularExpression[i])) {
                data.patternExpected[i] = Pattern.compile(data.regularExpression[i]);
            }
            if (!Utils.isEmpty(data.regularExpressionNotAllowed[i])) {
                data.patternDisallowed[i] = Pattern.compile(data.regularExpressionNotAllowed[i]);
            }
        }
        return true;
    }
    return false;
}

Example 29 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class TransWebSocketEngineAdapter method getSubSteps.

@SuppressWarnings("unchecked")
private List<StepMetaDataCombi> getSubSteps(Transformation transformation, StepMetaDataCombi combi) {
    HashMap<String, Transformation> config = ((Optional<HashMap<String, Transformation>>) transformation.getConfig(TransMetaConverter.SUB_TRANSFORMATIONS_KEY)).orElse(Maps.newHashMap());
    StepMetaInterface smi = combi.stepMeta.getStepMetaInterface();
    return config.keySet().stream().filter(key -> stepHasDependency(combi, smi, key)).flatMap(key -> opsToSteps(config.get(key)).stream()).collect(Collectors.toList());
}

Example 30 with StepMetaDataCombi

use of org.pentaho.di.trans.step.StepMetaDataCombi in project pentaho-kettle by pentaho.

the class SingleThreadedTransExecutor method dispose.

public void dispose() throws KettleException {
    // 
    for (StepMetaDataCombi combi : trans.getSteps()) {
        combi.step.setOutputDone();
    }
    // 
    for (StepMetaDataCombi combi : steps) {
        combi.step.dispose(combi.meta, combi.data);
        combi.step.markStop();
    }
}