Examples with Hop org.pentaho.di.engine.api.model.Hop used on opensource projects

Example 1 with Hop

use of org.pentaho.di.engine.api.model.Hop in project pentaho-kettle by pentaho.

the class TransMetaConverter method convert.

public static Transformation convert(TransMeta transMeta) {
    final org.pentaho.di.engine.model.Transformation transformation = new org.pentaho.di.engine.model.Transformation(createTransformationId(transMeta));
    try {
        TransMeta copyTransMeta = (TransMeta) transMeta.realClone(false);
        cleanupDisabledHops(copyTransMeta);
        // Turn off lazy conversion for AEL for now
        disableLazyConversion(copyTransMeta);
        resolveStepMetaResources(copyTransMeta);
        copyTransMeta.getSteps().forEach(createOperation(transformation));
        findHops(copyTransMeta, hop -> true).forEach(createHop(transformation));
        transformation.setConfig(TRANS_META_CONF_KEY, copyTransMeta.getXML());
        transformation.setConfig(TRANS_META_NAME_CONF_KEY, Optional.ofNullable(transMeta.getName()).orElse(TRANS_DEFAULT_NAME));
        Map<String, Transformation> subTransformations = copyTransMeta.getResourceDependencies().stream().flatMap(resourceReference -> resourceReference.getEntries().stream()).filter(entry -> ResourceEntry.ResourceType.ACTIONFILE.equals(entry.getResourcetype())).collect(toMap(ResourceEntry::getResource, entry -> {
            try {
                Repository repository = copyTransMeta.getRepository();
                if (repository != null) {
                    Path path = Paths.get(entry.getResource());
                    RepositoryDirectoryInterface directory = repository.findDirectory(path.getParent().toString().replace(File.separator, "/"));
                    return convert(repository.loadTransformation(path.getFileName().toString(), directory, null, true, null));
                }
                return convert(new TransMeta(entry.getResource(), copyTransMeta.getParentVariableSpace()));
            } catch (KettleException e) {
                throw new RuntimeException(e);
            }
        }));
        transformation.setConfig(SUB_TRANSFORMATIONS_KEY, (Serializable) subTransformations);
    } catch (KettleException e) {
        Throwables.propagate(e);
    }
    return transformation;
}

Example 2 with Hop

use of org.pentaho.di.engine.api.model.Hop in project pentaho-kettle by pentaho.

the class TransMetaConverter method removeInactivePaths.

/**
 * Removes steps which cannot be reached using enabled hops. Steps removed along with every input and
 * output hops they have. Downstream steps processed recursively in the same way. Should be invoked with null second arg.
 *
 * @param trans trans object to process
 * @param steps
 */
private static void removeInactivePaths(TransMeta trans, List<StepMeta> steps) {
    if (steps == null) {
        List<TransHopMeta> disabledHops = findHops(trans, hop -> !hop.isEnabled());
        List<StepMeta> disabledSteps = disabledHops.stream().map(hop -> hop.getToStep()).collect(Collectors.toList());
        removeInactivePaths(trans, disabledSteps);
    } else {
        for (StepMeta step : steps) {
            List<TransHopMeta> enabledInHops = findHops(trans, hop -> hop.getToStep().equals(step) && hop.isEnabled());
            List<TransHopMeta> disabledInHops = findHops(trans, hop -> hop.getToStep().equals(step) && !hop.isEnabled());
            if (enabledInHops.size() == 0) {
                List<StepMeta> nextSteps = findHops(trans, hop -> hop.getFromStep().equals(step)).stream().map(TransHopMeta::getToStep).collect(Collectors.toList());
                findHops(trans, hop -> hop.getToStep().equals(step) || hop.getFromStep().equals(step)).forEach(trans::removeTransHop);
                trans.getSteps().remove(step);
                removeInactivePaths(trans, nextSteps);
            } else {
                disabledInHops.forEach(trans::removeTransHop);
            }
        }
    }
}

Example 3 with Hop

use of org.pentaho.di.engine.api.model.Hop in project pentaho-kettle by pentaho.

the class TransMetaConverterTest method errorHops.

@Test
public void errorHops() throws Exception {
    TransMeta meta = new TransMeta();
    meta.setFilename("fileName");
    StepMeta from = new StepMeta("step1", stepMetaInterface);
    meta.addStep(from);
    StepMeta to = new StepMeta("step2", stepMetaInterface);
    meta.addStep(to);
    meta.addTransHop(new TransHopMeta(from, to));
    StepMeta error = new StepMeta("errorHandler", stepMetaInterface);
    meta.addStep(error);
    TransHopMeta errorHop = new TransHopMeta(from, error);
    errorHop.setErrorHop(true);
    meta.addTransHop(errorHop);
    Transformation trans = TransMetaConverter.convert(meta);
    Map<String, List<Hop>> hops = trans.getHops().stream().collect(Collectors.groupingBy(Hop::getType));
    List<Hop> normalHops = hops.get(Hop.TYPE_NORMAL);
    assertThat(normalHops.size(), is(1));
    assertThat(normalHops.get(0).getTo().getId(), is("step2"));
    List<Hop> errorHops = hops.get(Hop.TYPE_ERROR);
    assertThat(errorHops.size(), is(1));
    assertThat(errorHops.get(0).getTo().getId(), is("errorHandler"));
    assertThat(hops.values().stream().flatMap(List::stream).map(Hop::getFrom).map(Operation::getId).collect(Collectors.toList()), everyItem(equalTo("step1")));
}

Example 4 with Hop

use of org.pentaho.di.engine.api.model.Hop in project pentaho-kettle by pentaho.

the class TransMetaConverterTest method testDisabledHops.

@Test
public void testDisabledHops() {
    TransMeta trans = new TransMeta();
    StepMeta start = new StepMeta("Start", stepMetaInterface);
    trans.addStep(start);
    StepMeta withEnabledHop = new StepMeta("WithEnabledHop", stepMetaInterface);
    trans.addStep(withEnabledHop);
    StepMeta withDisabledHop = new StepMeta("WithDisabledHop", stepMetaInterface);
    trans.addStep(withDisabledHop);
    StepMeta shouldStay = new StepMeta("ShouldStay", stepMetaInterface);
    trans.addStep(shouldStay);
    StepMeta shouldNotStay = new StepMeta("ShouldNotStay", stepMetaInterface);
    trans.addStep(shouldNotStay);
    StepMeta withEnabledAndDisabledHops = new StepMeta("WithEnabledAndDisabledHops", stepMetaInterface);
    trans.addStep(withEnabledAndDisabledHops);
    StepMeta afterEnabledDisabled = new StepMeta("AfterEnabledDisabled", stepMetaInterface);
    trans.addStep(afterEnabledDisabled);
    trans.addTransHop(new TransHopMeta(start, withEnabledHop));
    trans.addTransHop(new TransHopMeta(start, withDisabledHop, false));
    trans.addTransHop(new TransHopMeta(withEnabledHop, shouldStay));
    trans.addTransHop(new TransHopMeta(withDisabledHop, shouldStay));
    trans.addTransHop(new TransHopMeta(withDisabledHop, shouldNotStay));
    trans.addTransHop(new TransHopMeta(start, withEnabledAndDisabledHops));
    trans.addTransHop(new TransHopMeta(withEnabledHop, withEnabledAndDisabledHops, false));
    trans.addTransHop(new TransHopMeta(withEnabledAndDisabledHops, afterEnabledDisabled));
    Transformation transformation = TransMetaConverter.convert(trans);
    List<String> steps = transformation.getOperations().stream().map(op -> op.getId()).collect(Collectors.toList());
    assertThat("Only 5 ops should exist", steps.size(), is(5));
    assertThat(steps, hasItems("Start", "WithEnabledHop", "ShouldStay", "WithEnabledAndDisabledHops", "AfterEnabledDisabled"));
    List<String> hops = transformation.getHops().stream().map(hop -> hop.getId()).collect(Collectors.toList());
    assertThat("Only 4 hops should exist", hops.size(), is(4));
    assertThat(hops, hasItems("Start -> WithEnabledHop", "WithEnabledHop -> ShouldStay", "Start -> WithEnabledAndDisabledHops", "WithEnabledAndDisabledHops -> AfterEnabledDisabled"));
}

Example 5 with Hop

use of org.pentaho.di.engine.api.model.Hop in project pentaho-kettle by pentaho.

the class TransMetaConverter method removeDisabledInputs.

/**
 * Removes input steps having only disabled output hops so they will not be executed.
 * @param transMeta transMeta to process
 */
private static void removeDisabledInputs(TransMeta transMeta) {
    List<StepMeta> unusedInputs = findHops(transMeta, hop -> !hop.isEnabled()).stream().map(hop -> hop.getFromStep()).filter(step -> isUnusedInput(transMeta, step)).collect(Collectors.toList());
    for (StepMeta unusedInput : unusedInputs) {
        List<TransHopMeta> outHops = transMeta.findAllTransHopFrom(unusedInput);
        List<StepMeta> subsequentSteps = outHops.stream().map(hop -> hop.getToStep()).collect(Collectors.toList());
        outHops.forEach(transMeta::removeTransHop);
        transMeta.getSteps().remove(unusedInput);
        removeInactivePaths(transMeta, subsequentSteps);
    }
}