Examples with IEngineComponent org.apache.hop.pipeline.engine.IEngineComponent used on opensource projects

Example 1 with IEngineComponent

use of org.apache.hop.pipeline.engine.IEngineComponent in project hop by apache.

the class InjectDataSetIntoTransformExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    if (!(pipeline instanceof LocalPipelineEngine)) {
        throw new HopPluginException("Unit tests can only run using a local pipeline engine type");
    }
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean dataSetEnabled = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_RUN_UNIT_TEST));
    if (log.isDetailed()) {
        log.logDetailed("Data Set enabled? " + dataSetEnabled);
    }
    if (!dataSetEnabled) {
        return;
    }
    String unitTestName = pipeline.getVariable(DataSetConst.VAR_UNIT_TEST_NAME);
    if (log.isDetailed()) {
        log.logDetailed("Unit test name: " + unitTestName);
    }
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        // 
        if (StringUtil.isEmpty(unitTestName)) {
            return;
        }
        PipelineUnitTest unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName);
        if (unitTest == null) {
            if (log.isDetailed()) {
                log.logDetailed("Unit test '" + unitTestName + "' could not be found");
            }
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            String transformName = transformMeta.getName();
            PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformName);
            if (inputLocation != null && StringUtils.isNotEmpty(inputLocation.getDataSetName())) {
                String inputDataSetName = inputLocation.getDataSetName();
                log.logDetailed("Data Set location found for transform '" + transformName + "' and data set  " + inputDataSetName);
                // We need to inject data from the data set with the specified name into the transform
                // 
                injectDataSetIntoTransform((LocalPipelineEngine) pipeline, inputDataSetName, metadataProvider, transformMeta, inputLocation);
            }
            // How about capturing rows for golden data review?
            // 
            PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
            if (goldenLocation != null) {
                String goldenDataSetName = goldenLocation.getDataSetName();
                if (!StringUtil.isEmpty(goldenDataSetName)) {
                    log.logDetailed("Capturing rows for validation at pipeline end, transform='" + transformMeta.getName() + "', golden set '" + goldenDataSetName);
                    final RowCollection rowCollection = new RowCollection();
                    // Create a row collection map if it's missing...
                    // 
                    @SuppressWarnings("unchecked") Map<String, RowCollection> collectionMap = (Map<String, RowCollection>) pipeline.getExtensionDataMap().get(DataSetConst.ROW_COLLECTION_MAP);
                    if (collectionMap == null) {
                        collectionMap = new HashMap<>();
                        pipeline.getExtensionDataMap().put(DataSetConst.ROW_COLLECTION_MAP, collectionMap);
                    }
                    // Keep the map for safe keeping...
                    // 
                    collectionMap.put(transformMeta.getName(), rowCollection);
                    // We'll capture the rows from this one and then evaluate them after execution...
                    // 
                    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
                    component.addRowListener(new RowAdapter() {

                        @Override
                        public void rowReadEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            if (rowCollection.getRowMeta() == null) {
                                rowCollection.setRowMeta(rowMeta);
                            }
                            rowCollection.getRows().add(row);
                        }
                    });
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to inject data set rows", e);
    }
}

Example 2 with IEngineComponent

use of org.apache.hop.pipeline.engine.IEngineComponent in project hop by apache.

the class WriteToDataSetExtensionPoint method passTransformRowsToDataSet.

private void passTransformRowsToDataSet(final IPipelineEngine<PipelineMeta> pipeline, final PipelineMeta pipelineMeta, final TransformMeta transformMeta, final List<SourceToTargetMapping> mappings, final DataSet dataSet) throws HopException {
    // This is the transform to inject into the specified data set
    // 
    final IRowMeta setRowMeta = dataSet.getSetRowMeta();
    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
    final List<Object[]> transformsForDbRows = new ArrayList<>();
    component.addRowListener(new RowAdapter() {

        @Override
        public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
            Object[] transformForDbRow = RowDataUtil.allocateRowData(setRowMeta.size());
            for (SourceToTargetMapping mapping : mappings) {
                transformForDbRow[mapping.getTargetPosition()] = row[mapping.getSourcePosition()];
            }
            transformsForDbRows.add(transformForDbRow);
        }
    });
    // At the end of the pipeline, write it...
    // 
    pipeline.addExecutionFinishedListener(engine -> DataSetCsvUtil.writeDataSetData(pipeline, dataSet, setRowMeta, transformsForDbRows));
}

Example 3 with IEngineComponent

use of org.apache.hop.pipeline.engine.IEngineComponent in project hop by apache.

the class PipelinePainter method drawTransformStatusIndicator.

private void drawTransformStatusIndicator(TransformMeta transformMeta) throws HopException {
    if (transformMeta == null) {
        return;
    }
    // draw status indicator
    if (pipeline != null) {
        Point pt = transformMeta.getLocation();
        if (pt == null) {
            pt = new Point(50, 50);
        }
        Point screen = real2screen(pt.x, pt.y);
        int x = screen.x;
        int y = screen.y;
        if (pipeline != null) {
            List<IEngineComponent> transforms = pipeline.getComponentCopies(transformMeta.getName());
            for (IEngineComponent transform : transforms) {
                String transformStatus = transform.getStatusDescription();
                if (transformStatus != null && transformStatus.equalsIgnoreCase(EngineComponent.ComponentExecutionStatus.STATUS_FINISHED.getDescription())) {
                    gc.drawImage(EImage.SUCCESS, (x + iconSize) - (miniIconSize / 2) + 1, y - (miniIconSize / 2) - 1, magnification);
                }
            }
        }
    }
}

Example 4 with IEngineComponent

use of org.apache.hop.pipeline.engine.IEngineComponent in project hop by apache.

the class PipelinePainter method checkDrawSlowTransformIndicator.

private void checkDrawSlowTransformIndicator(TransformMeta transformMeta) {
    if (transformMeta == null) {
        return;
    }
    // draw optional performance indicator
    if (pipeline != null) {
        Point pt = transformMeta.getLocation();
        if (pt == null) {
            pt = new Point(50, 50);
        }
        Point screen = real2screen(pt.x, pt.y);
        int x = screen.x;
        int y = screen.y;
        List<IEngineComponent> components = pipeline.getComponents();
        for (IEngineComponent component : components) {
            if (component.getName().equals(transformMeta.getName())) {
                if (component.isRunning()) {
                    Long inputRowsValue = component.getInputBufferSize();
                    Long outputRowsValue = component.getOutputBufferSize();
                    if (inputRowsValue != null && outputRowsValue != null) {
                        long inputRows = inputRowsValue.longValue();
                        long outputRows = outputRowsValue.longValue();
                        // if the transform can't keep up with its input, mark it by drawing an animation
                        boolean isSlow = inputRows * 0.85 > outputRows;
                        if (isSlow) {
                            gc.setLineWidth(lineWidth + 1);
                            if (System.currentTimeMillis() % 2000 > 1000) {
                                gc.setForeground(EColor.BACKGROUND);
                                gc.setLineStyle(ELineStyle.SOLID);
                                gc.drawRectangle(x + 1, y + 1, iconSize - 2, iconSize - 2);
                                gc.setForeground(EColor.DARKGRAY);
                                gc.setLineStyle(ELineStyle.DOT);
                                gc.drawRectangle(x + 1, y + 1, iconSize - 2, iconSize - 2);
                            } else {
                                gc.setForeground(EColor.DARKGRAY);
                                gc.setLineStyle(ELineStyle.SOLID);
                                gc.drawRectangle(x + 1, y + 1, iconSize - 2, iconSize - 2);
                                gc.setForeground(EColor.BACKGROUND);
                                gc.setLineStyle(ELineStyle.DOT);
                                gc.drawRectangle(x + 1, y + 1, iconSize - 2, iconSize - 2);
                            }
                        }
                    }
                }
                gc.setLineStyle(ELineStyle.SOLID);
            }
        }
    }
}

Example 5 with IEngineComponent

use of org.apache.hop.pipeline.engine.IEngineComponent in project hop by apache.

the class PipelinePainter method drawTransformPerformanceTable.

private void drawTransformPerformanceTable(TransformMeta transformMeta) {
    if (transformMeta == null) {
        return;
    }
    // draw optional performance indicator
    if (pipeline != null) {
        Point pt = transformMeta.getLocation();
        if (pt == null) {
            pt = new Point(50, 50);
        }
        Point screen = real2screen(pt.x, pt.y);
        int x = screen.x;
        int y = screen.y;
        List<IEngineComponent> transforms = pipeline.getComponentCopies(transformMeta.getName());
        // draw mouse over performance indicator
        if (pipeline.isRunning()) {
            if (transformMeta.isSelected()) {
                // determine popup dimensions up front
                int popupX = x;
                int popupY = y;
                int popupWidth = 0;
                int popupHeight = 1;
                gc.setFont(EFont.SMALL);
                Point p = gc.textExtent("0000000000");
                int colWidth = p.x + MINI_ICON_MARGIN;
                int rowHeight = p.y + MINI_ICON_MARGIN;
                int titleWidth = 0;
                // calculate max title width to get the colum with
                String[] titles = PipelinePainter.getPeekTitles();
                for (String title : titles) {
                    Point titleExtent = gc.textExtent(title);
                    titleWidth = Math.max(titleExtent.x + MINI_ICON_MARGIN, titleWidth);
                    popupHeight += titleExtent.y + MINI_ICON_MARGIN;
                }
                popupWidth = titleWidth + 2 * MINI_ICON_MARGIN;
                // determine total popup width
                popupWidth += transforms.size() * colWidth;
                // determine popup position
                popupX = popupX + (iconSize - popupWidth) / 2;
                popupY = popupY - popupHeight - MINI_ICON_MARGIN;
                // draw the frame
                gc.setForeground(EColor.DARKGRAY);
                gc.setBackground(EColor.LIGHTGRAY);
                gc.setLineWidth(1);
                gc.fillRoundRectangle(popupX, popupY, popupWidth, popupHeight, 7, 7);
                // draw the title columns
                gc.setBackground(EColor.LIGHTGRAY);
                gc.drawRoundRectangle(popupX, popupY, popupWidth, popupHeight, 7, 7);
                for (int i = 0, barY = popupY; i < titles.length; i++) {
                    if (i % 2 == 1) {
                        gc.setBackground(EColor.BACKGROUND);
                    } else {
                        gc.setBackground(EColor.LIGHTGRAY);
                    }
                    gc.fillRoundRectangle(popupX + 1, barY + 1, popupWidth - 2, rowHeight, 7, 7);
                    barY += rowHeight;
                }
                // draw the header column
                int rowY = popupY + MINI_ICON_MARGIN;
                int rowX = popupX + MINI_ICON_MARGIN;
                gc.setForeground(EColor.BLACK);
                gc.setBackground(EColor.BACKGROUND);
                for (int i = 0; i < titles.length; i++) {
                    if (i % 2 == 1) {
                        gc.setBackground(EColor.BACKGROUND);
                    } else {
                        gc.setBackground(EColor.LIGHTGRAY);
                    }
                    gc.drawText(titles[i], rowX, rowY);
                    rowY += rowHeight;
                }
                // draw the values for each copy of the transform
                gc.setBackground(EColor.LIGHTGRAY);
                rowX += titleWidth;
                for (IEngineComponent transform : transforms) {
                    rowX += colWidth;
                    rowY = popupY + MINI_ICON_MARGIN;
                    String[] fields = getPeekFields(transform);
                    for (int i = 0; i < fields.length; i++) {
                        if (i % 2 == 1) {
                            gc.setBackground(EColor.BACKGROUND);
                        } else {
                            gc.setBackground(EColor.LIGHTGRAY);
                        }
                        drawTextRightAligned(fields[i], rowX, rowY);
                        rowY += rowHeight;
                    }
                }
            }
        }
    }
}