Examples with SampleType org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration.SampleType used on opensource projects

Example 1 with SampleType

use of org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration.SampleType in project hop by apache.

the class HopGuiPipelineGraph method addRowsSamplerToPipeline.

private void addRowsSamplerToPipeline(IPipelineEngine<PipelineMeta> pipeline) {
    if (!(pipeline.getPipelineRunConfiguration().getEngineRunConfiguration() instanceof LocalPipelineRunConfiguration)) {
        return;
    }
    LocalPipelineRunConfiguration lprConfig = (LocalPipelineRunConfiguration) pipeline.getPipelineRunConfiguration().getEngineRunConfiguration();
    if (StringUtils.isEmpty(lprConfig.getSampleTypeInGui())) {
        return;
    }
    try {
        SampleType sampleType = SampleType.valueOf(lprConfig.getSampleTypeInGui());
        if (sampleType == SampleType.None) {
            return;
        }
        final int sampleSize = Const.toInt(pipeline.resolve(lprConfig.getSampleSize()), 100);
        if (sampleSize <= 0) {
            return;
        }
        outputRowsMap = new HashMap<>();
        final Random random = new Random();
        for (final String transformName : pipelineMeta.getTransformNames()) {
            IEngineComponent component = pipeline.findComponent(transformName, 0);
            if (component != null) {
                component.addRowListener(new RowAdapter() {

                    int nrRows = 0;

                    @Override
                    public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                        RowBuffer rowBuffer = outputRowsMap.get(transformName);
                        if (rowBuffer == null) {
                            rowBuffer = new RowBuffer(rowMeta);
                            outputRowsMap.put(transformName, rowBuffer);
                            // 
                            if (sampleType == SampleType.Last) {
                                rowBuffer.setBuffer(Collections.synchronizedList(new LinkedList<>()));
                            } else {
                                rowBuffer.setBuffer(Collections.synchronizedList(new ArrayList<>()));
                            }
                        }
                        // 
                        if (sampleType != SampleType.None) {
                            try {
                                row = rowMeta.cloneRow(row);
                            } catch (HopValueException e) {
                                throw new HopTransformException("Error copying row for preview purposes", e);
                            }
                        }
                        switch(sampleType) {
                            case First:
                                {
                                    if (rowBuffer.size() < sampleSize) {
                                        rowBuffer.addRow(row);
                                    }
                                }
                                break;
                            case Last:
                                {
                                    rowBuffer.addRow(0, row);
                                    if (rowBuffer.size() > sampleSize) {
                                        rowBuffer.removeRow(rowBuffer.size() - 1);
                                    }
                                }
                                break;
                            case Random:
                                {
                                    // Reservoir sampling
                                    // 
                                    nrRows++;
                                    if (rowBuffer.size() < sampleSize) {
                                        rowBuffer.addRow(row);
                                    } else {
                                        int randomIndex = random.nextInt(nrRows);
                                        if (randomIndex < sampleSize) {
                                            rowBuffer.setRow(randomIndex, row);
                                        }
                                    }
                                }
                                break;
                        }
                    }
                });
            }
        }
    } catch (Exception e) {
    // Ignore : simply not recognized or empty
    }
}

Example 2 with SampleType

use of org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration.SampleType in project hop by apache.

the class HopGuiPipelineGraph method showTransformOutputData.

private boolean showTransformOutputData(TransformMeta dataTransformMeta, RowBuffer rowBuffer) {
    if (rowBuffer != null) {
        synchronized (rowBuffer.getBuffer()) {
            if (!rowBuffer.isEmpty()) {
                try {
                    String title = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.Header", dataTransformMeta.getName());
                    String message = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.OutputRows.Text", dataTransformMeta.getName());
                    String prefix = "";
                    if (pipeline != null && pipeline.getPipelineRunConfiguration() != null) {
                        PipelineRunConfiguration pipelineRunConfiguration = pipeline.getPipelineRunConfiguration();
                        if (pipelineRunConfiguration.getEngineRunConfiguration() instanceof LocalPipelineRunConfiguration) {
                            String sampleTypeInGui = ((LocalPipelineRunConfiguration) pipelineRunConfiguration.getEngineRunConfiguration()).getSampleTypeInGui();
                            if (StringUtils.isNotEmpty(sampleTypeInGui)) {
                                try {
                                    SampleType sampleType = SampleType.valueOf(sampleTypeInGui);
                                    switch(sampleType) {
                                        case None:
                                            break;
                                        case First:
                                            prefix = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.First.Text");
                                            break;
                                        case Last:
                                            prefix = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.Last.Text");
                                            break;
                                        case Random:
                                            prefix += BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.Random.Text");
                                            ;
                                            break;
                                        default:
                                            break;
                                    }
                                } catch (Exception ex) {
                                    LogChannel.UI.logError("Unknown sample type: " + sampleTypeInGui);
                                }
                            }
                        }
                    }
                    PreviewRowsDialog previewRowsDialog = new PreviewRowsDialog(hopGui.getShell(), variables, SWT.NONE, dataTransformMeta.getName(), rowBuffer.getRowMeta(), rowBuffer.getBuffer());
                    previewRowsDialog.setTitleMessage(title, prefix + message);
                    previewRowsDialog.open();
                } catch (Exception ex) {
                    new ErrorDialog(hopGui.getShell(), "Error", "Error showing preview dialog", ex);
                }
            }
        }
        return true;
    }
    return false;
}