Examples with RowAdapter org.apache.hop.pipeline.transform.RowAdapter used on opensource projects

Example 6 with RowAdapter

use of org.apache.hop.pipeline.transform.RowAdapter in project hop by apache.

the class CalculatorUnitTest method assertRoundGeneral.

public void assertRoundGeneral(final Object expectedResult, final CalculationType calcFunction, final Number value, final Long precision, final Long roundingMode, final int valueDataType, final int functionDataType) throws HopException {
    final String msg = getHopTypeName(valueDataType) + "->" + getHopTypeName(functionDataType) + " ";
    final RowMeta inputRowMeta = new RowMeta();
    final List<Object> inputValues = new ArrayList<>(3);
    final String fieldValue = "Value";
    final IValueMeta valueMeta;
    switch(valueDataType) {
        case IValueMeta.TYPE_BIGNUMBER:
            valueMeta = new ValueMetaBigNumber(fieldValue);
            break;
        case IValueMeta.TYPE_NUMBER:
            valueMeta = new ValueMetaNumber(fieldValue);
            break;
        case IValueMeta.TYPE_INTEGER:
            valueMeta = new ValueMetaInteger(fieldValue);
            break;
        default:
            throw new IllegalArgumentException(msg + "Unexpected value dataType: " + value.getClass().getName() + ". Long, Double or BigDecimal expected.");
    }
    inputRowMeta.addValueMeta(valueMeta);
    inputValues.add(value);
    final String fieldPrecision;
    final ValueMetaInteger precisionMeta;
    if (precision == null) {
        fieldPrecision = null;
        precisionMeta = null;
    } else {
        fieldPrecision = "Precision";
        precisionMeta = new ValueMetaInteger(fieldPrecision);
        inputRowMeta.addValueMeta(precisionMeta);
        inputValues.add(precision);
    }
    final String fieldRoundingMode;
    final ValueMetaInteger roundingModeMeta;
    if (roundingMode == null) {
        fieldRoundingMode = null;
        roundingModeMeta = null;
    } else {
        fieldRoundingMode = "RoundingMode";
        roundingModeMeta = new ValueMetaInteger(fieldRoundingMode);
        inputRowMeta.addValueMeta(roundingModeMeta);
        inputValues.add(roundingMode);
    }
    IRowSet inputRowSet = smh.getMockInputRowSet(inputValues.toArray());
    inputRowSet.setRowMeta(inputRowMeta);
    final String fieldA = inputRowMeta.size() > 0 ? inputRowMeta.getValueMetaList().get(0).getName() : null;
    final String fieldB = inputRowMeta.size() > 1 ? inputRowMeta.getValueMetaList().get(1).getName() : null;
    final String fieldC = inputRowMeta.size() > 2 ? inputRowMeta.getValueMetaList().get(2).getName() : null;
    final int resultDataType = functionDataType;
    final String fieldResult = "test";
    final int expectedResultRowSize = inputRowMeta.size() + 1;
    CalculatorMeta meta = new CalculatorMeta();
    meta.getFunctions().add(new CalculatorMetaFunction(fieldResult, calcFunction, fieldA, fieldB, fieldC, ValueMetaFactory.getValueMetaName(resultDataType), 2, 0, "", "", "", "", false));
    CalculatorData data = new CalculatorData();
    Calculator calculator = new Calculator(smh.transformMeta, meta, data, 0, smh.pipelineMeta, smh.pipeline);
    calculator.addRowSetToInputRowSets(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init();
    // Verify output
    try {
        calculator.addRowListener(new RowAdapter() {

            @Override
            public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                assertEquals(msg + " resultRowSize", expectedResultRowSize, rowMeta.size());
                final int fieldResultIndex = rowMeta.size() - 1;
                assertEquals(msg + " fieldResult", fieldResult, rowMeta.getValueMeta(fieldResultIndex).getName());
                assertEquals(msg, expectedResult, row[fieldResultIndex]);
            }
        });
        calculator.processRow();
    } catch (HopException ke) {
        ke.printStackTrace();
        fail(msg + ke.getMessage());
    }
}

Example 7 with RowAdapter

use of org.apache.hop.pipeline.transform.RowAdapter in project hop by apache.

the class WebServiceServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "WebServiceServlet.Log.WebServiceRequested"));
    }
    IVariables variables = pipelineMap.getHopServerConfig().getVariables();
    IHopMetadataProvider metadataProvider = pipelineMap.getHopServerConfig().getMetadataProvider();
    String webServiceName = request.getParameter("service");
    if (StringUtils.isEmpty(webServiceName)) {
        throw new ServletException("Please specify a service parameter pointing to the name of the web service object");
    }
    try {
        IHopMetadataSerializer<WebService> serializer = metadataProvider.getSerializer(WebService.class);
        WebService webService = serializer.load(webServiceName);
        if (webService == null) {
            throw new HopException("Unable to find web service '" + webServiceName + "'.  You can set the metadata_folder in the Hop server XML configuration");
        }
        if (!webService.isEnabled()) {
            throw new HopException("Web service '" + webServiceName + "' is disabled.");
        }
        String filename = variables.resolve(webService.getFilename());
        String transformName = variables.resolve(webService.getTransformName());
        String fieldName = variables.resolve(webService.getFieldName());
        String contentType = variables.resolve(webService.getContentType());
        if (StringUtils.isEmpty(contentType)) {
            response.setContentType("text/plain");
        } else {
            response.setContentType(contentType);
        }
        response.setCharacterEncoding(Const.XML_ENCODING);
        String serverObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        servletLoggingObject.setContainerObjectId(serverObjectId);
        // Load and start the pipeline
        // Output the data to the response output stream...
        // 
        PipelineMeta pipelineMeta = new PipelineMeta(filename, metadataProvider, true, variables);
        LocalPipelineEngine pipeline = new LocalPipelineEngine(pipelineMeta, variables, servletLoggingObject);
        pipeline.setContainerId(serverObjectId);
        // Set all the other parameters as variables/parameters...
        // 
        String[] pipelineParameters = pipelineMeta.listParameters();
        pipeline.copyParametersFromDefinitions(pipelineMeta);
        for (String requestParameter : request.getParameterMap().keySet()) {
            if ("service".equals(requestParameter)) {
                continue;
            }
            String requestParameterValue = request.getParameter(requestParameter);
            if (Const.indexOfString(requestParameter, pipelineParameters) < 0) {
                pipeline.setVariable(requestParameter, Const.NVL(requestParameterValue, ""));
            } else {
                pipeline.setParameterValue(requestParameter, Const.NVL(requestParameterValue, ""));
            }
        }
        pipeline.activateParameters(pipeline);
        // 
        if (webService.isListingStatus()) {
            PipelineExecutionConfiguration pipelineExecutionConfiguration = new PipelineExecutionConfiguration();
            PipelineConfiguration pipelineConfiguration = new PipelineConfiguration(pipelineMeta, pipelineExecutionConfiguration, new SerializableMetadataProvider(metadataProvider));
            getPipelineMap().addPipeline(pipelineMeta.getName(), serverObjectId, pipeline, pipelineConfiguration);
        }
        // Allocate the threads...
        pipeline.prepareExecution();
        final OutputStream outputStream = response.getOutputStream();
        // Add the row listener to the transform/field...
        // TODO: add to all copies
        // 
        IEngineComponent component = pipeline.findComponent(transformName, 0);
        component.addRowListener(new RowAdapter() {

            @Override
            public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                try {
                    String outputString = rowMeta.getString(row, fieldName, "");
                    outputStream.write(outputString.getBytes(StandardCharsets.UTF_8));
                    outputStream.flush();
                } catch (HopValueException e) {
                    throw new HopTransformException("Error getting output field '" + fieldName + " from row: " + rowMeta.toStringMeta(), e);
                } catch (IOException e) {
                    throw new HopTransformException("Error writing output of '" + fieldName + "'", e);
                }
            }
        });
        pipeline.startThreads();
        pipeline.waitUntilFinished();
        response.setStatus(HttpServletResponse.SC_OK);
    } catch (Exception e) {
        throw new ServletException("Error producing web service output", e);
    }
}

Example 8 with RowAdapter

use of org.apache.hop.pipeline.transform.RowAdapter in project hop by apache.

the class CsvProcessRowInParallelTest method processRows.

/**
 * So as not to heap up list of taken parameters, we are passing combi, but we expect to see
 * CsvInput class instances in it's content.
 */
private int processRows(TransformMetaDataCombi combi) throws Exception {
    CsvInput csvInput = (CsvInput) combi.transform;
    CsvInputData transformData = (CsvInputData) combi.data;
    CsvInputMeta transformMeta = (CsvInputMeta) combi.meta;
    final int[] writtenRows = { 0 };
    csvInput.addRowListener(new RowAdapter() {

        @Override
        public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
            writtenRows[0]++;
        }
    });
    boolean haveRowsToRead;
    do {
        haveRowsToRead = !csvInput.processRow();
    } while (!haveRowsToRead);
    csvInput.dispose();
    return writtenRows[0];
}

Example 9 with RowAdapter

use of org.apache.hop.pipeline.transform.RowAdapter in project hop by apache.

the class AzureListener method processRow.

@Override
public boolean processRow() throws HopException {
    // This thing is executed only once, rows are processed in the event processor later
    // 
    // Get the output fields starting from nothing
    // 
    data.outputRowMeta = new RowMeta();
    meta.getRegularRowMeta(data.outputRowMeta, this);
    data.outputField = resolve(meta.getOutputField());
    data.partitionIdField = resolve(meta.getPartitionIdField());
    data.offsetField = resolve(meta.getOffsetField());
    data.sequenceNumberField = resolve(meta.getSequenceNumberField());
    data.hostField = resolve(meta.getHostField());
    data.enqueuedTimeField = resolve(meta.getEnqueuedTimeField());
    String namespace = resolve(meta.getNamespace());
    String eventHubName = resolve(meta.getEventHubName());
    String sasKeyName = resolve(meta.getSasKeyName());
    String sasKey = resolve(meta.getSasKey());
    String consumerGroupName = resolve(meta.getConsumerGroupName());
    String storageContainerName = resolve(meta.getStorageContainerName());
    String storageConnectionString = resolve(meta.getStorageConnectionString());
    String batchTransformationFile = resolve(meta.getBatchPipeline());
    String batchInputTransform = resolve(meta.getBatchInputTransform());
    String batchOutputTransform = resolve(meta.getBatchOutputTransform());
    // 
    if (StringUtils.isNotEmpty(batchTransformationFile) && StringUtils.isNotEmpty(batchInputTransform)) {
        logBasic("Passing rows to a batching transformation running single threaded : " + batchTransformationFile);
        data.stt = true;
        data.sttMaxWaitTime = Const.toLong(resolve(meta.getBatchMaxWaitTime()), -1L);
        data.sttPipelineMeta = meta.loadBatchPipelineMeta(meta, metadataProvider, this);
        data.sttPipelineMeta.setPipelineType(PipelineMeta.PipelineType.SingleThreaded);
        data.sttPipeline = new LocalPipelineEngine(data.sttPipelineMeta, this, this);
        data.sttPipeline.setParent(getPipeline());
        data.sttPipeline.setParentPipeline(getPipeline());
        // Leave a trace for Spoon...
        // 
        getPipeline().addActiveSubPipeline(getTransformName(), data.sttPipeline);
        data.sttPipeline.prepareExecution();
        data.sttRowProducer = data.sttPipeline.addRowProducer(batchInputTransform, 0);
        if (StringUtils.isNotEmpty(batchOutputTransform)) {
            ITransform outputTransform = data.sttPipeline.findRunThread(batchOutputTransform);
            if (outputTransform == null) {
                throw new HopTransformException("Unable to find output transform '" + batchOutputTransform + "'in batch pipeline");
            }
            outputTransform.addRowListener(new RowAdapter() {

                @Override
                public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                    AzureListener.this.putRow(rowMeta, row);
                }
            });
        }
        data.sttPipeline.startThreads();
        data.sttExecutor = new SingleThreadedPipelineExecutor(data.sttPipeline);
        boolean ok = data.sttExecutor.init();
        if (!ok) {
            logError("Initializing batch transformation failed");
            stopAll();
            setErrors(1);
            return false;
        }
    } else {
        data.stt = false;
    }
    log.logDetailed("Creating connection string builder");
    data.connectionStringBuilder = new ConnectionStringBuilder().setNamespaceName(namespace).setEventHubName(eventHubName).setSasKeyName(sasKeyName).setSasKey(sasKey);
    log.logDetailed("Opening new executor service");
    data.executorService = Executors.newSingleThreadScheduledExecutor();
    log.logDetailed("Creating event hub client");
    try {
        data.eventHubClient = EventHubClient.createFromConnectionStringSync(data.connectionStringBuilder.toString(), data.executorService);
    } catch (Exception e) {
        throw new HopTransformException("Unable to create event hub client", e);
    }
    EventProcessorHost host;
    try {
        host = EventProcessorHost.EventProcessorHostBuilder.newBuilder(EventProcessorHost.createHostName("HopHost"), consumerGroupName).useAzureStorageCheckpointLeaseManager(storageConnectionString, storageContainerName, "hop").useEventHubConnectionString(data.connectionStringBuilder.toString()).build();
    } catch (Exception e) {
        throw new HopException("Unable to set up events host processor", e);
    }
    log.logDetailed("Set up events host named " + host.getHostName());
    EventProcessorOptions options = new EventProcessorOptions();
    options.setExceptionNotification(new AzureListenerErrorNotificationHandler(AzureListener.this));
    if (!StringUtils.isNotEmpty(meta.getBatchSize())) {
        options.setMaxBatchSize(Const.toInt(resolve(meta.getBatchSize()), 100));
    }
    if (!StringUtils.isNotEmpty(meta.getPrefetchSize())) {
        options.setPrefetchCount(Const.toInt(resolve(meta.getPrefetchSize()), 100));
    }
    data.executorService = Executors.newSingleThreadScheduledExecutor();
    try {
        data.eventHubClient = EventHubClient.createFromConnectionStringSync(data.connectionStringBuilder.toString(), data.executorService);
    } catch (Exception e) {
        throw new HopTransformException("Unable to create event hub client", e);
    }
    // Create our event processor which is going to actually send rows to the batch transformation
    // (or not)
    // and get rows from an optional output Transform.
    // 
    final AzureListenerEventProcessor eventProcessor = new AzureListenerEventProcessor(AzureListener.this, data, data.batchSize);
    // 
    if (data.stt && data.sttMaxWaitTime > 0) {
        // Add a timer to check every max wait time to see whether or not we have to do an
        // iteration...
        // 
        logBasic("Checking for stalled rows every 100ms to see if we exceed the maximum wait time: " + data.sttMaxWaitTime);
        try {
            Timer timer = new Timer();
            TimerTask timerTask = new TimerTask() {

                @Override
                public void run() {
                    // 
                    if (eventProcessor.getLastIterationTime() > 0) {
                        if (eventProcessor.getPassedRowsCount() > 0) {
                            long now = System.currentTimeMillis();
                            long diff = now - eventProcessor.getLastIterationTime();
                            if (diff > data.sttMaxWaitTime) {
                                logDetailed("Stalled rows detected with wait time of " + ((double) diff / 1000));
                                // 
                                try {
                                    eventProcessor.startWait();
                                    eventProcessor.doOneIteration();
                                } catch (Exception e) {
                                    throw new RuntimeException("Error in batch iteration when max wait time was exceeded", e);
                                } finally {
                                    eventProcessor.endWait();
                                }
                                logDetailed("Done processing after max wait time.");
                            }
                        }
                    }
                }
            };
            // Check ten times per second
            // 
            timer.schedule(timerTask, 100, 100);
        } catch (RuntimeException e) {
            throw new HopTransformException("Error in batch iteration when max wait time was exceeded", e);
        }
    }
    try {
        host.registerEventProcessorFactory(partitionContext -> eventProcessor).whenComplete((unused, e) -> {
            // 
            if (e != null) {
                logError("Failure while registering: " + e.toString());
                if (e.getCause() != null) {
                    logError("Inner exception: " + e.getCause().toString());
                }
                setErrors(1);
                stopAll();
                setOutputDone();
            }
        }).thenAccept(unused -> {
            // 
            while (!AzureListener.this.isStopped() && !AzureListener.this.outputIsDone()) {
                try {
                    Thread.sleep(0, 100);
                } catch (InterruptedException e) {
                // Ignore
                }
            }
        }).thenCompose(unused -> host.unregisterEventProcessor()).exceptionally(e -> {
            logError("Failure while unregistering: " + e.toString());
            if (e.getCause() != null) {
                logError("Inner exception: " + e.getCause().toString());
            }
            return null;
        }).get();
    } catch (Exception e) {
        throw new HopException("Error in event processor", e);
    }
    setOutputDone();
    return false;
}

Example 10 with RowAdapter

use of org.apache.hop.pipeline.transform.RowAdapter in project hop by apache.

the class CalculatorUnitTest method assertCalculatorReminder.

private void assertCalculatorReminder(final Object expectedResult, final Object[] values, final int[] types) throws Exception {
    RowMeta inputRowMeta = new RowMeta();
    for (int i = 0; i < types.length; i++) {
        switch(types[i]) {
            case IValueMeta.TYPE_BIGNUMBER:
                inputRowMeta.addValueMeta(new ValueMetaBigNumber("f" + i));
                break;
            case IValueMeta.TYPE_NUMBER:
                inputRowMeta.addValueMeta(new ValueMetaNumber("f" + i));
                break;
            case IValueMeta.TYPE_INTEGER:
                inputRowMeta.addValueMeta(new ValueMetaInteger("f" + i));
                break;
            default:
                throw new IllegalArgumentException("Unexpected value dataType: " + types[i] + ". Long, Double or BigDecimal expected.");
        }
    }
    IRowSet inputRowSet = null;
    try {
        inputRowSet = smh.getMockInputRowSet(new Object[][] { { values[0], values[1] } });
    } catch (Exception pe) {
        pe.printStackTrace();
        fail();
    }
    inputRowSet.setRowMeta(inputRowMeta);
    CalculatorMeta meta = new CalculatorMeta();
    meta.getFunctions().add(new CalculatorMetaFunction("res", CalculationType.REMAINDER, "f0", "f1", null, "Number", 0, 0, "", "", "", "", false));
    CalculatorData data = new CalculatorData();
    Calculator calculator = new Calculator(smh.transformMeta, meta, data, 0, smh.pipelineMeta, smh.pipeline);
    calculator.addRowSetToInputRowSets(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init();
    // Verify output
    try {
        calculator.addRowListener(new RowAdapter() {

            @Override
            public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                try {
                    assertEquals(expectedResult, row[2]);
                } catch (Exception pe) {
                    throw new HopTransformException(pe);
                }
            }
        });
        calculator.processRow();
    } catch (HopException ke) {
        ke.printStackTrace();
        fail();
    }
}