Examples with BlockingRowSet org.pentaho.di.core.BlockingRowSet used on opensource projects

Example 11 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class BaseStep method openRemoteInputStepSocketsOnce.

/**
 * Opens socket connections to the remote input steps of this step. <br>
 * This method should be used by steps that don't call getRow() first in which it is executed automatically. <br>
 * <b>This method should be called before any data is read from previous steps.</b> <br>
 * This action is executed only once.
 *
 * @throws KettleStepException
 */
protected void openRemoteInputStepSocketsOnce() throws KettleStepException {
    if (!remoteInputSteps.isEmpty()) {
        if (!remoteInputStepsInitialized) {
            // 
            synchronized (inputRowSetsLock) {
                for (RemoteStep remoteStep : remoteInputSteps) {
                    try {
                        BlockingRowSet rowSet = remoteStep.openReaderSocket(this);
                        inputRowSets.add(rowSet);
                    } catch (Exception e) {
                        throw new KettleStepException("Error opening reader socket to remote step '" + remoteStep + "'", e);
                    }
                }
            }
            remoteInputStepsInitialized = true;
        }
    }
}

Example 12 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class MappingInput method setConnectorSteps.

public void setConnectorSteps(StepInterface[] sourceSteps, List<MappingValueRename> valueRenames, String mappingStepname) {
    if (sourceSteps == null) {
        throw new IllegalArgumentException(BaseMessages.getString(PKG, "MappingInput.Exception.IllegalArgumentSourceStep"));
    }
    if (valueRenames == null) {
        throw new IllegalArgumentException(BaseMessages.getString(PKG, "MappingInput.Exception.IllegalArgumentValueRename"));
    }
    if (sourceSteps.length != 0) {
        if (mappingStepname == null) {
            throw new IllegalArgumentException(BaseMessages.getString(PKG, "MappingInput.Exception.IllegalArgumentStepName"));
        }
    }
    for (StepInterface sourceStep : sourceSteps) {
        // 
        if (!sourceStep.isMapping()) {
            // OK, before we leave, make sure there is a rowset that covers the path to this target step.
            // We need to create a new RowSet and add it to the Input RowSets of the target step
            // 
            BlockingRowSet rowSet = new BlockingRowSet(getTransMeta().getSizeRowset());
            // This is always a single copy, both for source and target...
            // 
            rowSet.setThreadNameFromToCopy(sourceStep.getStepname(), 0, mappingStepname, 0);
            // Make sure to connect it to both sides...
            // 
            sourceStep.addRowSetToOutputRowSets(rowSet);
            sourceStep.identifyErrorOutput();
            addRowSetToInputRowSets(rowSet);
        }
    }
    data.valueRenames = valueRenames;
    data.sourceSteps = sourceSteps;
}

Example 13 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class BaseStep method openRemoteOutputStepSocketsOnce.

/**
 * Opens socket connections to the remote output steps of this step. <br>
 * This method is called in method initBeforeStart() because it needs to connect to the server sockets (remote steps)
 * as soon as possible to avoid time-out situations. <br>
 * This action is executed only once.
 *
 * @throws KettleStepException
 */
protected void openRemoteOutputStepSocketsOnce() throws KettleStepException {
    if (!remoteOutputSteps.isEmpty()) {
        if (!remoteOutputStepsInitialized) {
            synchronized (outputRowSetsLock) {
                // 
                for (int c = 0; c < outputRowSets.size(); c++) {
                    RowSet rowSet = outputRowSets.get(c);
                    rowSet.setRemoteSlaveServerName(getVariable(Const.INTERNAL_VARIABLE_SLAVE_SERVER_NAME));
                    if (getVariable(Const.INTERNAL_VARIABLE_SLAVE_SERVER_NAME) == null) {
                        throw new KettleStepException("Variable '" + Const.INTERNAL_VARIABLE_SLAVE_SERVER_NAME + "' is not defined.");
                    }
                }
                // 
                for (RemoteStep remoteStep : remoteOutputSteps) {
                    try {
                        if (remoteStep.getTargetSlaveServerName() == null) {
                            throw new KettleStepException("The target slave server name is not defined for remote output step: " + remoteStep);
                        }
                        BlockingRowSet rowSet = remoteStep.openWriterSocket();
                        if (log.isDetailed()) {
                            logDetailed(BaseMessages.getString(PKG, "BaseStep.Log.OpenedWriterSocketToRemoteStep", remoteStep));
                        }
                        outputRowSets.add(rowSet);
                    } catch (IOException e) {
                        throw new KettleStepException("Error opening writer socket to remote step '" + remoteStep + "'", e);
                    }
                }
            }
            remoteOutputStepsInitialized = true;
        }
    }
}

Example 14 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class RemoteStep method openWriterSocket.

/**
 * Open a socket for writing.
 *
 * @return the RowSet created that will accept the rows for the remote step
 * @throws IOException
 */
public synchronized BlockingRowSet openWriterSocket() throws IOException {
    // Create an output row set: to be added to BaseStep.outputRowSets
    // 
    final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset());
    // Set the details for the source and target step as well as the target slave server.
    // This will help us determine the pre-calculated partition nr later in the game. (putRow())
    // 
    rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr);
    rowSet.setRemoteSlaveServerName(targetSlaveServerName);
    // Start a thread that will read out the output row set and send the data over the wire...
    // This will make everything else transparent, copying, distributing, including partitioning, etc.
    // 
    Runnable runnable = new Runnable() {

        public void run() {
            try {
                // Accept the socket, create a connection
                // This blocks until something comes through...
                // 
                socket = serverSocket.accept();
                // Create the output stream...
                OutputStream socketOut = socket.getOutputStream();
                if (compressingStreams) {
                    gzipOutputStream = new GZIPOutputStream(socketOut, 50000);
                    bufferedOutputStream = new BufferedOutputStream(gzipOutputStream, bufferSize);
                } else {
                    bufferedOutputStream = new BufferedOutputStream(socketOut, bufferSize);
                }
                socketOut = bufferedOutputStream;
                if (encryptingStreams && key != null) {
                    byte[] transKey = baseStep.getTransMeta().getKey();
                    Key unwrappedKey = null;
                    try {
                        unwrappedKey = CertificateGenEncryptUtil.decodeTransmittedKey(transKey, key, baseStep.getTransMeta().isPrivateKey());
                    } catch (InvalidKeyException ex) {
                        baseStep.logError("Invalid key was received", ex);
                    } catch (InvalidKeySpecException ex) {
                        baseStep.logError("Invalid key specification was received. Most probably public key was " + "sent instead of private or vice versa", ex);
                    } catch (Exception ex) {
                        baseStep.logError("Error occurred during encryption initialization", ex);
                    }
                    try {
                        Cipher decryptionCip = CertificateGenEncryptUtil.initDecryptionCipher(unwrappedKey, key);
                        socketOut = cipherOutputStream = new CipherOutputStream(bufferedOutputStream, decryptionCip);
                    } catch (InvalidKeyException ex) {
                        baseStep.logError("Invalid key was received", ex);
                    } catch (Exception ex) {
                        baseStep.logError("Error occurred during encryption initialization", ex);
                    }
                }
                outputStream = new DataOutputStream(socketOut);
                baseStep.logBasic("Server socket accepted for port [" + port + "], reading from server " + targetSlaveServerName);
                // get a row of data...
                Object[] rowData = baseStep.getRowFrom(rowSet);
                if (rowData != null) {
                    rowSet.getRowMeta().writeMeta(outputStream);
                }
                // 
                while (rowData != null && !baseStep.isStopped()) {
                    // It's too confusing to count these twice, so decrement
                    baseStep.decrementLinesRead();
                    baseStep.decrementLinesWritten();
                    // Write the row to the remote step via the output stream....
                    // 
                    rowSet.getRowMeta().writeData(outputStream, rowData);
                    baseStep.incrementLinesOutput();
                    if (baseStep.log.isDebug()) {
                        baseStep.logDebug("Sent row to port " + port + " : " + rowSet.getRowMeta().getString(rowData));
                    }
                    rowData = baseStep.getRowFrom(rowSet);
                }
                if (compressingStreams) {
                    outputStream.flush();
                    gzipOutputStream.finish();
                } else {
                    outputStream.flush();
                }
            } catch (Exception e) {
                baseStep.logError("Error writing to remote step", e);
                baseStep.setErrors(1);
                baseStep.stopAll();
            } finally {
                try {
                    if (socket != null) {
                        socket.shutdownOutput();
                    }
                } catch (Exception e) {
                    baseStep.logError("Error shutting down output channel on the server socket of remote step", e);
                    baseStep.setErrors(1L);
                    baseStep.stopAll();
                }
                try {
                    if (outputStream != null) {
                        outputStream.flush();
                        outputStream.close();
                        if (cipherOutputStream != null) {
                            cipherOutputStream.close();
                        }
                        bufferedOutputStream.close();
                        if (gzipOutputStream != null) {
                            gzipOutputStream.close();
                        }
                    }
                } catch (Exception e) {
                    baseStep.logError("Error shutting down output streams on the server socket of remote step", e);
                    baseStep.setErrors(1L);
                    baseStep.stopAll();
                }
                outputStream = null;
                bufferedOutputStream = null;
                gzipOutputStream = null;
                cipherOutputStream = null;
            // 
            // Now we can't close the server socket.
            // This would immediately kill all the remaining data on the client side.
            // The close of the server socket will happen when all the transformation in the cluster have finished.
            // Then Trans.cleanup() will be called.
            }
        }
    };
    // Fire this off in the in a separate thread...
    // 
    new Thread(runnable).start();
    // 
    return rowSet;
}

Example 15 with BlockingRowSet

use of org.pentaho.di.core.BlockingRowSet in project pentaho-kettle by pentaho.

the class RemoteStep method openReaderSocket.

public synchronized BlockingRowSet openReaderSocket(final BaseStep baseStep) throws IOException, KettleException {
    this.baseStep = baseStep;
    final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset());
    // Make sure we handle the case with multiple step copies running on a
    // slave...
    // 
    rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr);
    rowSet.setRemoteSlaveServerName(targetSlaveServerName);
    final int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port));
    final String realHostname = baseStep.environmentSubstitute(hostname);
    // Connect to the server socket (started during BaseStep.init())
    // Because the accept() call on the server socket can be called after we
    // reached this code
    // it is best to build in a retry loop with a time-out here.
    // 
    long startTime = System.currentTimeMillis();
    boolean connected = false;
    KettleException lastException = null;
    // // timeout with retry until connected
    while (!connected && (TIMEOUT_IN_SECONDS > (System.currentTimeMillis() - startTime) / 1000) && !baseStep.isStopped()) {
        try {
            socket = new Socket();
            socket.setReuseAddress(true);
            baseStep.logDetailed("Step variable MASTER_HOST : [" + baseStep.getVariable("MASTER_HOST") + "]");
            baseStep.logDetailed("Opening client (reader) socket to server [" + Const.NVL(realHostname, "") + ":" + port + "]");
            socket.connect(new InetSocketAddress(realHostname, portNumber), 5000);
            connected = true;
            InputStream socketStream = socket.getInputStream();
            if (compressingStreams) {
                gzipInputStream = new GZIPInputStream(socketStream);
                bufferedInputStream = new BufferedInputStream(gzipInputStream, bufferSize);
            } else {
                bufferedInputStream = new BufferedInputStream(socketStream, bufferSize);
            }
            socketStream = bufferedInputStream;
            if (encryptingStreams && key != null) {
                byte[] transKey = baseStep.getTransMeta().getKey();
                Key unwrappedKey = null;
                try {
                    unwrappedKey = CertificateGenEncryptUtil.decodeTransmittedKey(transKey, key, baseStep.getTransMeta().isPrivateKey());
                } catch (InvalidKeyException ex) {
                    baseStep.logError("Invalid key was received", ex);
                } catch (InvalidKeySpecException ex) {
                    baseStep.logError("Invalid key specification was received. Most probably public key was " + "sent instead of private or vice versa", ex);
                } catch (Exception ex) {
                    baseStep.logError("Error occurred during encryption initialization", ex);
                }
                try {
                    Cipher decryptionCip = CertificateGenEncryptUtil.initDecryptionCipher(unwrappedKey, key);
                    socketStream = cipherInputStream = new CipherInputStream(bufferedInputStream, decryptionCip);
                } catch (InvalidKeyException ex) {
                    baseStep.logError("Invalid key was received", ex);
                } catch (Exception ex) {
                    baseStep.logError("Error occurred during encryption initialization", ex);
                }
            }
            inputStream = new DataInputStream(socketStream);
            lastException = null;
        } catch (Exception e) {
            lastException = new KettleException("Unable to open socket to server " + realHostname + " port " + portNumber, e);
        }
        if (lastException != null) {
            // Sleep for a while
            try {
                Thread.sleep(250);
            } catch (InterruptedException e) {
                if (socket != null) {
                    socket.shutdownInput();
                    socket.shutdownOutput();
                    socket.close();
                    baseStep.logDetailed("Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
                }
                throw new KettleException("Interrupted while trying to connect to server socket: " + e.toString());
            }
        }
    }
    // See if all was OK...
    if (lastException != null) {
        baseStep.logError("Error initialising step: " + lastException.toString());
        if (socket != null) {
            socket.shutdownInput();
            socket.shutdownOutput();
            socket.close();
            baseStep.logDetailed("Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
        }
        throw lastException;
    } else {
        if (inputStream == null) {
            throw new KettleException("Unable to connect to the SocketWriter in the " + TIMEOUT_IN_SECONDS + "s timeout period.");
        }
    }
    baseStep.logDetailed("Opened connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
    // Create a thread to take care of the reading from the client socket.
    // The rows read will be put in a RowSet buffer.
    // That buffer will hand over the rows to the step that has this RemoteStep
    // object defined
    // as a remote input step.
    // 
    Runnable runnable = new Runnable() {

        public void run() {
            try {
                // First read the row meta data from the socket...
                // 
                RowMetaInterface rowMeta = null;
                while (!baseStep.isStopped() && rowMeta == null) {
                    try {
                        rowMeta = new RowMeta(inputStream);
                    } catch (SocketTimeoutException e) {
                        rowMeta = null;
                    }
                }
                if (rowMeta == null) {
                    // leave now.
                    throw new KettleEOFException();
                }
                // And a first row of data...
                // 
                Object[] rowData = getRowOfData(rowMeta);
                // 
                while (rowData != null && !baseStep.isStopped()) {
                    baseStep.incrementLinesInput();
                    baseStep.decrementLinesRead();
                    if (baseStep.log.isDebug()) {
                        baseStep.logDebug("Received row from remote step: " + rowMeta.getString(rowData));
                    }
                    baseStep.putRowTo(rowMeta, rowData, rowSet);
                    baseStep.decrementLinesWritten();
                    rowData = getRowOfData(rowMeta);
                }
            } catch (KettleEOFException e) {
                // 
                if (baseStep.log.isDebug()) {
                    baseStep.logDebug("Finished reading from remote step on server " + hostname + " port " + portNumber);
                }
            } catch (Exception e) {
                baseStep.logError("Error reading from client socket to remote step", e);
                baseStep.setErrors(1);
                baseStep.stopAll();
            } finally {
                // Close the input socket
                if (socket != null && !socket.isClosed() && !socket.isInputShutdown()) {
                    try {
                        socket.shutdownInput();
                    } catch (Exception e) {
                        baseStep.logError("Error shutting down input channel on client socket connection to remote step", e);
                    }
                }
                if (socket != null && !socket.isClosed() && !socket.isOutputShutdown()) {
                    try {
                        socket.shutdownOutput();
                    } catch (Exception e) {
                        baseStep.logError("Error shutting down output channel on client socket connection to remote step", e);
                    }
                }
                if (socket != null && !socket.isClosed()) {
                    try {
                        socket.close();
                    } catch (Exception e) {
                        baseStep.logError("Error shutting down client socket connection to remote step", e);
                    }
                }
                if (inputStream != null) {
                    try {
                        inputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                    inputStream = null;
                }
                if (cipherInputStream != null) {
                    try {
                        cipherInputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                }
                cipherInputStream = null;
                if (bufferedInputStream != null) {
                    try {
                        bufferedInputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                }
                bufferedInputStream = null;
                if (gzipInputStream != null) {
                    try {
                        gzipInputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                }
                gzipInputStream = null;
                baseStep.logDetailed("Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
            }
            // signal baseStep that nothing else comes from this step.
            // 
            rowSet.setDone();
        }
    };
    new Thread(runnable).start();
    return rowSet;
}