Examples with KettleEOFException org.pentaho.di.core.exception.KettleEOFException used on opensource projects

Example 1 with KettleEOFException

use of org.pentaho.di.core.exception.KettleEOFException in project pentaho-kettle by pentaho.

the class ValueMetaBase method readMetaData.

/**
 * Load the attributes of this particular value meta object from the input stream. Loading the type is not handled
 * here, this should be read from the stream previously!
 *
 * @param inputStream
 *          the input stream to read from
 * @throws KettleFileException
 *           In case there was a IO problem
 * @throws KettleEOFException
 *           If we reached the end of the stream
 */
@Override
public void readMetaData(DataInputStream inputStream) throws KettleFileException, KettleEOFException {
    // 
    try {
        // Handle storage type
        storageType = inputStream.readInt();
        // Read the data in the index
        switch(storageType) {
            case STORAGE_TYPE_INDEXED:
                int indexSize = inputStream.readInt();
                if (indexSize < 0) {
                    index = null;
                } else {
                    index = new Object[indexSize];
                    for (int i = 0; i < indexSize; i++) {
                        switch(type) {
                            case TYPE_STRING:
                                index[i] = readString(inputStream);
                                break;
                            case TYPE_NUMBER:
                                index[i] = readNumber(inputStream);
                                break;
                            case TYPE_INTEGER:
                                index[i] = readInteger(inputStream);
                                break;
                            case TYPE_DATE:
                                index[i] = readDate(inputStream);
                                break;
                            case TYPE_BIGNUMBER:
                                index[i] = readBigNumber(inputStream);
                                break;
                            case TYPE_BOOLEAN:
                                index[i] = readBoolean(inputStream);
                                break;
                            case TYPE_BINARY:
                                index[i] = readBinary(inputStream);
                                break;
                            default:
                                throw new KettleFileException(toString() + " : Unable to de-serialize indexed storage type for data type " + getType());
                        }
                    }
                }
                break;
            case STORAGE_TYPE_BINARY_STRING:
                // well..
                if (inputStream.readBoolean()) {
                    storageMetadata = new ValueMetaBase(inputStream);
                }
                break;
            default:
                break;
        }
        // name
        name = readString(inputStream);
        // length & precision
        length = inputStream.readInt();
        precision = inputStream.readInt();
        // Origin
        origin = readString(inputStream);
        // Comments
        comments = readString(inputStream);
        // formatting Mask, decimal, grouping, currency
        conversionMask = readString(inputStream);
        decimalSymbol = readString(inputStream);
        groupingSymbol = readString(inputStream);
        currencySymbol = readString(inputStream);
        trimType = inputStream.readInt();
        // Case sensitivity
        caseInsensitive = inputStream.readBoolean();
        // Collator locale
        setCollatorLocale(Locale.forLanguageTag(readString(inputStream)));
        // Collator disabled
        collatorDisabled = inputStream.readBoolean();
        // Collator strength
        collatorStrength = inputStream.readInt();
        // Sorting type
        sortedDescending = inputStream.readBoolean();
        // Output padding?
        outputPaddingEnabled = inputStream.readBoolean();
        // is date parsing lenient?
        // 
        dateFormatLenient = inputStream.readBoolean();
        // What is the date format locale?
        // 
        String strDateFormatLocale = readString(inputStream);
        if (Utils.isEmpty(strDateFormatLocale)) {
            dateFormatLocale = null;
        } else {
            dateFormatLocale = EnvUtil.createLocale(strDateFormatLocale);
        }
        // What is the time zone to use for date parsing?
        // 
        String strTimeZone = readString(inputStream);
        if (Utils.isEmpty(strTimeZone)) {
            dateFormatTimeZone = TimeZone.getDefault();
        } else {
            dateFormatTimeZone = EnvUtil.createTimeZone(strTimeZone);
        }
        // is string to number parsing lenient?
        lenientStringToNumber = inputStream.readBoolean();
    } catch (EOFException e) {
        throw new KettleEOFException(e);
    } catch (IOException e) {
        throw new KettleFileException(toString() + " : Unable to read value metadata from input stream", e);
    }
}

Example 2 with KettleEOFException

use of org.pentaho.di.core.exception.KettleEOFException in project pentaho-kettle by pentaho.

the class ValueMetaTimestamp method readData.

@Override
public Object readData(DataInputStream inputStream) throws KettleFileException, KettleEOFException, SocketTimeoutException {
    try {
        // Is the value NULL?
        if (inputStream.readBoolean()) {
            // done
            return null;
        }
        switch(storageType) {
            case STORAGE_TYPE_NORMAL:
                // Handle Content -- only when not NULL
                long time = inputStream.readLong();
                int nanos = inputStream.readInt();
                Timestamp timestamp = new Timestamp(time);
                timestamp.setNanos(nanos);
                return timestamp;
            case STORAGE_TYPE_BINARY_STRING:
                return readBinaryString(inputStream);
            case STORAGE_TYPE_INDEXED:
                // just an index: 4-bytes should be enough.
                return readSmallInteger(inputStream);
            default:
                throw new KettleFileException(toString() + " : Unknown storage type " + getStorageType());
        }
    } catch (EOFException e) {
        throw new KettleEOFException(e);
    } catch (SocketTimeoutException e) {
        throw e;
    } catch (IOException e) {
        throw new KettleFileException(toString() + " : Unable to read value timestamp data from input stream", e);
    }
}

Example 3 with KettleEOFException

use of org.pentaho.di.core.exception.KettleEOFException in project pentaho-kettle by pentaho.

the class CubeInput method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    if (first) {
        first = false;
        meta = (CubeInputMeta) smi;
        data = (CubeInputData) sdi;
        realRowLimit = Const.toInt(environmentSubstitute(meta.getRowLimit()), 0);
    }
    try {
        Object[] r = data.meta.readData(data.dis);
        // fill the rowset(s). (sleeps if full)
        putRow(data.meta, r);
        incrementLinesInput();
        if (realRowLimit > 0 && getLinesInput() >= realRowLimit) {
            // finished!
            setOutputDone();
            return false;
        }
    } catch (KettleEOFException eof) {
        setOutputDone();
        return false;
    } catch (SocketTimeoutException e) {
        // shouldn't happen on files
        throw new KettleException(e);
    }
    if (checkFeedback(getLinesInput())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "CubeInput.Log.LineNumber") + getLinesInput());
        }
    }
    return true;
}

Example 4 with KettleEOFException

use of org.pentaho.di.core.exception.KettleEOFException in project pentaho-kettle by pentaho.

the class SocketReader method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (SocketReaderMeta) smi;
    data = (SocketReaderData) sdi;
    try {
        Object[] r;
        if (first) {
            // Connect to the server socket (started during init)
            // Because the accept() call on the server socket can be called after we reached this code
            // it is best to build in a retry loop with a time-out here.
            // 
            long startTime = System.currentTimeMillis();
            boolean connected = false;
            KettleException lastException = null;
            // // timeout with retry until connected
            while (!connected && (TIMEOUT_IN_SECONDS > (System.currentTimeMillis() - startTime) / 1000) && !isStopped()) {
                try {
                    int port = Integer.parseInt(environmentSubstitute(meta.getPort()));
                    int bufferSize = Integer.parseInt(environmentSubstitute(meta.getBufferSize()));
                    data.socket = new Socket(environmentSubstitute(meta.getHostname()), port);
                    connected = true;
                    if (meta.isCompressed()) {
                        data.outputStream = new DataOutputStream(new BufferedOutputStream(new GZIPOutputStream(data.socket.getOutputStream()), bufferSize));
                        data.inputStream = new DataInputStream(new BufferedInputStream(new GZIPInputStream(data.socket.getInputStream()), bufferSize));
                    } else {
                        data.outputStream = new DataOutputStream(new BufferedOutputStream(data.socket.getOutputStream(), bufferSize));
                        data.inputStream = new DataInputStream(new BufferedInputStream(data.socket.getInputStream(), bufferSize));
                    }
                    lastException = null;
                } catch (Exception e) {
                    lastException = new KettleException("Unable to open socket to server " + environmentSubstitute(meta.getHostname()) + " port " + environmentSubstitute(meta.getPort()), e);
                }
                if (lastException != null) {
                    // Sleep for a second
                    Thread.sleep(1000);
                }
            }
            if (lastException != null) {
                logError("Error initialising step: " + lastException.toString());
                logError(Const.getStackTracker(lastException));
                if (data.socket != null) {
                    data.socket.shutdownInput();
                    data.socket.shutdownOutput();
                    data.socket.close();
                    logError("Closed connection to data socket to " + environmentSubstitute(meta.getHostname()) + " port " + environmentSubstitute(meta.getPort()));
                }
                throw lastException;
            } else {
                if (data.inputStream == null) {
                    throw new KettleException("Unable to connect to the SocketWriter in the " + TIMEOUT_IN_SECONDS + "s timeout period.");
                }
            }
            // This is the metadata
            data.rowMeta = new RowMeta(data.inputStream);
            first = false;
        }
        r = data.rowMeta.readData(data.inputStream);
        incrementLinesInput();
        if (checkFeedback(getLinesInput())) {
            logBasic(BaseMessages.getString(PKG, "SocketReader.Log.LineNumber") + getLinesInput());
        }
        putRow(data.rowMeta, r);
    } catch (KettleEOFException e) {
        // finished reading.
        setOutputDone();
        return false;
    } catch (Exception e) {
        if (data.socket != null) {
            try {
                data.socket.shutdownInput();
                data.socket.shutdownOutput();
                data.socket.close();
                logError("Closed connection to data socket to " + environmentSubstitute(meta.getHostname()) + " port " + environmentSubstitute(meta.getPort()));
            } catch (IOException e1) {
                logError("Failed to close connection to data socket to " + environmentSubstitute(meta.getHostname()) + " port " + environmentSubstitute(meta.getPort()));
            }
        }
        throw new KettleException(e);
    }
    return true;
}

Example 5 with KettleEOFException

use of org.pentaho.di.core.exception.KettleEOFException in project pentaho-kettle by pentaho.

the class RemoteStep method openReaderSocket.

public synchronized BlockingRowSet openReaderSocket(final BaseStep baseStep) throws IOException, KettleException {
    this.baseStep = baseStep;
    final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset());
    // Make sure we handle the case with multiple step copies running on a
    // slave...
    // 
    rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr);
    rowSet.setRemoteSlaveServerName(targetSlaveServerName);
    final int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port));
    final String realHostname = baseStep.environmentSubstitute(hostname);
    // Connect to the server socket (started during BaseStep.init())
    // Because the accept() call on the server socket can be called after we
    // reached this code
    // it is best to build in a retry loop with a time-out here.
    // 
    long startTime = System.currentTimeMillis();
    boolean connected = false;
    KettleException lastException = null;
    // // timeout with retry until connected
    while (!connected && (TIMEOUT_IN_SECONDS > (System.currentTimeMillis() - startTime) / 1000) && !baseStep.isStopped()) {
        try {
            socket = new Socket();
            socket.setReuseAddress(true);
            baseStep.logDetailed("Step variable MASTER_HOST : [" + baseStep.getVariable("MASTER_HOST") + "]");
            baseStep.logDetailed("Opening client (reader) socket to server [" + Const.NVL(realHostname, "") + ":" + port + "]");
            socket.connect(new InetSocketAddress(realHostname, portNumber), 5000);
            connected = true;
            InputStream socketStream = socket.getInputStream();
            if (compressingStreams) {
                gzipInputStream = new GZIPInputStream(socketStream);
                bufferedInputStream = new BufferedInputStream(gzipInputStream, bufferSize);
            } else {
                bufferedInputStream = new BufferedInputStream(socketStream, bufferSize);
            }
            socketStream = bufferedInputStream;
            if (encryptingStreams && key != null) {
                byte[] transKey = baseStep.getTransMeta().getKey();
                Key unwrappedKey = null;
                try {
                    unwrappedKey = CertificateGenEncryptUtil.decodeTransmittedKey(transKey, key, baseStep.getTransMeta().isPrivateKey());
                } catch (InvalidKeyException ex) {
                    baseStep.logError("Invalid key was received", ex);
                } catch (InvalidKeySpecException ex) {
                    baseStep.logError("Invalid key specification was received. Most probably public key was " + "sent instead of private or vice versa", ex);
                } catch (Exception ex) {
                    baseStep.logError("Error occurred during encryption initialization", ex);
                }
                try {
                    Cipher decryptionCip = CertificateGenEncryptUtil.initDecryptionCipher(unwrappedKey, key);
                    socketStream = cipherInputStream = new CipherInputStream(bufferedInputStream, decryptionCip);
                } catch (InvalidKeyException ex) {
                    baseStep.logError("Invalid key was received", ex);
                } catch (Exception ex) {
                    baseStep.logError("Error occurred during encryption initialization", ex);
                }
            }
            inputStream = new DataInputStream(socketStream);
            lastException = null;
        } catch (Exception e) {
            lastException = new KettleException("Unable to open socket to server " + realHostname + " port " + portNumber, e);
        }
        if (lastException != null) {
            // Sleep for a while
            try {
                Thread.sleep(250);
            } catch (InterruptedException e) {
                if (socket != null) {
                    socket.shutdownInput();
                    socket.shutdownOutput();
                    socket.close();
                    baseStep.logDetailed("Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
                }
                throw new KettleException("Interrupted while trying to connect to server socket: " + e.toString());
            }
        }
    }
    // See if all was OK...
    if (lastException != null) {
        baseStep.logError("Error initialising step: " + lastException.toString());
        if (socket != null) {
            socket.shutdownInput();
            socket.shutdownOutput();
            socket.close();
            baseStep.logDetailed("Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
        }
        throw lastException;
    } else {
        if (inputStream == null) {
            throw new KettleException("Unable to connect to the SocketWriter in the " + TIMEOUT_IN_SECONDS + "s timeout period.");
        }
    }
    baseStep.logDetailed("Opened connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
    // Create a thread to take care of the reading from the client socket.
    // The rows read will be put in a RowSet buffer.
    // That buffer will hand over the rows to the step that has this RemoteStep
    // object defined
    // as a remote input step.
    // 
    Runnable runnable = new Runnable() {

        public void run() {
            try {
                // First read the row meta data from the socket...
                // 
                RowMetaInterface rowMeta = null;
                while (!baseStep.isStopped() && rowMeta == null) {
                    try {
                        rowMeta = new RowMeta(inputStream);
                    } catch (SocketTimeoutException e) {
                        rowMeta = null;
                    }
                }
                if (rowMeta == null) {
                    // leave now.
                    throw new KettleEOFException();
                }
                // And a first row of data...
                // 
                Object[] rowData = getRowOfData(rowMeta);
                // 
                while (rowData != null && !baseStep.isStopped()) {
                    baseStep.incrementLinesInput();
                    baseStep.decrementLinesRead();
                    if (baseStep.log.isDebug()) {
                        baseStep.logDebug("Received row from remote step: " + rowMeta.getString(rowData));
                    }
                    baseStep.putRowTo(rowMeta, rowData, rowSet);
                    baseStep.decrementLinesWritten();
                    rowData = getRowOfData(rowMeta);
                }
            } catch (KettleEOFException e) {
                // 
                if (baseStep.log.isDebug()) {
                    baseStep.logDebug("Finished reading from remote step on server " + hostname + " port " + portNumber);
                }
            } catch (Exception e) {
                baseStep.logError("Error reading from client socket to remote step", e);
                baseStep.setErrors(1);
                baseStep.stopAll();
            } finally {
                // Close the input socket
                if (socket != null && !socket.isClosed() && !socket.isInputShutdown()) {
                    try {
                        socket.shutdownInput();
                    } catch (Exception e) {
                        baseStep.logError("Error shutting down input channel on client socket connection to remote step", e);
                    }
                }
                if (socket != null && !socket.isClosed() && !socket.isOutputShutdown()) {
                    try {
                        socket.shutdownOutput();
                    } catch (Exception e) {
                        baseStep.logError("Error shutting down output channel on client socket connection to remote step", e);
                    }
                }
                if (socket != null && !socket.isClosed()) {
                    try {
                        socket.close();
                    } catch (Exception e) {
                        baseStep.logError("Error shutting down client socket connection to remote step", e);
                    }
                }
                if (inputStream != null) {
                    try {
                        inputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                    inputStream = null;
                }
                if (cipherInputStream != null) {
                    try {
                        cipherInputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                }
                cipherInputStream = null;
                if (bufferedInputStream != null) {
                    try {
                        bufferedInputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                }
                bufferedInputStream = null;
                if (gzipInputStream != null) {
                    try {
                        gzipInputStream.close();
                    } catch (Exception e) {
                        baseStep.logError("Error closing input stream on socket connection to remote step", e);
                    }
                }
                gzipInputStream = null;
                baseStep.logDetailed("Closed connection to server socket to read rows from remote step on server " + realHostname + " port " + portNumber + " - Local port=" + socket.getLocalPort());
            }
            // signal baseStep that nothing else comes from this step.
            // 
            rowSet.setDone();
        }
    };
    new Thread(runnable).start();
    return rowSet;
}