Examples with DataTransferClient com.qubole.rubix.spi.DataTransferClientFactory.DataTransferClient used on opensource projects

Example 1 with DataTransferClient

use of com.qubole.rubix.spi.DataTransferClientFactory.DataTransferClient in project rubix by qubole.

the class NonLocalReadRequestChain method call.

@Override
public Long call() throws Exception {
    log.debug(String.format("Read Request threadName: %s, Non Local read Executor threadName: %s", threadName, Thread.currentThread().getName()));
    Thread.currentThread().setName(threadName);
    if (readRequests.size() == 0) {
        return 0L;
    }
    checkState(isLocked, "Trying to execute Chain without locking");
    int dataReadInPreviousCycle = 0;
    for (ReadRequest readRequest : readRequests) {
        readFromNonLocalCache += dataReadInPreviousCycle;
        if (cancelled) {
            propagateCancel(this.getClass().getName());
        }
        try (DataTransferClient dataTransferClient = getClient(remoteNodeName, conf)) {
            int nread = 0;
            /*
        SocketChannels does not support timeouts when used directly, because timeout is used only by streams.
        We get this working by wrapping it in ReadableByteChannel.
        Ref - https://technfun.wordpress.com/2009/01/29/networking-in-java-non-blocking-nio-blocking-nio-and-io/
        */
            InputStream inStream = dataTransferClient.getSocketChannel().socket().getInputStream();
            ReadableByteChannel wrappedChannel = Channels.newChannel(inStream);
            ByteBuffer buf = DataTransferClientHelper.writeHeaders(conf, new DataTransferHeader(readRequest.getActualReadStart(), readRequest.getActualReadLengthIntUnsafe(), fileSize, lastModified, clusterType, filePath));
            try {
                dataTransferClient.getSocketChannel().write(buf);
            } catch (IOException e) {
                dataTransferClient.getSocketChannel().close();
                throw e;
            }
            int bytesread = 0;
            ByteBuffer dst = ByteBuffer.wrap(readRequest.destBuffer, readRequest.getDestBufferOffset(), readRequest.destBuffer.length - readRequest.getDestBufferOffset());
            while (bytesread != readRequest.getActualReadLengthIntUnsafe()) {
                try {
                    nread = wrappedChannel.read(dst);
                } catch (IOException e) {
                    log.warn("Error in reading..closing socket channel: " + dataTransferClient.getSocketChannel(), e);
                    dataTransferClient.getSocketChannel().close();
                    throw e;
                }
                if (nread == -1) {
                    log.warn("Error reading from Local Transfer Server");
                    dataTransferClient.getSocketChannel().close();
                    throw new IOException("Error reading from Local Transfer Server");
                }
                bytesread += nread;
                dst.position(bytesread + readRequest.getDestBufferOffset());
            }
            dataReadInPreviousCycle = bytesread;
        } catch (Exception e) {
            if (strictMode) {
                log.warn("Error reading data from node : " + remoteNodeName, e);
                throw Throwables.propagate(e);
            } else {
                log.warn("Error in reading from node: " + remoteNodeName + " Using direct reads", e);
                CustomMetricsReporterProvider.getCustomMetricsReporter().addMetric(NON_LOCAL_FALLBACK_TO_DIRECT_READ);
                return directReadRequest(readRequests.indexOf(readRequest));
            }
        } finally {
            if (statistics != null) {
                statistics.incrementBytesRead(readFromNonLocalCache);
            }
        }
    }
    readFromNonLocalCache += dataReadInPreviousCycle;
    log.debug(String.format("Read %d bytes internally from node %s", readFromNonLocalCache, remoteNodeName));
    return readFromNonLocalCache;
}