Examples with IOE org.apache.druid.java.util.common.IOE used on opensource projects

Example 16 with IOE

use of org.apache.druid.java.util.common.IOE in project druid by druid-io.

the class S3DataSegmentPuller method buildFileObject.

public FileObject buildFileObject(final URI uri) throws AmazonServiceException {
    final CloudObjectLocation coords = new CloudObjectLocation(S3Utils.checkURI(uri));
    final String path = uri.getPath();
    return new FileObject() {

        S3Object s3Object = null;

        S3ObjectSummary objectSummary = null;

        @Override
        public URI toUri() {
            return uri;
        }

        @Override
        public String getName() {
            final String ext = Files.getFileExtension(path);
            return Files.getNameWithoutExtension(path) + (Strings.isNullOrEmpty(ext) ? "" : ("." + ext));
        }

        /**
         * Returns an input stream for a s3 object. The returned input stream is not thread-safe.
         */
        @Override
        public InputStream openInputStream() throws IOException {
            try {
                if (s3Object == null) {
                    // lazily promote to full GET
                    s3Object = s3Client.getObject(coords.getBucket(), coords.getPath());
                }
                final InputStream in = s3Object.getObjectContent();
                final Closer closer = Closer.create();
                closer.register(in);
                closer.register(s3Object);
                return new FilterInputStream(in) {

                    @Override
                    public void close() throws IOException {
                        closer.close();
                    }
                };
            } catch (AmazonServiceException e) {
                throw new IOE(e, "Could not load S3 URI [%s]", uri);
            }
        }

        @Override
        public OutputStream openOutputStream() {
            throw new UOE("Cannot stream S3 output");
        }

        @Override
        public Reader openReader(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open reader");
        }

        @Override
        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open character sequence");
        }

        @Override
        public Writer openWriter() {
            throw new UOE("Cannot open writer");
        }

        @Override
        public long getLastModified() {
            if (s3Object != null) {
                return s3Object.getObjectMetadata().getLastModified().getTime();
            }
            if (objectSummary == null) {
                objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.getBucket(), coords.getPath());
            }
            return objectSummary.getLastModified().getTime();
        }

        @Override
        public boolean delete() {
            throw new UOE("Cannot delete S3 items anonymously. jetS3t doesn't support authenticated deletes easily.");
        }
    };
}

Example 17 with IOE

use of org.apache.druid.java.util.common.IOE in project druid by druid-io.

the class LocalDataSegmentPusher method pushToPath.

@Override
public DataSegment pushToPath(File dataSegmentFile, DataSegment segment, String storageDirSuffix) throws IOException {
    final File baseStorageDir = config.getStorageDirectory();
    final File outDir = new File(baseStorageDir, storageDirSuffix);
    log.debug("Copying segment[%s] to local filesystem at location[%s]", segment.getId(), outDir.toString());
    if (dataSegmentFile.equals(outDir)) {
        long size = 0;
        for (File file : dataSegmentFile.listFiles()) {
            size += file.length();
        }
        return segment.withLoadSpec(makeLoadSpec(outDir.toURI())).withSize(size).withBinaryVersion(SegmentUtils.getVersionFromDir(dataSegmentFile));
    }
    final File tmpOutDir = new File(config.getStorageDirectory(), makeIntermediateDir());
    log.debug("Creating intermediate directory[%s] for segment[%s].", tmpOutDir.toString(), segment.getId());
    FileUtils.mkdirp(tmpOutDir);
    try {
        final File tmpIndexFile = new File(tmpOutDir, INDEX_FILENAME);
        final long size = compressSegment(dataSegmentFile, tmpIndexFile);
        final DataSegment dataSegment = segment.withLoadSpec(makeLoadSpec(new File(outDir, INDEX_FILENAME).toURI())).withSize(size).withBinaryVersion(SegmentUtils.getVersionFromDir(dataSegmentFile));
        FileUtils.mkdirp(outDir);
        final File indexFileTarget = new File(outDir, tmpIndexFile.getName());
        if (!tmpIndexFile.renameTo(indexFileTarget)) {
            throw new IOE("Failed to rename [%s] to [%s]", tmpIndexFile, indexFileTarget);
        }
        return dataSegment;
    } finally {
        FileUtils.deleteDirectory(tmpOutDir);
    }
}

Example 18 with IOE

use of org.apache.druid.java.util.common.IOE in project druid by druid-io.

the class FileTaskLogs method killOlderThan.

@Override
public void killOlderThan(final long timestamp) throws IOException {
    File taskLogDir = config.getDirectory();
    if (taskLogDir.exists()) {
        if (!taskLogDir.isDirectory()) {
            throw new IOE("taskLogDir [%s] must be a directory.", taskLogDir);
        }
        File[] files = taskLogDir.listFiles(f -> f.lastModified() < timestamp);
        for (File file : files) {
            log.info("Deleting local task log [%s].", file.getAbsolutePath());
            org.apache.commons.io.FileUtils.forceDelete(file);
            if (Thread.currentThread().isInterrupted()) {
                throw new IOException(new InterruptedException("Thread interrupted. Couldn't delete all tasklogs."));
            }
        }
    }
}

Example 19 with IOE

use of org.apache.druid.java.util.common.IOE in project druid by druid-io.

the class HdfsFileTimestampVersionFinderTest method setupStatic.

@BeforeClass
public static void setupStatic() throws IOException {
    hdfsTmpDir = File.createTempFile("hdfsHandlerTest", "dir");
    if (!hdfsTmpDir.delete()) {
        throw new IOE("Unable to delete hdfsTmpDir [%s]", hdfsTmpDir.getAbsolutePath());
    }
    conf = new Configuration(true);
    fileSystem = new LocalFileSystem();
    fileSystem.initialize(hdfsTmpDir.toURI(), conf);
    fileSystem.setWorkingDirectory(new Path(hdfsTmpDir.toURI()));
    final File tmpFile = File.createTempFile("hdfsHandlerTest", ".data");
    tmpFile.delete();
    try {
        Files.copy(new ByteArrayInputStream(pathByteContents), tmpFile.toPath());
        try (OutputStream stream = fileSystem.create(filePath)) {
            Files.copy(tmpFile.toPath(), stream);
        }
    } finally {
        tmpFile.delete();
    }
}

Example 20 with IOE

use of org.apache.druid.java.util.common.IOE in project druid by druid-io.

the class DruidLeaderClient method go.

/**
 * Executes a Request object aimed at the leader. Throws IOException if the leader cannot be located.
 */
public <T, H extends FullResponseHolder<T>> H go(Request request, HttpResponseHandler<H, H> responseHandler) throws IOException, InterruptedException {
    Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS));
    for (int counter = 0; counter < MAX_RETRIES; counter++) {
        final H fullResponseHolder;
        try {
            try {
                fullResponseHolder = httpClient.go(request, responseHandler).get();
            } catch (ExecutionException e) {
                // Unwrap IOExceptions and ChannelExceptions, re-throw others
                Throwables.propagateIfInstanceOf(e.getCause(), IOException.class);
                Throwables.propagateIfInstanceOf(e.getCause(), ChannelException.class);
                throw new RE(e, "HTTP request to[%s] failed", request.getUrl());
            }
        } catch (IOException | ChannelException ex) {
            // can happen if the node is stopped.
            log.warn(ex, "Request[%s] failed.", request.getUrl());
            try {
                if (request.getUrl().getQuery() == null) {
                    request = withUrl(request, new URL(StringUtils.format("%s%s", getCurrentKnownLeader(false), request.getUrl().getPath())));
                } else {
                    request = withUrl(request, new URL(StringUtils.format("%s%s?%s", getCurrentKnownLeader(false), request.getUrl().getPath(), request.getUrl().getQuery())));
                }
                continue;
            } catch (MalformedURLException e) {
                // Not an IOException; this is our own fault.
                throw new ISE(e, "failed to build url with path[%] and query string [%s].", request.getUrl().getPath(), request.getUrl().getQuery());
            }
        }
        if (HttpResponseStatus.TEMPORARY_REDIRECT.equals(fullResponseHolder.getResponse().getStatus())) {
            String redirectUrlStr = fullResponseHolder.getResponse().headers().get("Location");
            if (redirectUrlStr == null) {
                throw new IOE("No redirect location is found in response from url[%s].", request.getUrl());
            }
            log.info("Request[%s] received redirect response to location [%s].", request.getUrl(), redirectUrlStr);
            final URL redirectUrl;
            try {
                redirectUrl = new URL(redirectUrlStr);
            } catch (MalformedURLException ex) {
                throw new IOE(ex, "Malformed redirect location is found in response from url[%s], new location[%s].", request.getUrl(), redirectUrlStr);
            }
            // update known leader location
            currentKnownLeader.set(StringUtils.format("%s://%s:%s", redirectUrl.getProtocol(), redirectUrl.getHost(), redirectUrl.getPort()));
            request = withUrl(request, redirectUrl);
        } else {
            return fullResponseHolder;
        }
    }
    throw new IOE("Retries exhausted, couldn't fulfill request to [%s].", request.getUrl());
}