Examples with IndexingProxyStreamInput org.codelibs.elasticsearch.idxproxy.stream.IndexingProxyStreamInput used on opensource projects

Example 1 with IndexingProxyStreamInput

use of org.codelibs.elasticsearch.idxproxy.stream.IndexingProxyStreamInput in project elasticsearch-indexing-proxy by codelibs.

the class RequestSender method process.

private void process(final long filePosition) {
    if (logger.isDebugEnabled()) {
        logger.debug("RequestSender(" + index + ") processes " + filePosition);
    }
    path = dataPath.resolve(String.format(dataFileFormat, filePosition) + IndexingProxyPlugin.DATA_EXTENTION);
    if (FileAccessUtils.existsFile(path)) {
        logger.info("[Sender][{}] Indexing: {}", index, path.toAbsolutePath());
        requestPosition = 0;
        try {
            processRequests(AccessController.doPrivileged((PrivilegedAction<IndexingProxyStreamInput>) () -> {
                try {
                    return new IndexingProxyStreamInput(Files.newInputStream(path), namedWriteableRegistry);
                } catch (final IOException e) {
                    throw new ElasticsearchException("Failed to read " + path.toAbsolutePath(), e);
                }
            }));
        // continue
        } catch (final Exception e) {
            retryWithError("Failed to access " + path.toAbsolutePath(), e);
        // retry
        }
    } else {
        if (logger.isDebugEnabled()) {
            logger.debug("{} does not exist.", path.toAbsolutePath());
        }
        long next = getNextValue(filePosition);
        for (long i = 0; i < senderLookupFiles; i++) {
            if (FileAccessUtils.existsFile(dataPath.resolve(String.format(dataFileFormat, next) + IndexingProxyPlugin.DATA_EXTENTION))) {
                logger.warn("[Sender][" + index + "] file_id " + filePosition + " is skipped. Moving to file_id " + next);
                processNext(next);
                return;
            // continue
            }
            next = getNextValue(next);
        }
        threadPool.schedule(senderInterval, Names.GENERIC, this);
    // retry
    }
}

Example 2 with IndexingProxyStreamInput

use of org.codelibs.elasticsearch.idxproxy.stream.IndexingProxyStreamInput in project elasticsearch-indexing-proxy by codelibs.

the class IndexingProxyService method dumpRequests.

public void dumpRequests(final int filePosition, ActionListener<String> listener) {
    final Path path = dataPath.resolve(String.format(dataFileFormat, filePosition) + IndexingProxyPlugin.DATA_EXTENTION);
    if (FileAccessUtils.existsFile(path)) {
        try (IndexingProxyStreamInput streamInput = AccessController.doPrivileged((PrivilegedAction<IndexingProxyStreamInput>) () -> {
            try {
                return new IndexingProxyStreamInput(Files.newInputStream(path), namedWriteableRegistry);
            } catch (final IOException e) {
                throw new ElasticsearchException("Failed to read " + path.toAbsolutePath(), e);
            }
        })) {
            final StringBuilder buf = new StringBuilder(10000);
            while (streamInput.available() > 0) {
                final short classType = streamInput.readShort();
                switch(classType) {
                    case RequestUtils.TYPE_DELETE:
                        DeleteRequest deleteRequest = RequestUtils.createDeleteRequest(client, streamInput, null).request();
                        buf.append(deleteRequest.toString());
                        break;
                    case RequestUtils.TYPE_DELETE_BY_QUERY:
                        DeleteByQueryRequest deleteByQueryRequest = RequestUtils.createDeleteByQueryRequest(client, streamInput, null).request();
                        buf.append(deleteByQueryRequest.toString());
                        buf.append(' ');
                        buf.append(deleteByQueryRequest.getSearchRequest().toString().replace("\n", ""));
                        break;
                    case RequestUtils.TYPE_INDEX:
                        IndexRequest indexRequest = RequestUtils.createIndexRequest(client, streamInput, null).request();
                        buf.append(indexRequest.toString());
                        break;
                    case RequestUtils.TYPE_UPDATE:
                        UpdateRequest updateRequest = RequestUtils.createUpdateRequest(client, streamInput, null).request();
                        buf.append("update {[").append(updateRequest.index()).append("][").append(updateRequest.type()).append("][").append(updateRequest.id()).append("] source[").append(updateRequest.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string()).append("]}");
                        break;
                    case RequestUtils.TYPE_UPDATE_BY_QUERY:
                        UpdateByQueryRequest updateByQueryRequest = RequestUtils.createUpdateByQueryRequest(client, streamInput, null).request();
                        buf.append(updateByQueryRequest.toString());
                        buf.append(' ');
                        buf.append(updateByQueryRequest.getSearchRequest().toString().replace("\n", ""));
                        break;
                    case RequestUtils.TYPE_BULK:
                        BulkRequest bulkRequest = RequestUtils.createBulkRequest(client, streamInput, null).request();
                        buf.append("bulk [");
                        buf.append(bulkRequest.requests().stream().map(req -> {
                            if (req instanceof UpdateRequest) {
                                UpdateRequest upreq = (UpdateRequest) req;
                                try {
                                    return "update {[" + upreq.index() + "][" + upreq.type() + "][" + upreq.id() + "] source[" + upreq.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string() + "]}";
                                } catch (IOException e) {
                                    return e.getMessage();
                                }
                            } else {
                                return req.toString();
                            }
                        }).collect(Collectors.joining(",")));
                        buf.append("]");
                        break;
                    default:
                        listener.onFailure(new ElasticsearchException("Unknown request type: " + classType));
                }
                buf.append('\n');
            }
            listener.onResponse(buf.toString());
        } catch (IOException e) {
            listener.onFailure(e);
        }
    } else {
        listener.onFailure(new ElasticsearchException("The data file does not exist: " + dataPath));
    }
}