Examples with BlobShard io.crate.blob.v2.BlobShard used on opensource projects

Example 1 with BlobShard

use of io.crate.blob.v2.BlobShard in project crate by crate.

the class BlobTransferTarget method restoreTransferStatus.

private BlobTransferStatus restoreTransferStatus(PutChunkReplicaRequest request) {
    logger.trace("Restoring transferContext for PutChunkReplicaRequest with transferId {}", request.transferId);
    DiscoveryNodes nodes = clusterService.state().getNodes();
    DiscoveryNode recipientNodeId = nodes.get(request.sourceNodeId);
    String senderNodeId = nodes.getLocalNodeId();
    BlobTransferInfoResponse transferInfoResponse = (BlobTransferInfoResponse) transportService.submitRequest(recipientNodeId, BlobHeadRequestHandler.Actions.GET_TRANSFER_INFO, new BlobInfoRequest(senderNodeId, request.transferId), TransportRequestOptions.EMPTY, new FutureTransportResponseHandler<TransportResponse>() {

        @Override
        public TransportResponse newInstance() {
            return new BlobTransferInfoResponse();
        }
    }).txGet();
    BlobShard blobShard = blobIndicesService.blobShardSafe(request.shardId());
    DigestBlob digestBlob = DigestBlob.resumeTransfer(blobShard.blobContainer(), transferInfoResponse.digest, request.transferId, request.currentPos);
    assert digestBlob != null : "DigestBlob couldn't be restored";
    BlobTransferStatus status;
    status = new BlobTransferStatus(transferInfoResponse.index, request.transferId, digestBlob);
    activeTransfers.put(request.transferId, status);
    logger.trace("Restored transferStatus for digest {} transferId: {}", transferInfoResponse.digest, request.transferId);
    transportService.submitRequest(recipientNodeId, BlobHeadRequestHandler.Actions.GET_BLOB_HEAD, new GetBlobHeadRequest(senderNodeId, request.transferId(), request.currentPos), TransportRequestOptions.EMPTY, EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
    return status;
}

Example 2 with BlobShard

use of io.crate.blob.v2.BlobShard in project crate by crate.

the class TransportDeleteBlobAction method shardOperationOnReplica.

@Override
protected void shardOperationOnReplica(DeleteBlobRequest request) {
    logger.warn("shardOperationOnReplica operating on replica but relocation is not implemented {}", request);
    BlobShard blobShard = blobIndicesService.blobShardSafe(request.shardId());
    blobShard.delete(request.id());
}

Example 3 with BlobShard

use of io.crate.blob.v2.BlobShard in project crate by crate.

the class RecoveryTests method testPrimaryRelocationWhileIndexing.

@Test
public void testPrimaryRelocationWhileIndexing() throws Exception {
    final int numberOfRelocations = 1;
    final int numberOfWriters = 2;
    final String node1 = internalCluster().startNode();
    BlobAdminClient blobAdminClient = internalCluster().getInstance(BlobAdminClient.class, node1);
    logger.trace("--> creating test index ...");
    Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build();
    blobAdminClient.createBlobTable("test", indexSettings).get();
    logger.trace("--> starting [node2] ...");
    final String node2 = internalCluster().startNode();
    ensureGreen();
    final AtomicLong idGenerator = new AtomicLong();
    final AtomicLong indexCounter = new AtomicLong();
    final AtomicBoolean stop = new AtomicBoolean(false);
    Thread[] writers = new Thread[numberOfWriters];
    final CountDownLatch stopLatch = new CountDownLatch(writers.length);
    logger.trace("--> starting {} blob upload threads", writers.length);
    final List<String> uploadedDigests = Collections.synchronizedList(new ArrayList<String>(writers.length));
    for (int i = 0; i < writers.length; i++) {
        final int indexerId = i;
        writers[i] = new Thread() {

            @Override
            public void run() {
                try {
                    logger.trace("**** starting blob upload thread {}", indexerId);
                    while (!stop.get()) {
                        long id = idGenerator.incrementAndGet();
                        String digest = uploadFile(internalCluster().client(node1), genFile(id));
                        uploadedDigests.add(digest);
                        indexCounter.incrementAndGet();
                    }
                    logger.trace("**** done indexing thread {}", indexerId);
                } catch (Exception e) {
                    logger.warn("**** failed indexing thread {}", e, indexerId);
                } finally {
                    stopLatch.countDown();
                }
            }
        };
        writers[i].setName("blob-uploader-thread");
        // dispatch threads from parent, ignoring possible leaking threads
        writers[i].setDaemon(true);
        writers[i].start();
    }
    logger.trace("--> waiting for 2 blobs to be uploaded ...");
    while (uploadedDigests.size() < 2) {
        Thread.sleep(10);
    }
    logger.trace("--> 2 blobs uploaded");
    // increase time between chunks in order to make sure that the upload is taking place while relocating
    timeBetweenChunks.set(10);
    logger.trace("--> starting relocations...");
    for (int i = 0; i < numberOfRelocations; i++) {
        String fromNode = (i % 2 == 0) ? node1 : node2;
        String toNode = node1.equals(fromNode) ? node2 : node1;
        logger.trace("--> START relocate the shard from {} to {}", fromNode, toNode);
        internalCluster().client(node1).admin().cluster().prepareReroute().add(new MoveAllocationCommand(new ShardId(BlobIndex.fullIndexName("test"), 0), fromNode, toNode)).execute().actionGet();
        ClusterHealthResponse clusterHealthResponse = internalCluster().client(node1).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForRelocatingShards(0).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
        clusterHealthResponse = internalCluster().client(node2).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForRelocatingShards(0).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
        logger.trace("--> DONE relocate the shard from {} to {}", fromNode, toNode);
    }
    logger.trace("--> done relocations");
    logger.trace("--> marking and waiting for upload threads to stop ...");
    timeBetweenChunks.set(0);
    stop.set(true);
    assertThat(stopLatch.await(60, TimeUnit.SECONDS), is(true));
    logger.trace("--> uploading threads stopped");
    logger.trace("--> expected {} got {}", indexCounter.get(), uploadedDigests.size());
    assertEquals(indexCounter.get(), uploadedDigests.size());
    BlobIndicesService blobIndicesService = internalCluster().getInstance(BlobIndicesService.class, node2);
    for (String digest : uploadedDigests) {
        BlobShard blobShard = blobIndicesService.localBlobShard(BlobIndex.fullIndexName("test"), digest);
        long length = blobShard.blobContainer().getFile(digest).length();
        assertThat(length, greaterThanOrEqualTo(1L));
    }
    for (Thread writer : writers) {
        writer.join(6000);
    }
}

Example 4 with BlobShard

use of io.crate.blob.v2.BlobShard in project crate by crate.

the class HttpBlobHandler method partialContentResponse.

private void partialContentResponse(String range, HttpRequest request, String index, final String digest) throws IOException {
    assert range != null : "Getting partial response but no byte-range is not present.";
    Matcher matcher = CONTENT_RANGE_PATTERN.matcher(range);
    if (!matcher.matches()) {
        LOGGER.warn("Invalid byte-range: {}; returning full content", range);
        fullContentResponse(request, index, digest);
        return;
    }
    BlobShard blobShard = localBlobShard(index, digest);
    final RandomAccessFile raf = blobShard.blobContainer().getRandomAccessFile(digest);
    long start;
    long end;
    try {
        try {
            start = Long.parseLong(matcher.group(1));
            if (start > raf.length()) {
                LOGGER.warn("416 Requested Range not satisfiable");
                simpleResponse(HttpResponseStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
                raf.close();
                return;
            }
            end = raf.length() - 1;
            if (!matcher.group(2).equals("")) {
                end = Long.parseLong(matcher.group(2));
            }
        } catch (NumberFormatException ex) {
            LOGGER.error("Couldn't parse Range Header", ex);
            start = 0;
            end = raf.length();
        }
        HttpResponse response = prepareResponse(PARTIAL_CONTENT);
        HttpHeaders.setContentLength(response, end - start + 1);
        response.headers().set(CONTENT_RANGE, "bytes " + start + "-" + end + "/" + raf.length());
        setDefaultGetHeaders(response);
        ctx.getChannel().write(response);
        ChannelFuture writeFuture = transferFile(digest, raf, start, end - start + 1);
        if (!HttpHeaders.isKeepAlive(request)) {
            writeFuture.addListener(ChannelFutureListener.CLOSE);
        }
    } catch (Throwable t) {
        /*
             * Make sure RandomAccessFile is closed when exception is raised.
             * In case of success, the ChannelFutureListener in "transferFile" will take care
             * that the resources are released.
             */
        raf.close();
        throw t;
    }
}

Example 5 with BlobShard

use of io.crate.blob.v2.BlobShard in project crate by crate.

the class HttpBlobHandler method fullContentResponse.

private void fullContentResponse(HttpRequest request, String index, final String digest) throws IOException {
    BlobShard blobShard = localBlobShard(index, digest);
    HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
    final RandomAccessFile raf = blobShard.blobContainer().getRandomAccessFile(digest);
    try {
        HttpHeaders.setContentLength(response, raf.length());
        setDefaultGetHeaders(response);
        LOGGER.trace("HttpResponse: {}", response);
        Channel channel = ctx.getChannel();
        channel.write(response);
        ChannelFuture writeFuture;
        if (sslEnabled) {
            // Cannot use zero-copy with HTTPS.
            writeFuture = channel.write(new ChunkedFile(raf, 0, raf.length(), 8192));
        } else {
            writeFuture = transferFile(digest, raf, 0, raf.length());
        }
        if (!HttpHeaders.isKeepAlive(request)) {
            writeFuture.addListener(ChannelFutureListener.CLOSE);
        }
    } catch (Throwable t) {
        /*
             * Make sure RandomAccessFile is closed when exception is raised.
             * In case of success, the ChannelFutureListener in "transferFile" will take care
             * that the resources are released.
             */
        raf.close();
        throw t;
    }
}