Examples with ByteArraySegment io.pravega.common.util.ByteArraySegment used on opensource projects

Example 1 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class DataFrameOutputStream method write.

@Override
public void write(byte[] data, int offset, int length) throws IOException {
    Exceptions.checkNotClosed(this.closed, this);
    Preconditions.checkState(this.currentFrame != null, "No current frame exists. Most likely no record is started.");
    int totalBytesWritten = 0;
    int attemptsWithNoProgress = 0;
    while (totalBytesWritten < length) {
        int bytesWritten = this.currentFrame.append(new ByteArraySegment(data, offset + totalBytesWritten, length - totalBytesWritten));
        attemptsWithNoProgress = bytesWritten == 0 ? attemptsWithNoProgress + 1 : 0;
        if (attemptsWithNoProgress > 1) {
            // We had two consecutive attempts to write to a frame with no progress made.
            throw new IOException("Unable to make progress in serializing to DataFrame.");
        }
        // Update positions.
        totalBytesWritten += bytesWritten;
        if (totalBytesWritten < length) {
            // We were only able to write this partially because the current frame is full. Seal it and create a new one.
            this.currentFrame.endEntry(false);
            flush();
            createNewFrame();
            startNewRecordInCurrentFrame(false);
        }
    }
}

Example 2 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class S3Mock method putObject.

public void putObject(String bucketName, String key, Range range, Object content) {
    String objectName = getObjectName(bucketName, key);
    synchronized (this.objects) {
        ObjectData od = this.objects.get(objectName);
        if (od == null) {
            throw new S3Exception(String.format("Object '%s/%s' does not exist.", bucketName, key), 404, "NoSuchKey", key);
        }
        final int startOffset = (int) (long) range.getFirst();
        final int length = (int) (range.getLast() + 1 - range.getFirst());
        BufferView objectContent = od.content;
        if (startOffset < objectContent.getLength()) {
            objectContent = new ByteArraySegment(objectContent.slice(0, startOffset).getCopy());
        } else if (startOffset > objectContent.getLength()) {
            throw new S3Exception("", HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE, "InvalidRange", "");
        }
        InputStream source = (InputStream) content;
        byte[] buffer = new byte[length];
        try {
            int copyLength = StreamHelpers.readAll(source, buffer, 0, length);
            assert copyLength == length;
        } catch (IOException ex) {
            throw new S3Exception("Copy error", HttpStatus.SC_INTERNAL_SERVER_ERROR);
        }
        od.content = BufferView.wrap(Arrays.asList(objectContent, new ByteArraySegment(buffer)));
    }
}

Example 3 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class S3Mock method putObject.

public PutObjectResult putObject(PutObjectRequest request) {
    // Fix passed in object metadata.
    if (request.getObjectMetadata() != null) {
        request.setObjectMetadata(new S3ObjectMetadata().withContentType(request.getObjectMetadata().getContentType()).withContentLength(request.getObjectMetadata().getContentLength()));
    }
    String objectName = getObjectName(request.getBucketName(), request.getKey());
    synchronized (this.objects) {
        if (this.objects.containsKey(objectName)) {
            throw new S3Exception(String.format("Object '%s/%s' exists.", request.getBucketName(), request.getKey()), 0, "PreconditionFailed", "");
        }
        if (null != request.getObject()) {
            try {
                val bufferView = new ByteArraySegment(((InputStream) request.getObject()).readAllBytes());
                this.objects.put(objectName, new ObjectData(bufferView, request.getAcl()));
            } catch (IOException ex) {
                throw new S3Exception("Copy error", HttpStatus.SC_INTERNAL_SERVER_ERROR);
            }
        } else {
            this.objects.put(objectName, new ObjectData(BufferView.empty(), request.getAcl()));
        }
        return new PutObjectResult();
    }
}

Example 4 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class FileSystemStorageTest method testReplaceRecoveryWithTempFileOnly.

/**
 * Tests the behavior of {@link FileSystemStorage} when a previous invocation of {@link FileSystemStorage#replace}
 * has only partially completed (created a Temp file and deleted the original file).
 */
@Test
public void testReplaceRecoveryWithTempFileOnly() throws Exception {
    val segmentName = "BaseSegment";
    val tempSegmentName = getTempSegmentName(segmentName);
    val segmentHandle = FileSystemSegmentHandle.writeHandle(segmentName);
    val data1 = new ByteArraySegment(new byte[] { 1, 2 });
    val data2 = new ByteArraySegment(new byte[] { 3, 4, 5 });
    // We use "base" for direct access to files.
    @Cleanup val base = new FileSystemStorage(this.adapterConfig);
    Assert.assertFalse(base.supportsReplace());
    AssertExtensions.assertThrows("", () -> base.replace(segmentHandle, data1), ex -> ex instanceof UnsupportedOperationException);
    // Create a FileSystemStorageWithReplace, which is what we are testing.
    @Cleanup val s = (FileSystemStorage.FileSystemStorageWithReplace) base.withReplaceSupport();
    // Exists (does not perform auto-recovery).
    withPartialReplace(segmentName, null, data2, base, () -> {
        Assert.assertTrue(s.exists(segmentName));
        Assert.assertFalse(base.exists(segmentName));
        Assert.assertTrue(base.exists(tempSegmentName));
    });
    // GetStreamSegmentInfo (performs auto-recovery).
    withPartialReplace(segmentName, null, data2, base, () -> {
        val si = s.getStreamSegmentInfo(segmentName);
        Assert.assertEquals(data2.getLength(), si.getLength());
        Assert.assertEquals(data2.getLength(), base.getStreamSegmentInfo(segmentName).getLength());
        Assert.assertFalse(base.exists(tempSegmentName));
    });
    // OpenRead (performs auto-recovery) + Read.
    withPartialReplace(segmentName, null, data2, base, () -> {
        val rh = s.openRead(segmentName);
        val rb = new byte[data2.getLength()];
        s.read(rh, 0, rb, 0, rb.length);
        Assert.assertArrayEquals(data2.array(), rb);
        Assert.assertEquals(data2.getLength(), base.getStreamSegmentInfo(segmentName).getLength());
        Assert.assertFalse(base.exists(tempSegmentName));
    });
    // OpenWrite (performs auto-recovery) + Write + Seal.
    withPartialReplace(segmentName, null, data2, base, () -> {
        val wh = s.openWrite(segmentName);
        s.write(wh, data2.getLength(), data1.getReader(), data1.getLength());
        s.seal(wh);
        val si = s.getStreamSegmentInfo(segmentName);
        Assert.assertTrue(si.isSealed());
        Assert.assertEquals(data2.getLength() + data1.getLength(), si.getLength());
        Assert.assertEquals(data2.getLength() + data1.getLength(), base.getStreamSegmentInfo(segmentName).getLength());
        Assert.assertFalse(base.exists(tempSegmentName));
    });
    // Concat (performs auto-recovery on source segment).
    withPartialReplace(segmentName, null, data2, base, () -> {
        base.seal(base.openWrite(getTempSegmentName(segmentName)));
        val targetSegment = s.create("Target");
        try {
            s.concat(targetSegment, 0, segmentName);
            val si = s.getStreamSegmentInfo(targetSegment.getSegmentName());
            Assert.assertFalse(si.isSealed());
            Assert.assertEquals(data2.getLength(), si.getLength());
            Assert.assertFalse(base.exists(segmentName));
            Assert.assertFalse(base.exists(tempSegmentName));
        } finally {
            s.delete(targetSegment);
        }
    });
    // Replace (with fake handle) - perform auto-recovery and subsequent replace.
    withPartialReplace(segmentName, null, data2, base, () -> {
        s.replace(segmentHandle, data1);
        val si = s.getStreamSegmentInfo(segmentName);
        Assert.assertEquals(data1.getLength(), si.getLength());
        Assert.assertEquals(data1.getLength(), base.getStreamSegmentInfo(segmentName).getLength());
        val rb = new byte[data1.getLength()];
        s.read(segmentHandle, 0, rb, 0, rb.length);
        Assert.assertArrayEquals(data1.array(), rb);
        Assert.assertFalse(base.exists(tempSegmentName));
    });
    // Delete (should clean up everything).
    withPartialReplace(segmentName, null, data2, base, () -> {
        s.delete(segmentHandle);
        Assert.assertFalse(s.exists(segmentName));
        Assert.assertFalse(base.exists(segmentName));
        Assert.assertFalse(base.exists(tempSegmentName));
    });
    // Create (performs auto-recovery).
    withPartialReplace(segmentName, null, data2, base, () -> {
        AssertExtensions.assertThrows("", () -> s.create(segmentName), ex -> ex instanceof StreamSegmentExistsException);
        Assert.assertTrue(s.exists(segmentName));
        Assert.assertTrue(base.exists(segmentName));
        Assert.assertFalse(base.exists(tempSegmentName));
    });
}

Example 5 with ByteArraySegment

use of io.pravega.common.util.ByteArraySegment in project pravega by pravega.

the class S3Mock method putObject.

PutObjectResponse putObject(PutObjectRequest request, RequestBody requestBody) {
    String objectName = getObjectName(request.bucket(), request.key());
    synchronized (this.objects) {
        if (this.objects.containsKey(objectName)) {
            throw S3Exception.builder().build();
        }
        if (null != requestBody) {
            try (val inputStream = requestBody.contentStreamProvider().newStream()) {
                val bufferView = new ByteArraySegment(inputStream.readAllBytes());
                this.objects.put(objectName, new ObjectData(bufferView, request.acl()));
            } catch (IOException ex) {
                throw getException("Copy error", "Copy error", HttpStatus.SC_INTERNAL_SERVER_ERROR);
            }
        } else {
            this.objects.put(objectName, new ObjectData(BufferView.empty(), request.acl()));
        }
        return PutObjectResponse.builder().build();
    }
}