Search in sources :

Example 6 with ImmutableDate

use of io.pravega.common.util.ImmutableDate in project pravega by pravega.

the class SegmentStatsRecorderTest method setup.

@Before
public void setup() {
    AutoScaleProcessor processor = mock(AutoScaleProcessor.class);
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    CompletableFuture<SegmentProperties> toBeReturned = CompletableFuture.completedFuture(new SegmentProperties() {

        @Override
        public String getName() {
            return STREAM_SEGMENT_NAME;
        }

        @Override
        public boolean isSealed() {
            return false;
        }

        @Override
        public boolean isDeleted() {
            return false;
        }

        @Override
        public ImmutableDate getLastModified() {
            return null;
        }

        @Override
        public long getStartOffset() {
            return 0;
        }

        @Override
        public long getLength() {
            return 0;
        }

        @Override
        public Map<UUID, Long> getAttributes() {
            Map<UUID, Long> map = new HashMap<>();
            map.put(Attributes.SCALE_POLICY_TYPE, 0L);
            map.put(Attributes.SCALE_POLICY_RATE, 10L);
            latch.complete(null);
            return map;
        }
    });
    when(store.getStreamSegmentInfo(STREAM_SEGMENT_NAME, false, Duration.ofMinutes(1))).thenReturn(toBeReturned);
    statsRecorder = new SegmentStatsRecorderImpl(processor, store, 10000, 2, TimeUnit.SECONDS, executor, maintenanceExecutor);
}
Also used : StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) ImmutableDate(io.pravega.common.util.ImmutableDate) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) HashMap(java.util.HashMap) Map(java.util.Map) Before(org.junit.Before)

Example 7 with ImmutableDate

use of io.pravega.common.util.ImmutableDate in project pravega by pravega.

the class ChunkedSegmentStorage method getStreamSegmentInfo.

@Override
public CompletableFuture<SegmentProperties> getStreamSegmentInfo(String streamSegmentName, Duration timeout) {
    checkInitialized();
    return executeParallel(() -> {
        val traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName);
        val timer = new Timer();
        Preconditions.checkNotNull(streamSegmentName, "streamSegmentName");
        log.debug("{} getStreamSegmentInfo - started segment={}.", logPrefix, streamSegmentName);
        return tryWith(metadataStore.beginTransaction(true, streamSegmentName), txn -> txn.get(streamSegmentName).thenApplyAsync(storageMetadata -> {
            SegmentMetadata segmentMetadata = (SegmentMetadata) storageMetadata;
            checkSegmentExists(streamSegmentName, segmentMetadata);
            segmentMetadata.checkInvariants();
            val retValue = StreamSegmentInformation.builder().name(streamSegmentName).sealed(segmentMetadata.isSealed()).length(segmentMetadata.getLength()).startOffset(segmentMetadata.getStartOffset()).lastModified(new ImmutableDate(segmentMetadata.getLastModified())).build();
            log.debug("{} getStreamSegmentInfo - finished segment={} latency={}.", logPrefix, streamSegmentName, timer.getElapsedMillis());
            LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, retValue);
            return retValue;
        }, executor), executor).handleAsync((v, ex) -> {
            if (null != ex) {
                log.debug("{} getStreamSegmentInfo - exception segment={}.", logPrefix, streamSegmentName, ex);
                handleException(streamSegmentName, ex);
            }
            return v;
        }, executor);
    }, streamSegmentName);
}
Also used : lombok.val(lombok.val) Arrays(java.util.Arrays) Storage(io.pravega.segmentstore.storage.Storage) ScheduledFuture(java.util.concurrent.ScheduledFuture) StreamSegmentInformation(io.pravega.segmentstore.contracts.StreamSegmentInformation) StreamSegmentNotExistsException(io.pravega.segmentstore.contracts.StreamSegmentNotExistsException) StorageNotPrimaryException(io.pravega.segmentstore.storage.StorageNotPrimaryException) SLTS_STORAGE_USED_PERCENTAGE(io.pravega.shared.MetricsNames.SLTS_STORAGE_USED_PERCENTAGE) StorageFullException(io.pravega.segmentstore.storage.StorageFullException) ImmutableDate(io.pravega.common.util.ImmutableDate) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) SegmentHandle(io.pravega.segmentstore.storage.SegmentHandle) STORAGE_METADATA_SIZE(io.pravega.shared.MetricsNames.STORAGE_METADATA_SIZE) Duration(java.time.Duration) MetadataTransaction(io.pravega.segmentstore.storage.metadata.MetadataTransaction) ChunkMetadata(io.pravega.segmentstore.storage.metadata.ChunkMetadata) StorageMetadataWritesFencedOutException(io.pravega.segmentstore.storage.metadata.StorageMetadataWritesFencedOutException) CompletionException(java.util.concurrent.CompletionException) GuardedBy(javax.annotation.concurrent.GuardedBy) StatusFlags(io.pravega.segmentstore.storage.metadata.StatusFlags) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) StreamSegmentExistsException(io.pravega.segmentstore.contracts.StreamSegmentExistsException) ConcurrentModificationException(java.util.ConcurrentModificationException) Futures(io.pravega.common.concurrent.Futures) Getter(lombok.Getter) SegmentRollingPolicy(io.pravega.segmentstore.storage.SegmentRollingPolicy) SLTS_STORAGE_USED_BYTES(io.pravega.shared.MetricsNames.SLTS_STORAGE_USED_BYTES) Exceptions(io.pravega.common.Exceptions) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Callable(java.util.concurrent.Callable) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) SLTS_DELETE_LATENCY(io.pravega.segmentstore.storage.chunklayer.ChunkStorageMetrics.SLTS_DELETE_LATENCY) HashSet(java.util.HashSet) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) MultiKeySequentialProcessor(io.pravega.common.concurrent.MultiKeySequentialProcessor) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) SLTS_CREATE_LATENCY(io.pravega.segmentstore.storage.chunklayer.ChunkStorageMetrics.SLTS_CREATE_LATENCY) SLTS_DELETE_COUNT(io.pravega.segmentstore.storage.chunklayer.ChunkStorageMetrics.SLTS_DELETE_COUNT) LoggerHelpers(io.pravega.common.LoggerHelpers) NameUtils(io.pravega.shared.NameUtils) Iterator(java.util.Iterator) SLTS_CREATE_COUNT(io.pravega.segmentstore.storage.chunklayer.ChunkStorageMetrics.SLTS_CREATE_COUNT) Executor(java.util.concurrent.Executor) STORAGE_METADATA_NUM_CHUNKS(io.pravega.shared.MetricsNames.STORAGE_METADATA_NUM_CHUNKS) INTERNAL_SCOPE_PREFIX(io.pravega.shared.NameUtils.INTERNAL_SCOPE_PREFIX) lombok.val(lombok.val) Timer(io.pravega.common.Timer) Beta(com.google.common.annotations.Beta) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) ChunkMetadataStore(io.pravega.segmentstore.storage.metadata.ChunkMetadataStore) ReadIndexBlockMetadata(io.pravega.segmentstore.storage.metadata.ReadIndexBlockMetadata) Preconditions(com.google.common.base.Preconditions) InputStream(java.io.InputStream) SegmentMetadata(io.pravega.segmentstore.storage.metadata.SegmentMetadata) ImmutableDate(io.pravega.common.util.ImmutableDate) Timer(io.pravega.common.Timer)

Example 8 with ImmutableDate

use of io.pravega.common.util.ImmutableDate in project pravega by pravega.

the class FileSystemStorage method doGetStreamSegmentInfo.

protected SegmentProperties doGetStreamSegmentInfo(String streamSegmentName) throws IOException {
    long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName);
    PosixFileAttributes attrs = Files.readAttributes(Paths.get(config.getRoot(), streamSegmentName), PosixFileAttributes.class);
    StreamSegmentInformation information = StreamSegmentInformation.builder().name(streamSegmentName).length(attrs.size()).sealed(!(attrs.permissions().contains(OWNER_WRITE))).lastModified(new ImmutableDate(attrs.creationTime().toMillis())).build();
    LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName);
    return information;
}
Also used : StreamSegmentInformation(io.pravega.segmentstore.contracts.StreamSegmentInformation) ImmutableDate(io.pravega.common.util.ImmutableDate) PosixFileAttributes(java.nio.file.attribute.PosixFileAttributes)

Example 9 with ImmutableDate

use of io.pravega.common.util.ImmutableDate in project pravega by pravega.

the class DataRecoveryTest method testRepairLogEditOperationCreateSegmentProperties.

@Test
public void testRepairLogEditOperationCreateSegmentProperties() throws IOException {
    // Setup command object.
    STATE.set(new AdminCommandState());
    Properties pravegaProperties = new Properties();
    pravegaProperties.setProperty("pravegaservice.container.count", "1");
    pravegaProperties.setProperty("pravegaservice.clusterName", "pravega0");
    STATE.get().getConfigBuilder().include(pravegaProperties);
    CommandArgs args = new CommandArgs(List.of("0"), STATE.get());
    DurableDataLogRepairCommand command = Mockito.spy(new DurableDataLogRepairCommand(args));
    // Create a SegmentProperties object via the command logic and verify that it is equal to the expected one.
    long timestamp = System.currentTimeMillis();
    Map<AttributeId, Long> attributes = new HashMap<>();
    UUID uuid = UUID.randomUUID();
    attributes.put(AttributeId.fromUUID(uuid), 10L);
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(2L).doReturn(3L).doReturn(1L).doReturn(10L).doReturn(timestamp).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("test").doReturn(uuid.toString()).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    SegmentProperties segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(attributes).lastModified(new ImmutableDate(timestamp)).build();
    Assert.assertEquals(segmentProperties, command.createSegmentProperties());
    // Induce exceptions during the process of creating attributes to check error handling.
    segmentProperties = StreamSegmentInformation.builder().name("test").startOffset(2).length(3).storageLength(1).sealed(true).deleted(false).sealedInStorage(true).deletedInStorage(false).attributes(new HashMap<>()).lastModified(new ImmutableDate(timestamp)).build();
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(2L).doReturn(3L).doReturn(1L).doReturn(timestamp).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("test").doThrow(NumberFormatException.class).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    Assert.assertEquals(segmentProperties, command.createSegmentProperties());
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(true).doReturn(false).when(command).confirmContinue();
    Mockito.doReturn(2L).doReturn(3L).doReturn(1L).doReturn(timestamp).when(command).getLongUserInput(Mockito.any());
    Mockito.doReturn("test").doThrow(NullPointerException.class).when(command).getStringUserInput(Mockito.any());
    Mockito.doReturn(true).doReturn(true).doReturn(false).doReturn(false).when(command).getBooleanUserInput(Mockito.any());
    Assert.assertEquals(segmentProperties, command.createSegmentProperties());
}
Also used : CommandArgs(io.pravega.cli.admin.CommandArgs) ImmutableDate(io.pravega.common.util.ImmutableDate) HashMap(java.util.HashMap) AttributeId(io.pravega.segmentstore.contracts.AttributeId) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) SegmentProperties(io.pravega.segmentstore.contracts.SegmentProperties) Properties(java.util.Properties) UUID(java.util.UUID) AdminCommandState(io.pravega.cli.admin.AdminCommandState) Test(org.junit.Test)

Example 10 with ImmutableDate

use of io.pravega.common.util.ImmutableDate in project pravega by pravega.

the class StreamSegmentMetadataTests method testCopyFrom.

private void testCopyFrom(StreamSegmentMetadata baseMetadata) {
    baseMetadata.setStorageLength(1233);
    baseMetadata.updateAttributes(generateAttributes(new Random(0)));
    baseMetadata.setLastModified(new ImmutableDate());
    baseMetadata.markDeleted();
    baseMetadata.markDeletedInStorage();
    baseMetadata.markInactive();
    baseMetadata.setLastUsed(1545895);
    baseMetadata.markPinned();
    // Normal metadata copy.
    StreamSegmentMetadata newMetadata = new StreamSegmentMetadata(baseMetadata.getName(), baseMetadata.getId(), baseMetadata.getContainerId());
    newMetadata.copyFrom(baseMetadata);
    Assert.assertTrue("copyFrom copied the Active flag too.", newMetadata.isActive());
    // Force the base metadata to update its core attributes with the correct type. Do this after the copyFrom call.
    baseMetadata.refreshDerivedProperties();
    SegmentMetadataComparer.assertEquals("Metadata copy:", baseMetadata, newMetadata);
    Assert.assertEquals("Metadata copy: getLastUsed differs.", baseMetadata.getLastUsed(), newMetadata.getLastUsed());
    // Verify we cannot copy from different StreamSegments.
    AssertExtensions.assertThrows("copyFrom allowed copying from a metadata with a different Segment Name", () -> new StreamSegmentMetadata("foo", SEGMENT_ID, CONTAINER_ID).copyFrom(baseMetadata), ex -> ex instanceof IllegalArgumentException);
    AssertExtensions.assertThrows("copyFrom allowed copying from a metadata with a different Segment Id", () -> new StreamSegmentMetadata(SEGMENT_NAME, -SEGMENT_ID, CONTAINER_ID).copyFrom(baseMetadata), ex -> ex instanceof IllegalArgumentException);
}
Also used : ImmutableDate(io.pravega.common.util.ImmutableDate) Random(java.util.Random)

Aggregations

ImmutableDate (io.pravega.common.util.ImmutableDate)10 StreamSegmentInformation (io.pravega.segmentstore.contracts.StreamSegmentInformation)5 SegmentProperties (io.pravega.segmentstore.contracts.SegmentProperties)4 HashMap (java.util.HashMap)4 AttributeId (io.pravega.segmentstore.contracts.AttributeId)3 S3ObjectMetadata (com.emc.object.s3.S3ObjectMetadata)2 AccessControlList (com.emc.object.s3.bean.AccessControlList)2 AdminCommandState (io.pravega.cli.admin.AdminCommandState)2 CommandArgs (io.pravega.cli.admin.CommandArgs)2 PosixFileAttributes (java.nio.file.attribute.PosixFileAttributes)2 Beta (com.google.common.annotations.Beta)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1 Exceptions (io.pravega.common.Exceptions)1 LoggerHelpers (io.pravega.common.LoggerHelpers)1 Timer (io.pravega.common.Timer)1 Futures (io.pravega.common.concurrent.Futures)1 MultiKeySequentialProcessor (io.pravega.common.concurrent.MultiKeySequentialProcessor)1 AttributeUpdate (io.pravega.segmentstore.contracts.AttributeUpdate)1 AttributeUpdateCollection (io.pravega.segmentstore.contracts.AttributeUpdateCollection)1