Examples with Slice com.datatorrent.netlet.util.Slice used on opensource projects

Example 91 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class ManagedStateTestUtils method validateBucketOnFileSystem.

/**
 * Validates the bucket data on the File System.
 * @param fileAccess        file access
 * @param bucketId          bucket id
 * @param unsavedBucket     bucket data to compare with.
 * @param keysPerTimeBucket num keys per time bucket
 * @throws IOException
 */
public static void validateBucketOnFileSystem(FileAccess fileAccess, long bucketId, Map<Slice, Bucket.BucketedValue> unsavedBucket, int keysPerTimeBucket) throws IOException {
    RemoteIterator<LocatedFileStatus> iterator = fileAccess.listFiles(bucketId);
    TreeMap<Slice, Slice> fromDisk = Maps.newTreeMap(new SliceComparator());
    int size = 0;
    while (iterator.hasNext()) {
        LocatedFileStatus fileStatus = iterator.next();
        String timeBucketStr = fileStatus.getPath().getName();
        if (timeBucketStr.equals(BucketsFileSystem.META_FILE_NAME) || timeBucketStr.endsWith(".tmp")) {
            // ignoring meta file
            continue;
        }
        LOG.debug("bucket {} time-bucket {}", bucketId, timeBucketStr);
        FileAccess.FileReader reader = fileAccess.getReader(bucketId, timeBucketStr);
        reader.readFully(fromDisk);
        size += keysPerTimeBucket;
        Assert.assertEquals("size of bucket " + bucketId, size, fromDisk.size());
    }
    Assert.assertEquals("size of bucket " + bucketId, unsavedBucket.size(), fromDisk.size());
    Map<Slice, Slice> testBucket = Maps.transformValues(unsavedBucket, new Function<Bucket.BucketedValue, Slice>() {

        @Override
        public Slice apply(@Nullable Bucket.BucketedValue input) {
            assert input != null;
            return input.getValue();
        }
    });
    Assert.assertEquals("data of bucket" + bucketId, testBucket, fromDisk);
}

Example 92 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class ManagedTimeUnifiedStateImplTest method testPutWithMultipleValuesForAKey.

@Test
public void testPutWithMultipleValuesForAKey() {
    Slice one = ManagedStateTestUtils.getSliceFor("1");
    testMeta.managedState.setup(testMeta.operatorContext);
    long time = System.currentTimeMillis();
    testMeta.managedState.beginWindow(0);
    testMeta.managedState.put(time, one, one);
    Slice two = ManagedStateTestUtils.getSliceFor("2");
    testMeta.managedState.put(time, one, two);
    Slice value = testMeta.managedState.getSync(time, one);
    testMeta.managedState.endWindow();
    Assert.assertEquals("value overwritten", two, value);
    testMeta.managedState.teardown();
}

Example 93 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class StateTrackerTest method testMultipleEvictions.

@Test
public void testMultipleEvictions() throws InterruptedException {
    testMeta.managedState.latch = new CountDownLatch(2);
    testMeta.managedState.setup(testMeta.operatorContext);
    Slice one = ManagedStateTestUtils.getSliceFor("1");
    testMeta.managedState.beginWindow(System.currentTimeMillis());
    testMeta.managedState.put(1, one, one);
    Slice two = ManagedStateTestUtils.getSliceFor("2");
    testMeta.managedState.put(2, two, two);
    testMeta.managedState.endWindow();
    testMeta.managedState.latch.await();
    testMeta.managedState.teardown();
    Assert.assertEquals("freed bucket", Sets.newHashSet(1L, 2L), testMeta.managedState.freedBuckets);
}

Example 94 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class StateTrackerTest method testBucketPrevention.

@Test
public void testBucketPrevention() throws InterruptedException {
    testMeta.managedState.setDurationPreventingFreeingSpace(Duration.standardDays(2));
    testMeta.managedState.setStateTracker(new MockStateTracker());
    testMeta.managedState.latch = new CountDownLatch(1);
    testMeta.managedState.setup(testMeta.operatorContext);
    Slice one = ManagedStateTestUtils.getSliceFor("1");
    testMeta.managedState.beginWindow(System.currentTimeMillis());
    testMeta.managedState.put(1, one, one);
    Slice two = ManagedStateTestUtils.getSliceFor("2");
    testMeta.managedState.put(2, two, two);
    testMeta.managedState.endWindow();
    testMeta.managedState.latch.await();
    testMeta.managedState.teardown();
    Assert.assertEquals("no buckets triggered", 0, testMeta.managedState.freedBuckets.size());
}

Example 95 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class SpillableTestUtils method checkValue.

public static <T> void checkValue(SpillableStateStore store, long bucketId, byte[] bytes, T expectedValue, int offset, Serde<T> serde) {
    Slice slice = store.getSync(bucketId, new Slice(bytes));
    if (slice == null || slice.length == 0) {
        if (expectedValue != null) {
            Assert.assertEquals(expectedValue, slice);
        } else {
            return;
        }
    }
    T string = serde.deserialize(new Input(slice.buffer, slice.offset + offset, slice.length));
    Assert.assertEquals(expectedValue, string);
}