Examples with Slice com.datatorrent.netlet.util.Slice used on opensource projects

Example 56 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-malhar by apache.

the class SliceBloomFilterTest method testBloomFilterForBytes.

@Test
public void testBloomFilterForBytes() {
    final int maxSliceLength = 1000;
    Random random = new Random();
    final byte[] bytes = new byte[loop + maxSliceLength];
    random.nextBytes(bytes);
    long beginTime = System.currentTimeMillis();
    SliceBloomFilter bloomFilter = new SliceBloomFilter(100000, 0.99);
    for (int i = 0; i < loop; i++) {
        bloomFilter.put(new Slice(bytes, i, i % maxSliceLength + 1));
    }
    for (int i = 0; i < loop; i++) {
        Assert.assertTrue(bloomFilter.mightContain(new Slice(bytes, i, i % maxSliceLength + 1)));
    }
}

Example 57 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-core by apache.

the class JsonStreamCodec method toByteArray.

@Override
public Slice toByteArray(T o) {
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    try {
        mapper.writeValue(bos, o);
        byte[] bytes = bos.toByteArray();
        return new Slice(bytes, 0, bytes.length);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

Example 58 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-core by apache.

the class TupleRecorder method setup.

public void setup(Operator operator, Map<Class<?>, Class<? extends StringCodec<?>>> codecs) {
    try {
        storage.setup();
        setStartTime(System.currentTimeMillis());
        if (id == null) {
            id = String.valueOf(startTime);
        }
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        bos.write((VERSION + "\n").getBytes());
        RecordInfo recordInfo = new RecordInfo();
        recordInfo.startTime = startTime;
        recordInfo.appId = appId;
        streamCodec = new JsonStreamCodec<>(codecs);
        if (operator != null) {
            BeanInfo beanInfo = Introspector.getBeanInfo(operator.getClass());
            PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
            for (PropertyDescriptor pd : propertyDescriptors) {
                String name = pd.getName();
                Method readMethod = pd.getReadMethod();
                if (readMethod != null) {
                    readMethod.setAccessible(true);
                    try {
                        Slice f = streamCodec.toByteArray(readMethod.invoke(operator));
                        recordInfo.properties.put(name, new ObjectMapperString(f.stringValue()));
                    } catch (Throwable t) {
                        logger.warn("Cannot serialize property {} for operator {}", name, operator.getClass());
                        recordInfo.properties.put(name, null);
                    }
                }
            }
        }
        Slice f = streamCodec.toByteArray(recordInfo);
        bos.write(f.buffer, f.offset, f.length);
        bos.write("\n".getBytes());
        for (PortInfo pi : portMap.values()) {
            f = streamCodec.toByteArray(pi);
            bos.write(f.buffer, f.offset, f.length);
            bos.write("\n".getBytes());
        }
        storage.writeMetaData(bos.toByteArray());
        if (wsClient != null) {
            recordingNameTopic = "applications." + appId + ".tupleRecorder." + getStartTime();
            setupWsClient();
        }
    } catch (Exception ex) {
        logger.error("Trouble setting up tuple recorder", ex);
    }
}

Example 59 with Slice

use of com.datatorrent.netlet.util.Slice in project apex-core by apache.

the class FSStatsRecorder method recordOperators.

@Override
public void recordOperators(List<OperatorInfo> operatorList, long timestamp) throws IOException {
    for (OperatorInfo operatorInfo : operatorList) {
        FSPartFileCollection operatorStorage;
        if (!logicalOperatorStorageMap.containsKey(operatorInfo.name)) {
            operatorStorage = new FSPartFileCollection();
            operatorStorage.setBasePath(basePath + "/operators/" + operatorInfo.name);
            operatorStorage.setup();
            operatorStorage.writeMetaData((VERSION + "\n").getBytes());
            logicalOperatorStorageMap.put(operatorInfo.name, operatorStorage);
        } else {
            operatorStorage = logicalOperatorStorageMap.get(operatorInfo.name);
        }
        if (!knownOperators.contains(operatorInfo.id)) {
            knownOperators.add(operatorInfo.id);
            Map<String, Object> fieldMap = extractRecordFields(operatorInfo, "meta");
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            Slice f = streamCodec.toByteArray(fieldMap);
            bos.write(f.buffer, f.offset, f.length);
            bos.write("\n".getBytes());
            queue.add(new WriteOperation(operatorStorage, bos.toByteArray(), true));
        }
        Map<String, Object> fieldMap = extractRecordFields(operatorInfo, "stats");
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        Slice f = streamCodec.toByteArray(fieldMap);
        bos.write((operatorInfo.id + ":").getBytes());
        bos.write((String.valueOf(timestamp) + ":").getBytes());
        bos.write(f.buffer, f.offset, f.length);
        bos.write("\n".getBytes());
        queue.add(new WriteOperation(operatorStorage, bos.toByteArray(), false));
    }
}

Example 60 with Slice

use of com.datatorrent.netlet.util.Slice in project beam by apache.

the class ApexTimerInternals method fireReadyTimers.

/**
   * Fire the timers that are ready. These are the timers
   * that are registered to be triggered at a time before the current time.
   * Timer processing may register new timers, which can cause the returned
   * timestamp to be before the the current time. The caller may repeat
   * the call until such backdated timers are cleared.
   * @return minimum timestamp of registered timers.
   */
public long fireReadyTimers(long currentTime, TimerProcessor<K> timerProcessor, TimeDomain timeDomain) {
    TimerSet timers = getTimerSet(timeDomain);
    // move minTimestamp first,
    // timer additions that result from firing may modify it
    timers.minTimestamp = currentTime;
    // we keep the timers to return in a different list and launch them later
    // because we cannot prevent a trigger from registering another timer,
    // which would lead to concurrent modification exception.
    Multimap<Slice, TimerInternals.TimerData> toFire = HashMultimap.create();
    Iterator<Map.Entry<Slice, Set<Slice>>> it = timers.activeTimers.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<Slice, Set<Slice>> keyWithTimers = it.next();
        Iterator<Slice> timerIt = keyWithTimers.getValue().iterator();
        while (timerIt.hasNext()) {
            try {
                TimerData timerData = CoderUtils.decodeFromByteArray(timers.timerDataCoder, timerIt.next().buffer);
                if (timerData.getTimestamp().isBefore(currentTime)) {
                    toFire.put(keyWithTimers.getKey(), timerData);
                    timerIt.remove();
                }
            } catch (CoderException e) {
                throw new RuntimeException(e);
            }
        }
        if (keyWithTimers.getValue().isEmpty()) {
            it.remove();
        }
    }
    // fire ready timers
    if (!toFire.isEmpty()) {
        for (Slice keyBytes : toFire.keySet()) {
            try {
                K key = CoderUtils.decodeFromByteArray(keyCoder, keyBytes.buffer);
                timerProcessor.fireTimer(key, toFire.get(keyBytes));
            } catch (CoderException e) {
                throw new RuntimeException(e);
            }
        }
    }
    return timers.minTimestamp;
}