Examples with AttributeDescriptor cz.o2.proxima.repository.AttributeDescriptor used on opensource projects

Example 16 with AttributeDescriptor

use of cz.o2.proxima.repository.AttributeDescriptor in project proxima-platform by O2-Czech-Republic.

the class TransactionResourceManager method runObservations.

/**
 * Observe all transactional families with given observer.
 *
 * @param name name of the observer (will be appended with name of the family)
 * @param requestObserver the observer (need not be synchronized)
 */
@Override
public void runObservations(String name, BiConsumer<StreamElement, Pair<Long, Object>> updateConsumer, CommitLogObserver requestObserver) {
    final CommitLogObserver effectiveObserver;
    if (isNotThreadSafe(requestObserver)) {
        effectiveObserver = requestObserver;
    } else {
        effectiveObserver = new ThreadPooledObserver(direct.getContext().getExecutorService(), requestObserver, getDeclaredParallelism(requestObserver).orElse(Runtime.getRuntime().availableProcessors()));
    }
    List<Set<String>> families = direct.getRepository().getAllEntities().filter(EntityDescriptor::isTransactional).flatMap(e -> e.getAllAttributes().stream()).filter(a -> a.getTransactionMode() != TransactionMode.NONE).map(AttributeDescriptor::getTransactionalManagerFamilies).map(Sets::newHashSet).distinct().collect(Collectors.toList());
    CountDownLatch initializedLatch = new CountDownLatch(families.size());
    families.stream().map(this::toRequestStatePair).forEach(p -> {
        DirectAttributeFamilyDescriptor requestFamily = p.getFirst();
        DirectAttributeFamilyDescriptor stateFamily = p.getSecond();
        String consumerName = name + "-" + requestFamily.getDesc().getName();
        log.info("Starting to observe family {} with URI {} and associated state family {} as {}", requestFamily, requestFamily.getDesc().getStorageUri(), stateFamily, consumerName);
        CommitLogReader reader = Optionals.get(requestFamily.getCommitLogReader());
        CachedView view = stateViews.get(stateFamily);
        if (view == null) {
            view = Optionals.get(stateFamily.getCachedView());
            Duration ttl = Duration.ofMillis(cleanupIntervalMs);
            stateViews.put(stateFamily, view);
            view.assign(view.getPartitions(), updateConsumer, ttl);
        }
        initializedLatch.countDown();
        serverObservedFamilies.put(requestFamily, reader.observe(consumerName, repartitionHookForBeingActive(stateFamily, reader.getPartitions().size(), effectiveObserver)));
    });
    ExceptionUtils.unchecked(initializedLatch::await);
}

Example 17 with AttributeDescriptor

use of cz.o2.proxima.repository.AttributeDescriptor in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperator method getCachedView.

/**
 * Retrieve {@link CachedView} for given {@link AttributeDescriptor}s.
 *
 * @param attrs the attributes to find cached view for
 * @return optional cached view
 */
public Optional<CachedView> getCachedView(Collection<AttributeDescriptor<?>> attrs) {
    boolean hasTransactions = attrs.stream().anyMatch(a -> a.getTransactionMode() != TransactionMode.NONE);
    Optional<CachedView> view = getFamilyForAttributes(attrs, DirectAttributeFamilyDescriptor::hasCachedView).flatMap(DirectAttributeFamilyDescriptor::getCachedView);
    if (hasTransactions) {
        return view.map(v -> new TransactionalCachedView(this, v));
    }
    return view;
}

Example 18 with AttributeDescriptor

use of cz.o2.proxima.repository.AttributeDescriptor in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testProxyScanWithOffset.

@Test
public void testProxyScanWithOffset() {
    EntityDescriptor proxied = repo.getEntity("proxied");
    AttributeDescriptor<?> source = proxied.getAttribute("event.*");
    Set<DirectAttributeFamilyDescriptor> proxiedFamilies = direct.getFamiliesForAttribute(source);
    direct.getWriter(source).get().write(StreamElement.upsert(proxied, source, UUID.randomUUID().toString(), "key", "event.abc", System.currentTimeMillis(), "test".getBytes(StandardCharsets.UTF_8)), (s, exc) -> {
        assertTrue(s);
    });
    direct.getWriter(source).get().write(StreamElement.upsert(proxied, source, UUID.randomUUID().toString(), "key", "event.def", System.currentTimeMillis(), "test2".getBytes(StandardCharsets.UTF_8)), (s, exc) -> {
        assertTrue(s);
    });
    List<KeyValue<?>> kvs = new ArrayList<>();
    RandomAccessReader reader = proxiedFamilies.iterator().next().getRandomAccessReader().get();
    reader.scanWildcard("key", source, reader.fetchOffset(RandomAccessReader.Listing.ATTRIBUTE, "event.abc"), 1, kvs::add);
    assertEquals(1, kvs.size());
    assertEquals("test2", new String((byte[]) kvs.get(0).getValue()));
    assertEquals(source, kvs.get(0).getAttributeDescriptor());
    assertEquals("event.def", kvs.get(0).getAttribute());
    assertEquals("key", kvs.get(0).getKey());
}

Example 19 with AttributeDescriptor

use of cz.o2.proxima.repository.AttributeDescriptor in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testApplicationOfProxyTransformOnReplicatedDataWithTransform.

@Test(timeout = 10000)
public void testApplicationOfProxyTransformOnReplicatedDataWithTransform() throws InterruptedException {
    repo.reloadConfig(true, ConfigFactory.load().withFallback(ConfigFactory.load("test-replication-proxy.conf")).resolve());
    final EntityDescriptor dummy = repo.getEntity("dummy2");
    final EntityDescriptor event = repo.getEntity("event");
    final AttributeDescriptor<Object> data = event.getAttribute("data");
    final AttributeDescriptor<Object> raw = dummy.getAttribute("_e.*", true);
    TransformationRunner.runTransformations(repo, direct);
    CountDownLatch latch = new CountDownLatch(2);
    runAttributeReplicas(direct, tmp -> latch.countDown());
    assertTrue(direct.getWriter(data).isPresent());
    OnlineAttributeWriter writer = direct.getWriter(data).get();
    long now = System.currentTimeMillis();
    writer.write(StreamElement.upsert(dummy, data, "uuid", "gw", data.getName(), now, new byte[] { 1, 2 }), (succ, exc) -> {
        assertTrue(succ);
        latch.countDown();
    });
    latch.await();
    Optional<RandomAccessReader> reader = direct.getFamiliesForAttribute(raw).stream().filter(af -> af.getDesc().getAccess().canRandomRead()).findAny().flatMap(DirectAttributeFamilyDescriptor::getRandomAccessReader);
    assertTrue(reader.isPresent());
    assertTrue(reader.get().get("gw", raw.toAttributePrefix() + (now + 1), raw).isPresent());
    assertFalse(reader.get().get("gw", raw.toAttributePrefix() + now, raw).isPresent());
}

Example 20 with AttributeDescriptor

use of cz.o2.proxima.repository.AttributeDescriptor in project proxima-platform by O2-Czech-Republic.

the class DirectDataOperatorTest method testProxyScan.

@Test
public void testProxyScan() {
    EntityDescriptor proxied = repo.getEntity("proxied");
    AttributeDescriptor<?> source = proxied.getAttribute("event.*");
    Set<DirectAttributeFamilyDescriptor> proxiedFamilies = direct.getFamiliesForAttribute(source);
    assertTrue(direct.getWriter(source).isPresent());
    direct.getWriter(source).get().write(StreamElement.upsert(proxied, source, UUID.randomUUID().toString(), "key", "event.abc", System.currentTimeMillis(), "test".getBytes(StandardCharsets.UTF_8)), (s, exc) -> {
        assertTrue(s);
    });
    direct.getWriter(source).get().write(StreamElement.upsert(proxied, source, UUID.randomUUID().toString(), "key", "event.def", System.currentTimeMillis(), "test2".getBytes(StandardCharsets.UTF_8)), (s, exc) -> {
        assertTrue(s);
    });
    List<KeyValue<?>> kvs = new ArrayList<>();
    proxiedFamilies.iterator().next().getRandomAccessReader().get().scanWildcard("key", source, kvs::add);
    assertEquals(2, kvs.size());
    assertEquals("test", new String((byte[]) kvs.get(0).getValue()));
    assertEquals(source, kvs.get(0).getAttributeDescriptor());
    assertEquals("event.abc", kvs.get(0).getAttribute());
    assertEquals("key", kvs.get(0).getKey());
    assertEquals("test2", new String((byte[]) kvs.get(1).getValue()));
    assertEquals(source, kvs.get(1).getAttributeDescriptor());
    assertEquals("event.def", kvs.get(1).getAttribute());
    assertEquals("key", kvs.get(1).getKey());
}