Examples with IChecked org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked used on opensource projects

Example 1 with IChecked

use of org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked in project legend-engine by finos.

the class ExecutionNodeExecutor method visit.

@Override
public Result visit(GlobalGraphFetchExecutionNode globalGraphFetchExecutionNode) {
    final Span topSpan = GlobalTracer.get().activeSpan();
    final boolean isGraphRoot = globalGraphFetchExecutionNode.parentIndex == null;
    if (isGraphRoot) {
        final boolean enableConstraints = globalGraphFetchExecutionNode.enableConstraints == null ? false : globalGraphFetchExecutionNode.enableConstraints;
        final boolean checked = globalGraphFetchExecutionNode.checked == null ? false : globalGraphFetchExecutionNode.checked;
        // Handle batching at root level
        final AtomicLong rowCount = new AtomicLong(0L);
        final AtomicLong objectCount = new AtomicLong(0L);
        final DoubleSummaryStatistics memoryStatistics = new DoubleSummaryStatistics();
        GraphFetchResult graphFetchResult = (GraphFetchResult) globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
        Stream<?> objectStream = graphFetchResult.getGraphObjectsBatchStream().map(batch -> {
            List<?> parentObjects = batch.getObjectsForNodeIndex(0);
            boolean nonEmptyObjectList = !parentObjects.isEmpty();
            ExecutionState newState = new ExecutionState(this.executionState).setGraphObjectsBatch(batch);
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty() && nonEmptyObjectList) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, newState)));
            }
            rowCount.addAndGet(batch.getRowCount());
            if (nonEmptyObjectList) {
                long currentObjectCount = objectCount.addAndGet(parentObjects.size());
                memoryStatistics.accept(batch.getTotalObjectMemoryUtilization() / (parentObjects.size() * 1.0));
                if (graphFetchResult.getGraphFetchSpan() != null) {
                    Span graphFetchSpan = graphFetchResult.getGraphFetchSpan();
                    graphFetchSpan.setTag("batchCount", memoryStatistics.getCount());
                    graphFetchSpan.setTag("objectCount", currentObjectCount);
                    graphFetchSpan.setTag("avgMemoryUtilizationInBytesPerObject", memoryStatistics.getAverage());
                }
            }
            if (!nonEmptyObjectList) {
                if (topSpan != null && rowCount.get() > 0) {
                    topSpan.setTag("lastQueryRowCount", rowCount);
                }
            }
            if (checked) {
                return parentObjects.stream().map(x -> (IChecked<?>) x).map(x -> x.getValue() instanceof Constrained ? ((Constrained<?>) x.getValue()).toChecked(x.getSource(), enableConstraints) : x).collect(Collectors.toList());
            }
            if (enableConstraints) {
                return parentObjects.stream().map(x -> x instanceof Constrained ? ((Constrained<?>) x).withConstraintsApplied() : x).collect(Collectors.toList());
            }
            return parentObjects;
        }).flatMap(Collection::stream);
        boolean realizeAsConstant = this.executionState.inAllocation && ExecutionNodeResultHelper.isResultSizeRangeSet(globalGraphFetchExecutionNode) && ExecutionNodeResultHelper.isSingleRecordResult(globalGraphFetchExecutionNode);
        if (realizeAsConstant) {
            return new ConstantResult(objectStream.findFirst().orElseThrow(() -> new RuntimeException("Constant value not found")));
        }
        return new StreamingObjectResult<>(objectStream, new PartialClassBuilder(globalGraphFetchExecutionNode), graphFetchResult);
    } else {
        GraphObjectsBatch graphObjectsBatch = this.executionState.graphObjectsBatch;
        List<?> parentObjects = graphObjectsBatch.getObjectsForNodeIndex(globalGraphFetchExecutionNode.parentIndex);
        if ((parentObjects != null) && !parentObjects.isEmpty()) {
            if (globalGraphFetchExecutionNode.xStorePropertyFetchDetails != null && globalGraphFetchExecutionNode.xStorePropertyFetchDetails.supportsCaching && this.executionState.graphFetchCaches != null) {
                graphObjectsBatch.setXStorePropertyCachesForNodeIndex(globalGraphFetchExecutionNode.localGraphFetchExecutionNode.nodeIndex, findGraphFetchCacheByTargetCrossKeys(globalGraphFetchExecutionNode));
            }
            globalGraphFetchExecutionNode.localGraphFetchExecutionNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState));
            if (globalGraphFetchExecutionNode.children != null && !globalGraphFetchExecutionNode.children.isEmpty()) {
                globalGraphFetchExecutionNode.children.forEach(c -> c.accept(new ExecutionNodeExecutor(this.profiles, this.executionState)));
            }
        }
        return new ConstantResult(parentObjects);
    }
}

Example 2 with IChecked

use of org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked in project legend-engine by finos.

the class TestXmlDeserializer method testDeserializeFirmsWithElements.

@Test
public void testDeserializeFirmsWithElements() {
    XmlReader reader = reader("<world>", "  <firms>", "    <firm>", "      <name>Acme Inc.</name>", "      <ranking>2</ranking>", "    </firm>", "    <firm>", "      <ranking>1</ranking>", "      <name>Widget Engineering</name>", "    </firm>", "    <firm>", "      <name>Globex Corp</name>", "    </firm>", "  </firms>", "</world>");
    List<IChecked<Firm>> firms = new ArrayList<>();
    DeserializeContext<Firm> context = new DeserializeContext<>(reader, firms::add);
    ReadFirmWithoutSchema firmReader = new ReadFirmWithoutSchema();
    firmReader.read(context);
    Assert.assertFalse(reader.hasNext());
    Assert.assertEquals(3, firms.size());
    Assert.assertEquals(Arrays.asList("Acme Inc.", "Widget Engineering", "Globex Corp"), firms.stream().map(IChecked::getValue).map(Firm::getName).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(2L, 1L, null), firms.stream().map(IChecked::getValue).map(Firm::getRanking).collect(Collectors.toList()));
    List<XmlDataRecord> sources = firms.stream().map(IChecked::getSource).map(XmlDataRecord.class::cast).collect(Collectors.toList());
    Assert.assertEquals(Arrays.asList("<firm><name>Acme Inc.</name><ranking>2</ranking></firm>", "<firm><ranking>1</ranking><name>Widget Engineering</name></firm>", "<firm><name>Globex Corp</name></firm>"), sources.stream().map(XmlDataRecord::getRecord).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(1L, 2L, 3L), sources.stream().map(XmlDataRecord::getNumber).collect(Collectors.toList()));
}

Example 3 with IChecked

use of org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked in project legend-engine by finos.

the class TestXmlDeserializer method testDeserializeFirmWithEmployeesWithElementsNested.

@Test
public void testDeserializeFirmWithEmployeesWithElementsNested() {
    XmlReader reader = reader("<firm name=\"Acme Inc.\" ranking=\"2\">", "  <name>Acme Inc.</name>", "  <ranking>2</ranking>", "  <employees>", "    <employee>", "      <firstName>John</firstName>", "      <lastName>Doe</lastName>", "      <dateOfBirth>1991-02-10</dateOfBirth>", "      <isAlive>true</isAlive>", "      <heightInMeters>1.78</heightInMeters>", "    </employee>", "    <employee>", "      <firstName>Fred</firstName>", "      <lastName>Bloggs</lastName>", "      <dateOfBirth>1983-12-13</dateOfBirth>", "      <isAlive>false</isAlive>", "      <heightInMeters>1.65</heightInMeters>", "    </employee>", "  </employees>", "</firm>");
    List<IChecked<Firm>> firms = new ArrayList<>();
    DeserializeContext<Firm> context = new DeserializeContext<>(reader, firms::add);
    ReadFirmWithoutSchema firmReader = new ReadFirmWithoutSchema();
    firmReader.read(context);
    Assert.assertFalse(reader.hasNext());
    Assert.assertEquals(1, firms.size());
    Firm firm = firms.get(0).getValue();
    Assert.assertEquals("Acme Inc.", firm.getName());
    Assert.assertEquals(2L, firm.getRanking().longValue());
    Assert.assertEquals(2, firm.getEmployees().size());
    Assert.assertEquals(Arrays.asList("John", "Fred"), firm.getEmployees().stream().map(Person::getFirstName).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList("Doe", "Bloggs"), firm.getEmployees().stream().map(Person::getLastName).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList("1991-02-10", "1983-12-13"), firm.getEmployees().stream().map(Person::getDateOfBirth).map(Object::toString).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(true, false), firm.getEmployees().stream().map(Person::getIsAlive).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(1.78, 1.65), firm.getEmployees().stream().map(Person::getHeightInMeters).collect(Collectors.toList()));
}

Example 4 with IChecked

use of org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked in project legend-engine by finos.

the class TestXmlDeserializer method testDeserializeFirmWithEmployeesWithElements.

@Test
public void testDeserializeFirmWithEmployeesWithElements() {
    XmlReader reader = reader("<firm name=\"Acme Inc.\" ranking=\"2\">", "  <name>Acme Inc.</name>", "  <ranking>2</ranking>", "  <employees>", "    <firstName>John</firstName>", "    <lastName>Doe</lastName>", "    <dateOfBirth>1991-02-10</dateOfBirth>", "    <isAlive>true</isAlive>", "    <heightInMeters>1.78</heightInMeters>", "  </employees>", "  <employees>", "    <firstName>Fred</firstName>", "    <lastName>Bloggs</lastName>", "    <dateOfBirth>1983-12-13</dateOfBirth>", "    <isAlive>false</isAlive>", "    <heightInMeters>1.65</heightInMeters>", "  </employees>", "</firm>");
    List<IChecked<Firm>> firms = new ArrayList<>();
    DeserializeContext<Firm> context = new DeserializeContext<>(reader, firms::add);
    ReadFirmWithoutSchema firmReader = new ReadFirmWithoutSchema();
    firmReader.read(context);
    Assert.assertFalse(reader.hasNext());
    Assert.assertEquals(1, firms.size());
    Firm firm = firms.get(0).getValue();
    Assert.assertEquals("Acme Inc.", firm.getName());
    Assert.assertEquals(2L, firm.getRanking().longValue());
    Assert.assertEquals(2, firm.getEmployees().size());
    Assert.assertEquals(Arrays.asList("John", "Fred"), firm.getEmployees().stream().map(Person::getFirstName).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList("Doe", "Bloggs"), firm.getEmployees().stream().map(Person::getLastName).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList("1991-02-10", "1983-12-13"), firm.getEmployees().stream().map(Person::getDateOfBirth).map(Object::toString).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(true, false), firm.getEmployees().stream().map(Person::getIsAlive).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(1.78, 1.65), firm.getEmployees().stream().map(Person::getHeightInMeters).collect(Collectors.toList()));
}

Example 5 with IChecked

use of org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked in project legend-engine by finos.

the class TestXmlDeserializer method testDeserializeFirmsWithAttributes.

@Test
public void testDeserializeFirmsWithAttributes() {
    XmlReader reader = reader("<world>", "  <firms>", "    <firm name=\"Acme Inc.\" ranking=\"2\"/>", "    <firm name=\"Widget Engineering\" ranking=\"1\"/>", "    <firm name=\"Globex Corp\"/>", "  </firms>", "</world>");
    List<IChecked<Firm>> firms = new ArrayList<>();
    DeserializeContext<Firm> context = new DeserializeContext<>(reader, firms::add);
    ReadFirmWithoutSchema firmReader = new ReadFirmWithoutSchema();
    firmReader.read(context);
    Assert.assertFalse(reader.hasNext());
    Assert.assertEquals(3, firms.size());
    Assert.assertEquals(Arrays.asList("Acme Inc.", "Widget Engineering", "Globex Corp"), firms.stream().map(IChecked::getValue).map(Firm::getName).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(2L, 1L, null), firms.stream().map(IChecked::getValue).map(Firm::getRanking).collect(Collectors.toList()));
    List<XmlDataRecord> sources = firms.stream().map(IChecked::getSource).map(XmlDataRecord.class::cast).collect(Collectors.toList());
    Assert.assertEquals(Arrays.asList("<firm name=\"Acme Inc.\" ranking=\"2\"/>", "<firm name=\"Widget Engineering\" ranking=\"1\"/>", "<firm name=\"Globex Corp\"/>"), sources.stream().map(XmlDataRecord::getRecord).collect(Collectors.toList()));
    Assert.assertEquals(Arrays.asList(1L, 2L, 3L), sources.stream().map(XmlDataRecord::getNumber).collect(Collectors.toList()));
}