Search in sources :

Example 6 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project hive by apache.

the class SemanticAnalyzer method rewriteGroupingFunctionAST.

protected static ASTNode rewriteGroupingFunctionAST(final List<ASTNode> grpByAstExprs, ASTNode targetNode, final boolean noneSet) throws SemanticException {
    final MutableBoolean visited = new MutableBoolean(false);
    final MutableBoolean found = new MutableBoolean(false);
    TreeVisitorAction action = new TreeVisitorAction() {

        @Override
        public Object pre(Object t) {
            return t;
        }

        @Override
        public Object post(Object t) {
            ASTNode root = (ASTNode) t;
            if (root.getType() == HiveParser.TOK_FUNCTION && root.getChildCount() == 2) {
                ASTNode func = (ASTNode) ParseDriver.adaptor.getChild(root, 0);
                if (func.getText().equals("grouping")) {
                    ASTNode c = (ASTNode) ParseDriver.adaptor.getChild(root, 1);
                    visited.setValue(true);
                    for (int i = 0; i < grpByAstExprs.size(); i++) {
                        ASTNode grpByExpr = grpByAstExprs.get(i);
                        if (grpByExpr.toStringTree().equals(c.toStringTree())) {
                            ASTNode child1;
                            if (noneSet) {
                                // Query does not contain CUBE, ROLLUP, or GROUPING SETS, and thus,
                                // grouping should return 0
                                child1 = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, String.valueOf(0));
                            } else {
                                // We refer to grouping_id column
                                child1 = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL");
                                ParseDriver.adaptor.addChild(child1, ParseDriver.adaptor.create(HiveParser.Identifier, VirtualColumn.GROUPINGID.getName()));
                            }
                            ASTNode child2 = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, String.valueOf(IntMath.mod(-i - 1, grpByAstExprs.size())));
                            root.setChild(1, child1);
                            root.addChild(child2);
                            found.setValue(true);
                            break;
                        }
                    }
                }
            }
            return t;
        }
    };
    ASTNode newTargetNode = (ASTNode) new TreeVisitor(ParseDriver.adaptor).visit(targetNode, action);
    if (visited.booleanValue() && !found.booleanValue()) {
        throw new SemanticException(ErrorMsg.HIVE_GROUPING_FUNCTION_EXPR_NOT_IN_GROUPBY.getMsg());
    }
    return newTargetNode;
}
Also used : TreeVisitor(org.antlr.runtime.tree.TreeVisitor) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) TreeVisitorAction(org.antlr.runtime.tree.TreeVisitorAction) HivePrivilegeObject(org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject) CalciteSemanticException(org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException)

Example 7 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project hbase by apache.

the class TestSplitWalDataLoss method test.

@Test
public void test() throws IOException, InterruptedException {
    final HRegionServer rs = testUtil.getRSForFirstRegionInTable(tableName);
    final HRegion region = (HRegion) rs.getOnlineRegions(tableName).get(0);
    HRegion spiedRegion = spy(region);
    final MutableBoolean flushed = new MutableBoolean(false);
    final MutableBoolean reported = new MutableBoolean(false);
    doAnswer(new Answer<FlushResult>() {

        @Override
        public FlushResult answer(InvocationOnMock invocation) throws Throwable {
            synchronized (flushed) {
                flushed.setValue(true);
                flushed.notifyAll();
            }
            synchronized (reported) {
                while (!reported.booleanValue()) {
                    reported.wait();
                }
            }
            rs.getWAL(region.getRegionInfo()).abortCacheFlush(region.getRegionInfo().getEncodedNameAsBytes());
            throw new DroppedSnapshotException("testcase");
        }
    }).when(spiedRegion).internalFlushCacheAndCommit(Matchers.<WAL>any(), Matchers.<MonitoredTask>any(), Matchers.<PrepareFlushResult>any(), Matchers.<Collection<Store>>any());
    // Find region key; don't pick up key for hbase:meta by mistake.
    String key = null;
    for (Map.Entry<String, Region> entry : rs.onlineRegions.entrySet()) {
        if (entry.getValue().getRegionInfo().getTable().equals(this.tableName)) {
            key = entry.getKey();
            break;
        }
    }
    rs.onlineRegions.put(key, spiedRegion);
    Connection conn = testUtil.getConnection();
    try (Table table = conn.getTable(tableName)) {
        table.put(new Put(Bytes.toBytes("row0")).addColumn(family, qualifier, Bytes.toBytes("val0")));
    }
    long oldestSeqIdOfStore = region.getOldestSeqIdOfStore(family);
    LOG.info("CHANGE OLDEST " + oldestSeqIdOfStore);
    assertTrue(oldestSeqIdOfStore > HConstants.NO_SEQNUM);
    rs.cacheFlusher.requestFlush(spiedRegion, false);
    synchronized (flushed) {
        while (!flushed.booleanValue()) {
            flushed.wait();
        }
    }
    try (Table table = conn.getTable(tableName)) {
        table.put(new Put(Bytes.toBytes("row1")).addColumn(family, qualifier, Bytes.toBytes("val1")));
    }
    long now = EnvironmentEdgeManager.currentTime();
    rs.tryRegionServerReport(now - 500, now);
    synchronized (reported) {
        reported.setValue(true);
        reported.notifyAll();
    }
    while (testUtil.getRSForFirstRegionInTable(tableName) == rs) {
        Thread.sleep(100);
    }
    try (Table table = conn.getTable(tableName)) {
        Result result = table.get(new Get(Bytes.toBytes("row0")));
        assertArrayEquals(Bytes.toBytes("val0"), result.getValue(family, qualifier));
    }
}
Also used : Table(org.apache.hadoop.hbase.client.Table) DroppedSnapshotException(org.apache.hadoop.hbase.DroppedSnapshotException) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) Connection(org.apache.hadoop.hbase.client.Connection) PrepareFlushResult(org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult) FlushResult(org.apache.hadoop.hbase.regionserver.Region.FlushResult) Put(org.apache.hadoop.hbase.client.Put) PrepareFlushResult(org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult) Result(org.apache.hadoop.hbase.client.Result) FlushResult(org.apache.hadoop.hbase.regionserver.Region.FlushResult) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Get(org.apache.hadoop.hbase.client.Get) Map(java.util.Map) Test(org.junit.Test)

Example 8 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project midpoint by Evolveum.

the class SearchEvaluator method evaluate.

public <T extends ObjectType> PipelineData evaluate(SearchExpressionType searchExpression, PipelineData input, ExecutionContext context, OperationResult globalResult) throws ScriptExecutionException {
    Validate.notNull(searchExpression.getType());
    boolean noFetch = expressionHelper.getArgumentAsBoolean(searchExpression.getParameter(), PARAM_NO_FETCH, input, context, false, "search", globalResult);
    @SuppressWarnings({ "unchecked", "raw" }) Class<T> objectClass = (Class<T>) ObjectTypes.getObjectTypeFromTypeQName(searchExpression.getType()).getClassDefinition();
    ObjectQuery objectQuery = null;
    if (searchExpression.getQuery() != null) {
        try {
            objectQuery = QueryJaxbConvertor.createObjectQuery(objectClass, searchExpression.getQuery(), prismContext);
        } catch (SchemaException e) {
            throw new ScriptExecutionException("Couldn't parse object query due to schema exception", e);
        }
    } else if (searchExpression.getSearchFilter() != null) {
        // todo resolve variable references in the filter
        objectQuery = new ObjectQuery();
        try {
            ObjectFilter filter = QueryConvertor.parseFilter(searchExpression.getSearchFilter(), objectClass, prismContext);
            objectQuery.setFilter(filter);
        } catch (SchemaException e) {
            throw new ScriptExecutionException("Couldn't parse object filter due to schema exception", e);
        }
    }
    final String variableName = searchExpression.getVariable();
    final PipelineData oldVariableValue = variableName != null ? context.getVariable(variableName) : null;
    final PipelineData outputData = PipelineData.createEmpty();
    final MutableBoolean atLeastOne = new MutableBoolean(false);
    ResultHandler<T> handler = (object, parentResult) -> {
        context.checkTaskStop();
        atLeastOne.setValue(true);
        if (searchExpression.getScriptingExpression() != null) {
            if (variableName != null) {
                context.setVariable(variableName, PipelineData.create(object.getValue()));
            }
            JAXBElement<?> childExpression = searchExpression.getScriptingExpression();
            try {
                outputData.addAllFrom(scriptingExpressionEvaluator.evaluateExpression((ScriptingExpressionType) childExpression.getValue(), PipelineData.create(object.getValue()), context, globalResult));
                globalResult.setSummarizeSuccesses(true);
                globalResult.summarize();
            } catch (ScriptExecutionException e) {
                // todo think about this
                if (context.isContinueOnAnyError()) {
                    LoggingUtils.logUnexpectedException(LOGGER, "Exception when evaluating item from search result list.", e);
                } else {
                    throw new SystemException(e);
                }
            }
        } else {
            outputData.addValue(object.getValue());
        }
        return true;
    };
    try {
        Collection<SelectorOptions<GetOperationOptions>> options = operationsHelper.createGetOptions(searchExpression.getOptions(), noFetch);
        modelService.searchObjectsIterative(objectClass, objectQuery, handler, options, context.getTask(), globalResult);
    } catch (SchemaException | ObjectNotFoundException | SecurityViolationException | CommunicationException | ConfigurationException | ExpressionEvaluationException e) {
        // TODO continue on any error?
        throw new ScriptExecutionException("Couldn't execute searchObjects operation: " + e.getMessage(), e);
    }
    if (atLeastOne.isFalse()) {
        String matching = objectQuery != null ? "matching " : "";
        // temporary hack, this will be configurable
        context.println("Warning: no " + matching + searchExpression.getType().getLocalPart() + " object found");
    }
    if (variableName != null) {
        context.setVariable(variableName, oldVariableValue);
    }
    return outputData;
}
Also used : ObjectType(com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType) OperationsHelper(com.evolveum.midpoint.model.impl.scripting.helpers.OperationsHelper) OperationResult(com.evolveum.midpoint.schema.result.OperationResult) Autowired(org.springframework.beans.factory.annotation.Autowired) Trace(com.evolveum.midpoint.util.logging.Trace) com.evolveum.midpoint.util.exception(com.evolveum.midpoint.util.exception) ExpressionHelper(com.evolveum.midpoint.model.impl.scripting.helpers.ExpressionHelper) ObjectFilter(com.evolveum.midpoint.prism.query.ObjectFilter) ResultHandler(com.evolveum.midpoint.schema.ResultHandler) SelectorOptions(com.evolveum.midpoint.schema.SelectorOptions) ScriptExecutionException(com.evolveum.midpoint.model.api.ScriptExecutionException) JAXBElement(javax.xml.bind.JAXBElement) Collection(java.util.Collection) ExecutionContext(com.evolveum.midpoint.model.impl.scripting.ExecutionContext) LoggingUtils(com.evolveum.midpoint.util.logging.LoggingUtils) QueryConvertor(com.evolveum.midpoint.prism.marshaller.QueryConvertor) PipelineData(com.evolveum.midpoint.model.impl.scripting.PipelineData) QueryJaxbConvertor(com.evolveum.midpoint.prism.query.QueryJaxbConvertor) Component(org.springframework.stereotype.Component) ScriptingExpressionType(com.evolveum.midpoint.xml.ns._public.model.scripting_3.ScriptingExpressionType) GetOperationOptions(com.evolveum.midpoint.schema.GetOperationOptions) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) ObjectTypes(com.evolveum.midpoint.schema.constants.ObjectTypes) ObjectQuery(com.evolveum.midpoint.prism.query.ObjectQuery) SearchExpressionType(com.evolveum.midpoint.xml.ns._public.model.scripting_3.SearchExpressionType) Validate(org.apache.commons.lang.Validate) TraceManager(com.evolveum.midpoint.util.logging.TraceManager) ScriptExecutionException(com.evolveum.midpoint.model.api.ScriptExecutionException) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) PipelineData(com.evolveum.midpoint.model.impl.scripting.PipelineData) ObjectFilter(com.evolveum.midpoint.prism.query.ObjectFilter) JAXBElement(javax.xml.bind.JAXBElement) ObjectQuery(com.evolveum.midpoint.prism.query.ObjectQuery) SelectorOptions(com.evolveum.midpoint.schema.SelectorOptions)

Example 9 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project midpoint by Evolveum.

the class ObjectQueryUtil method hasAllDefinitions.

public static boolean hasAllDefinitions(ObjectFilter filter) {
    final MutableBoolean hasAllDefinitions = new MutableBoolean(true);
    Visitor visitor = f -> {
        if (f instanceof ValueFilter) {
            ItemDefinition definition = ((ValueFilter<?, ?>) f).getDefinition();
            if (definition == null) {
                hasAllDefinitions.setValue(false);
            }
        }
    };
    filter.accept(visitor);
    return hasAllDefinitions.booleanValue();
}
Also used : java.util(java.util) ObjectType(com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType) com.evolveum.midpoint.prism.query(com.evolveum.midpoint.prism.query) SchemaException(com.evolveum.midpoint.util.exception.SchemaException) S_AtomicFilterExit(com.evolveum.midpoint.prism.query.builder.S_AtomicFilterExit) PolyStringNormalizer(com.evolveum.midpoint.prism.polystring.PolyStringNormalizer) PrismDefaultPolyStringNormalizer(com.evolveum.midpoint.prism.polystring.PrismDefaultPolyStringNormalizer) ShadowKindType(com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowKindType) ItemPath(com.evolveum.midpoint.prism.path.ItemPath) ResourceShadowDiscriminator(com.evolveum.midpoint.schema.ResourceShadowDiscriminator) QueryBuilder(com.evolveum.midpoint.prism.query.builder.QueryBuilder) DOMUtil(com.evolveum.midpoint.util.DOMUtil) Visitor(com.evolveum.midpoint.prism.query.Visitor) CollectionUtils(org.apache.commons.collections.CollectionUtils) PolyStringType(com.evolveum.prism.xml.ns._public.types_3.PolyStringType) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) QName(javax.xml.namespace.QName) NotNull(org.jetbrains.annotations.NotNull) Validate(org.apache.commons.lang.Validate) com.evolveum.midpoint.prism(com.evolveum.midpoint.prism) PolyString(com.evolveum.midpoint.prism.polystring.PolyString) ShadowType(com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType) QueryType(com.evolveum.prism.xml.ns._public.query_3.QueryType) Visitor(com.evolveum.midpoint.prism.query.Visitor) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean)

Example 10 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project apex-core by apache.

the class StramRecoveryTest method testWriteAheadLog.

@Test
public void testWriteAheadLog() throws Exception {
    final MutableInt flushCount = new MutableInt();
    final MutableBoolean isClosed = new MutableBoolean(false);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new FSStorageAgent(testMeta.getPath(), null));
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    Journal j = scm.getJournal();
    ByteArrayOutputStream bos = new ByteArrayOutputStream() {

        @Override
        public void flush() throws IOException {
            super.flush();
            flushCount.increment();
        }

        @Override
        public void close() throws IOException {
            super.close();
            isClosed.setValue(true);
        }
    };
    j.setOutputStream(new DataOutputStream(bos));
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    String externalId = new MockContainer(scm, o1p1.getContainer()).container.getExternalId();
    assertEquals("flush count", 1, flushCount.intValue());
    o1p1.setState(PTOperator.State.ACTIVE);
    assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    assertEquals("flush count", 2, flushCount.intValue());
    assertEquals("is closed", false, isClosed.booleanValue());
    // this will close the stream. There are 2 calls to flush() during the close() - one in Kryo Output and one
    // in FilterOutputStream
    j.setOutputStream(null);
    assertEquals("flush count", 4, flushCount.intValue());
    assertEquals("is closed", true, isClosed.booleanValue());
    // output stream is closed, so state will be changed without recording it in the journal
    o1p1.setState(PTOperator.State.INACTIVE);
    assertEquals(PTOperator.State.INACTIVE, o1p1.getState());
    assertEquals("flush count", 4, flushCount.intValue());
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    InetSocketAddress addr1 = InetSocketAddress.createUnresolved("host1", 1);
    PTContainer c1 = plan.getContainers().get(0);
    c1.setState(PTContainer.State.ALLOCATED);
    c1.host = "host1";
    c1.bufferServerAddress = addr1;
    c1.setAllocatedMemoryMB(2);
    c1.setRequiredMemoryMB(1);
    c1.setAllocatedVCores(3);
    c1.setRequiredVCores(4);
    j.setOutputStream(new DataOutputStream(bos));
    j.write(c1.getSetContainerState());
    c1.setExternalId(null);
    c1.setState(PTContainer.State.NEW);
    c1.setExternalId(null);
    c1.host = null;
    c1.bufferServerAddress = null;
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(externalId, c1.getExternalId());
    assertEquals(PTContainer.State.ALLOCATED, c1.getState());
    assertEquals("host1", c1.host);
    assertEquals(addr1, c1.bufferServerAddress);
    assertEquals(1, c1.getRequiredMemoryMB());
    assertEquals(2, c1.getAllocatedMemoryMB());
    assertEquals(3, c1.getAllocatedVCores());
    assertEquals(4, c1.getRequiredVCores());
    j.write(scm.getSetOperatorProperty("o1", "maxTuples", "100"));
    o1.setMaxTuples(10);
    j.setOutputStream(null);
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(100, o1.getMaxTuples());
    j.setOutputStream(new DataOutputStream(bos));
    scm.setOperatorProperty("o1", "maxTuples", "10");
    assertEquals(10, o1.getMaxTuples());
    o1.setMaxTuples(100);
    assertEquals(100, o1.getMaxTuples());
    j.setOutputStream(null);
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(10, o1.getMaxTuples());
    j.setOutputStream(new DataOutputStream(bos));
    scm.setPhysicalOperatorProperty(o1p1.getId(), "maxTuples", "50");
}
Also used : PhysicalPlan(com.datatorrent.stram.plan.physical.PhysicalPlan) PTOperator(com.datatorrent.stram.plan.physical.PTOperator) DataOutputStream(java.io.DataOutputStream) InetSocketAddress(java.net.InetSocketAddress) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) TestGeneratorInputOperator(com.datatorrent.stram.engine.TestGeneratorInputOperator) FSStorageAgent(com.datatorrent.common.util.FSStorageAgent) AsyncFSStorageAgent(com.datatorrent.common.util.AsyncFSStorageAgent) ByteArrayOutputStream(java.io.ByteArrayOutputStream) DataInputStream(java.io.DataInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) MutableInt(org.apache.commons.lang.mutable.MutableInt) PTContainer(com.datatorrent.stram.plan.physical.PTContainer) Test(org.junit.Test)

Aggregations

MutableBoolean (org.apache.commons.lang.mutable.MutableBoolean)27 Test (org.junit.Test)13 ArrayList (java.util.ArrayList)6 IOException (java.io.IOException)5 Map (java.util.Map)5 ExtendedBlockId (org.apache.hadoop.hdfs.ExtendedBlockId)5 Versioned (voldemort.versioning.Versioned)5 HashMap (java.util.HashMap)4 CacheVisitor (org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.CacheVisitor)4 VectorClock (voldemort.versioning.VectorClock)4 Slot (org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot)3 com.evolveum.midpoint.prism (com.evolveum.midpoint.prism)2 ItemPath (com.evolveum.midpoint.prism.path.ItemPath)2 PolyString (com.evolveum.midpoint.prism.polystring.PolyString)2 Source (com.evolveum.midpoint.repo.common.expression.Source)2 ObjectType (com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType)2 LinkedMap (org.apache.commons.collections.map.LinkedMap)2 Validate (org.apache.commons.lang.Validate)2 Configuration (org.apache.hadoop.conf.Configuration)2 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)2