Examples with IntHashTable com.ociweb.pronghorn.pipe.util.hash.IntHashTable used on opensource projects

Example 1 with IntHashTable

use of com.ociweb.pronghorn.pipe.util.hash.IntHashTable in project GreenLightning by oci-pronghorn.

the class MessagePubSubStage method startup.

@Override
public void startup() {
    super.startup();
    tempSubject.initBuffers();
    int incomingPipeCount = incomingSubsAndPubsPipe.length;
    // for each pipe we must keep track of the consumed marks before sending the ack back
    int outgoingPipeCount = outgoingMessagePipes.length;
    consumedMarks = new long[incomingPipeCount][outgoingPipeCount];
    pendingAck = new boolean[incomingPipeCount];
    requiredConsumes = new int[incomingPipeCount];
    // maximum count of outgoing pipes * 2 for extra hash room
    deDupeTable = new IntHashTable(IntHashTable.computeBits(outgoingPipeCount * 2));
    this.subscriberLists = new int[initialSubscriptions * subscriberListSize];
    Arrays.fill(this.subscriberLists, (short) -1);
    // must support extraction for wild cards.
    this.localSubscriptionTrie = new TrieParser(initialSubscriptions * estimatedAvgTopicLength, 1, false, true);
    // this reader is set up for complete text only, all topics are sent in complete.
    this.localSubscriptionTrieReader = new TrieParserReader(2, true);
    this.pendingPublish = new int[subscriberListSize];
    processStartupSubscriptions(hardware.consumeStartupSubscriptions());
}

Example 2 with IntHashTable

use of com.ociweb.pronghorn.pipe.util.hash.IntHashTable in project PronghornPipes by oci-pronghorn.

the class IntHashTableTest method visitorTest.

@Test
public void visitorTest() {
    int testBits = 9;
    int extra = (1 << testBits) + 1;
    IntHashTable ht = new IntHashTable(testBits);
    int j = (1 << testBits);
    while (--j > 0) {
        assertTrue(IntHashTable.setItem(ht, j, j * 7));
    }
    // out of space
    assertFalse(IntHashTable.setItem(ht, extra, extra * 7));
    // keep array to know if every key gets visited
    final boolean[] foundValues = new boolean[(1 << testBits) - 1];
    IntHashTableVisitor visitor = new IntHashTableVisitor() {

        @Override
        public void visit(int key, int value) {
            // check that the right value was found with this key
            assertEquals(key * 7, value);
            // check that we only visit each key once
            assertFalse(foundValues[(int) key - 1]);
            foundValues[(int) key - 1] = true;
        }
    };
    ht.visit(ht, visitor);
    // error if we find any key that was not visited
    int i = foundValues.length;
    while (--i >= 0) {
        if (!foundValues[i]) {
            fail("Did not visit key " + (i + 1));
        }
    }
}

Example 3 with IntHashTable

use of com.ociweb.pronghorn.pipe.util.hash.IntHashTable in project PronghornPipes by oci-pronghorn.

the class IntHashTableTest method addToHashTable.

@Test
public void addToHashTable() {
    int testBits = 9;
    int extra = (1 << testBits) + 1;
    IntHashTable ht = new IntHashTable(testBits);
    int j = (1 << testBits);
    while (--j > 0) {
        assertTrue(IntHashTable.setItem(ht, j, j * 7));
    }
    assertFalse(IntHashTable.setItem(ht, extra, extra * 7));
    j = (1 << testBits);
    while (--j > 0) {
        assertTrue(IntHashTable.hasItem(ht, j));
        assertTrue(0 != IntHashTable.getItem(ht, j));
        assertEquals("at position " + j, j * 7, IntHashTable.getItem(ht, j));
    }
}

Example 4 with IntHashTable

use of com.ociweb.pronghorn.pipe.util.hash.IntHashTable in project GreenLightning by oci-pronghorn.

the class BuilderImpl method buildStages.

public void buildStages(MsgRuntime runtime) {
    IntHashTable subscriptionPipeLookup2 = MsgRuntime.getSubPipeLookup(runtime);
    GraphManager gm = MsgRuntime.getGraphManager(runtime);
    Pipe<NetResponseSchema>[] httpClientResponsePipes = GraphManager.allPipesOfTypeWithNoProducer(gm, NetResponseSchema.instance);
    Pipe<MessageSubscription>[] subscriptionPipes = GraphManager.allPipesOfTypeWithNoProducer(gm, MessageSubscription.instance);
    Pipe<TrafficOrderSchema>[] orderPipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, TrafficOrderSchema.instance);
    Pipe<ClientHTTPRequestSchema>[] httpClientRequestPipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, ClientHTTPRequestSchema.instance);
    Pipe<MessagePubSub>[] messagePubSub = GraphManager.allPipesOfTypeWithNoConsumer(gm, MessagePubSub.instance);
    Pipe<IngressMessages>[] ingressMessagePipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, IngressMessages.instance);
    // TODO: no longer right now that we have no cops..
    int commandChannelCount = orderPipes.length;
    int eventSchemas = 0;
    IDX_MSG = (IntHashTable.isEmpty(subscriptionPipeLookup2) && subscriptionPipes.length == 0 && messagePubSub.length == 0) ? -1 : eventSchemas++;
    IDX_NET = useNetClient(httpClientRequestPipes) ? eventSchemas++ : -1;
    // 20 seconds
    long timeout = 20_000;
    int maxGoPipeId = 0;
    Pipe<TrafficReleaseSchema>[][] masterGoOut = new Pipe[eventSchemas][0];
    Pipe<TrafficAckSchema>[][] masterAckIn = new Pipe[eventSchemas][0];
    if (IDX_MSG >= 0) {
        masterGoOut[IDX_MSG] = new Pipe[messagePubSub.length];
        masterAckIn[IDX_MSG] = new Pipe[messagePubSub.length];
    }
    if (IDX_NET >= 0) {
        masterGoOut[IDX_NET] = new Pipe[httpClientResponsePipes.length];
        masterAckIn[IDX_NET] = new Pipe[httpClientResponsePipes.length];
    }
    int copGoAck = commandChannelCount;
    // logger.info("command channel count to be checked {}",copGoAck);
    while (--copGoAck >= 0) {
        Pipe<TrafficReleaseSchema>[] goOut = new Pipe[eventSchemas];
        Pipe<TrafficAckSchema>[] ackIn = new Pipe[eventSchemas];
        // only setup the go and in pipes if the cop is used.
        if (null != orderPipes[copGoAck]) {
            int features = getFeatures(gm, orderPipes[copGoAck]);
            boolean hasConnections = false;
            if ((features & Behavior.DYNAMIC_MESSAGING) != 0) {
                hasConnections = true;
                maxGoPipeId = populateGoAckPipes(maxGoPipeId, masterGoOut, masterAckIn, goOut, ackIn, IDX_MSG);
            }
            if ((features & Behavior.NET_REQUESTER) != 0) {
                hasConnections = true;
                maxGoPipeId = populateGoAckPipes(maxGoPipeId, masterGoOut, masterAckIn, goOut, ackIn, IDX_NET);
            }
            TrafficCopStage.newInstance(gm, timeout, orderPipes[copGoAck], ackIn, goOut, runtime, this);
        } else {
            logger.info("oops get features skipped since no cops but needed for private topics");
        }
    // if (true | hasConnections) {
    // TrafficCopStage trafficCopStage = new TrafficCopStage(gm,
    // timeout, orderPipes[t],
    // ackIn, goOut,
    // runtime, this);
    // } else {
    // //this optimization can no longer be done due to the use of shutdown on command channel.
    // //    revisit this later...
    // //TODO: we can reintroduce this as long has we have a stage here which does shutdown on -1;
    // PipeCleanerStage.newInstance(gm, orderPipes[t]);
    // }
    }
    initChannelBlocker(maxGoPipeId);
    buildHTTPClientGraph(runtime, httpClientResponsePipes, httpClientRequestPipes, masterGoOut, masterAckIn);
    // TODO: only create when subscriptionPipeLookup is not empty and subscriptionPipes has zero length.
    if (IDX_MSG < 0) {
        logger.trace("saved some resources by not starting up the unused pub sub service.");
    } else {
        if (!isAllPrivateTopics) {
            // logger.info("builder created pub sub");
            createMessagePubSubStage(runtime, subscriptionPipeLookup2, ingressMessagePipes, messagePubSub, masterGoOut[IDX_MSG], masterAckIn[IDX_MSG], subscriptionPipes);
        }
    }
}

Example 5 with IntHashTable

use of com.ociweb.pronghorn.pipe.util.hash.IntHashTable in project PronghornPipes by oci-pronghorn.

the class StructRegistry method setAssoc.

private boolean setAssoc(Object localObject, int structIdx, int fieldIdx) {
    this.fieldLocals[structIdx][fieldIdx] = localObject;
    if (null == this.fieldAttachedIndex[structIdx]) {
        this.fieldAttachedIndex[structIdx] = new IntHashTable(IntHashTable.computeBits(this.fieldLocals[structIdx].length * 2));
    }
    int identityHashCode = System.identityHashCode(localObject);
    assert (0 != identityHashCode) : "can not insert null";
    assert (!IntHashTable.hasItem(this.fieldAttachedIndex[structIdx], identityHashCode)) : "These objects are too similar or was attached twice, System.identityHash must be unique. Choose different objects";
    if (IntHashTable.hasItem(this.fieldAttachedIndex[structIdx], identityHashCode)) {
        logger.warn("Unable to add object {} as an association, Another object with an identical System.identityHash is already held. Try a different object.", localObject);
        return false;
    } else {
        if (!IntHashTable.setItem(this.fieldAttachedIndex[structIdx], identityHashCode, fieldIdx)) {
            // we are out of space
            this.fieldAttachedIndex[structIdx] = IntHashTable.doubleSize(this.fieldAttachedIndex[structIdx]);
            if (!IntHashTable.setItem(this.fieldAttachedIndex[structIdx], identityHashCode, fieldIdx)) {
                throw new RuntimeException("internal error");
            }
        }
        assert (fieldIdx == IntHashTable.getItem(this.fieldAttachedIndex[structIdx], identityHashCode));
        return true;
    }
}