Examples with Bid org.apache.beam.sdk.nexmark.model.Bid used on opensource projects

Example 6 with Bid

use of org.apache.beam.sdk.nexmark.model.Bid in project beam by apache.

the class SessionSideInputJoinTest method inputOutputSameEvents.

/**
 * A smoke test that the count of input bids and outputs are the same, to help diagnose flakiness
 * in more complex tests.
 */
@Test
@Category(NeedsRunner.class)
public void inputOutputSameEvents() throws Exception {
    NexmarkConfiguration config = NexmarkConfiguration.DEFAULT.copy();
    config.sideInputType = NexmarkUtils.SideInputType.DIRECT;
    config.numEventGenerators = 1;
    config.numEvents = 5000;
    config.sideInputRowCount = 10;
    config.sideInputNumShards = 3;
    PCollection<KV<Long, String>> sideInput = NexmarkUtils.prepareSideInput(p, config);
    try {
        PCollection<Event> input = p.apply(NexmarkUtils.batchEventsSource(config));
        PCollection<Bid> justBids = input.apply(NexmarkQueryUtil.JUST_BIDS);
        PCollection<Long> bidCount = justBids.apply("Count Bids", Count.globally());
        NexmarkQueryTransform<Bid> query = new SessionSideInputJoin(config);
        query.setSideInput(sideInput);
        PCollection<TimestampedValue<Bid>> output = (PCollection<TimestampedValue<Bid>>) input.apply(new NexmarkQuery(config, query));
        PCollection<Long> outputCount = output.apply(Window.into(new GlobalWindows())).apply("Count outputs", Count.globally());
        PAssert.that(PCollectionList.of(bidCount).and(outputCount).apply(Flatten.pCollections())).satisfies(counts -> {
            assertThat(Iterables.size(counts), equalTo(2));
            assertThat(Iterables.get(counts, 0), greaterThan(0L));
            assertThat(Iterables.get(counts, 0), equalTo(Iterables.get(counts, 1)));
            return null;
        });
        p.run();
    } finally {
        NexmarkUtils.cleanUpSideInput(config);
    }
}

Example 7 with Bid

use of org.apache.beam.sdk.nexmark.model.Bid in project beam by apache.

the class WinningBids method expand.

@Override
public PCollection<AuctionBid> expand(PCollection<Event> events) {
    // Window auctions and bids into custom auction windows. New people events will be discarded.
    // This will allow us to bring bids and auctions together irrespective of how long
    // each auction is open for.
    events = events.apply("Window", Window.into(auctionOrBidWindowFn));
    // Key auctions by their id.
    PCollection<KV<Long, Auction>> auctionsById = events.apply(NexmarkQueryUtil.JUST_NEW_AUCTIONS).apply("AuctionById:", NexmarkQueryUtil.AUCTION_BY_ID);
    // Key bids by their auction id.
    PCollection<KV<Long, Bid>> bidsByAuctionId = events.apply(NexmarkQueryUtil.JUST_BIDS).apply("BidByAuction", NexmarkQueryUtil.BID_BY_AUCTION);
    // Find the highest price valid bid for each closed auction.
    return // Join auctions and bids.
    KeyedPCollectionTuple.of(NexmarkQueryUtil.AUCTION_TAG, auctionsById).and(NexmarkQueryUtil.BID_TAG, bidsByAuctionId).apply(CoGroupByKey.create()).apply(name + ".Join", ParDo.of(new DoFn<KV<Long, CoGbkResult>, AuctionBid>() {

        private final Counter noAuctionCounter = Metrics.counter(name, "noAuction");

        private final Counter underReserveCounter = Metrics.counter(name, "underReserve");

        private final Counter noValidBidsCounter = Metrics.counter(name, "noValidBids");

        @ProcessElement
        public void processElement(ProcessContext c) {
            @Nullable Auction auction = c.element().getValue().getOnly(NexmarkQueryUtil.AUCTION_TAG, null);
            if (auction == null) {
                // We have bids without a matching auction. Give up.
                noAuctionCounter.inc();
                return;
            }
            // Find the current winning bid for auction.
            // The earliest bid with the maximum price above the reserve wins.
            Bid bestBid = null;
            for (Bid bid : c.element().getValue().getAll(NexmarkQueryUtil.BID_TAG)) {
                // Bids too late for their auction will have been
                // filtered out by the window merge function.
                checkState(bid.dateTime.compareTo(auction.expires) < 0);
                if (bid.price < auction.reserve) {
                    // Bid price is below auction reserve.
                    underReserveCounter.inc();
                    continue;
                }
                if (bestBid == null || Bid.PRICE_THEN_DESCENDING_TIME.compare(bid, bestBid) > 0) {
                    bestBid = bid;
                }
            }
            if (bestBid == null) {
                // We don't have any valid bids for auction.
                noValidBidsCounter.inc();
                return;
            }
            c.output(new AuctionBid(auction, bestBid));
        }
    }));
}

Example 8 with Bid

use of org.apache.beam.sdk.nexmark.model.Bid in project beam by apache.

the class WinningBidsSimulator method flushBidsWithoutAuctions.

/**
 * Try to match bids without auctions to auctions.
 */
private void flushBidsWithoutAuctions() {
    Iterator<Bid> itr = bidsWithoutAuctions.iterator();
    while (itr.hasNext()) {
        Bid bid = itr.next();
        if (captureBestBid(bid, false)) {
            NexmarkUtils.info("bid now accounted for: %s", bid);
            itr.remove();
        }
    }
}

Example 9 with Bid

use of org.apache.beam.sdk.nexmark.model.Bid in project beam by apache.

the class Query1 method expand.

@Override
public PCollection<Bid> expand(PCollection<Event> events) {
    return events.apply(NexmarkQueryUtil.JUST_BIDS).apply(name + ".ToEuros", ParDo.of(new DoFn<Bid, Bid>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            Bid bid = c.element();
            c.output(new Bid(bid.auction, bid.bidder, (bid.price * 89) / 100, bid.dateTime, bid.extra));
        }
    }));
}

Example 10 with Bid

use of org.apache.beam.sdk.nexmark.model.Bid in project beam by apache.

the class Query11 method expand.

@Override
public PCollection<BidsPerSession> expand(PCollection<Event> events) {
    PCollection<Long> bidders = events.apply(NexmarkQueryUtil.JUST_BIDS).apply(name + ".Rekey", ParDo.of(new DoFn<Bid, Long>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            Bid bid = c.element();
            c.output(bid.bidder);
        }
    }));
    PCollection<Long> biddersWindowed = bidders.apply(Window.<Long>into(Sessions.withGapDuration(Duration.standardSeconds(configuration.windowSizeSec))).triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(configuration.maxLogEvents))).discardingFiredPanes().withAllowedLateness(Duration.standardSeconds(configuration.occasionalDelaySec / 2)));
    return biddersWindowed.apply(Count.perElement()).apply(name + ".ToResult", ParDo.of(new DoFn<KV<Long, Long>, BidsPerSession>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            c.output(new BidsPerSession(c.element().getKey(), c.element().getValue()));
        }
    }));
}