Examples with AuctionBid org.apache.beam.sdk.nexmark.model.AuctionBid used on opensource projects

Example 1 with AuctionBid

use of org.apache.beam.sdk.nexmark.model.AuctionBid in project beam by apache.

the class WinningBidsSimulator method nextWinningBid.

/**
 * Return the next winning bid for an expired auction relative to {@code timestamp}. Return null
 * if no more winning bids, in which case all expired auctions will have been removed from our
 * state. Retire auctions in order of expire time.
 */
@Nullable
private TimestampedValue<AuctionBid> nextWinningBid(Instant timestamp) {
    Map<Instant, List<Long>> toBeRetired = new TreeMap<>();
    for (Map.Entry<Long, Auction> entry : openAuctions.entrySet()) {
        if (entry.getValue().expires.compareTo(timestamp) <= 0) {
            List<Long> idsAtTime = toBeRetired.computeIfAbsent(entry.getValue().expires, k -> new ArrayList<>());
            idsAtTime.add(entry.getKey());
        }
    }
    for (Map.Entry<Instant, List<Long>> entry : toBeRetired.entrySet()) {
        for (long id : entry.getValue()) {
            Auction auction = openAuctions.get(id);
            NexmarkUtils.info("retiring auction: %s", auction);
            openAuctions.remove(id);
            Bid bestBid = bestBids.get(id);
            if (bestBid != null) {
                TimestampedValue<AuctionBid> result = TimestampedValue.of(new AuctionBid(auction, bestBid), auction.expires);
                NexmarkUtils.info("winning: %s", result);
                return result;
            }
        }
    }
    return null;
}

Example 2 with AuctionBid

use of org.apache.beam.sdk.nexmark.model.AuctionBid in project beam by apache.

the class Query4 method expand.

@Override
public PCollection<CategoryPrice> expand(PCollection<Event> events) {
    PCollection<AuctionBid> winningBids = events.apply(Filter.by(new AuctionOrBid())).apply(new WinningBids(name + ".WinningBids", configuration));
    // Monitor winning bids
    winningBids = winningBids.apply(name + ".WinningBidsMonitor", winningBidsMonitor.getTransform());
    return winningBids.apply(name + ".Rekey", ParDo.of(new DoFn<AuctionBid, KV<Long, Long>>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            Auction auction = c.element().auction;
            Bid bid = c.element().bid;
            c.output(KV.of(auction.category, bid.price));
        }
    })).apply(Window.into(SlidingWindows.of(Duration.standardSeconds(configuration.windowSizeSec)).every(Duration.standardSeconds(configuration.windowPeriodSec)))).apply(Mean.<Long, Long>perKey().withHotKeyFanout(configuration.fanout)).apply(name + ".Project", ParDo.of(new DoFn<KV<Long, Double>, CategoryPrice>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            c.output(new CategoryPrice(c.element().getKey(), Math.round(c.element().getValue()), c.pane().isLast()));
        }
    }));
}

Example 3 with AuctionBid

use of org.apache.beam.sdk.nexmark.model.AuctionBid in project beam by apache.

the class WinningBids method expand.

@Override
public PCollection<AuctionBid> expand(PCollection<Event> events) {
    // Window auctions and bids into custom auction windows. New people events will be discarded.
    // This will allow us to bring bids and auctions together irrespective of how long
    // each auction is open for.
    events = events.apply("Window", Window.into(auctionOrBidWindowFn));
    // Key auctions by their id.
    PCollection<KV<Long, Auction>> auctionsById = events.apply(NexmarkQueryUtil.JUST_NEW_AUCTIONS).apply("AuctionById:", NexmarkQueryUtil.AUCTION_BY_ID);
    // Key bids by their auction id.
    PCollection<KV<Long, Bid>> bidsByAuctionId = events.apply(NexmarkQueryUtil.JUST_BIDS).apply("BidByAuction", NexmarkQueryUtil.BID_BY_AUCTION);
    // Find the highest price valid bid for each closed auction.
    return // Join auctions and bids.
    KeyedPCollectionTuple.of(NexmarkQueryUtil.AUCTION_TAG, auctionsById).and(NexmarkQueryUtil.BID_TAG, bidsByAuctionId).apply(CoGroupByKey.create()).apply(name + ".Join", ParDo.of(new DoFn<KV<Long, CoGbkResult>, AuctionBid>() {

        private final Counter noAuctionCounter = Metrics.counter(name, "noAuction");

        private final Counter underReserveCounter = Metrics.counter(name, "underReserve");

        private final Counter noValidBidsCounter = Metrics.counter(name, "noValidBids");

        @ProcessElement
        public void processElement(ProcessContext c) {
            @Nullable Auction auction = c.element().getValue().getOnly(NexmarkQueryUtil.AUCTION_TAG, null);
            if (auction == null) {
                // We have bids without a matching auction. Give up.
                noAuctionCounter.inc();
                return;
            }
            // Find the current winning bid for auction.
            // The earliest bid with the maximum price above the reserve wins.
            Bid bestBid = null;
            for (Bid bid : c.element().getValue().getAll(NexmarkQueryUtil.BID_TAG)) {
                // Bids too late for their auction will have been
                // filtered out by the window merge function.
                checkState(bid.dateTime.compareTo(auction.expires) < 0);
                if (bid.price < auction.reserve) {
                    // Bid price is below auction reserve.
                    underReserveCounter.inc();
                    continue;
                }
                if (bestBid == null || Bid.PRICE_THEN_DESCENDING_TIME.compare(bid, bestBid) > 0) {
                    bestBid = bid;
                }
            }
            if (bestBid == null) {
                // We don't have any valid bids for auction.
                noValidBidsCounter.inc();
                return;
            }
            c.output(new AuctionBid(auction, bestBid));
        }
    }));
}

Example 4 with AuctionBid

use of org.apache.beam.sdk.nexmark.model.AuctionBid in project beam by apache.

the class WinningBidsSimulator method run.

@Override
protected void run() {
    if (lastTimestamp.compareTo(BoundedWindow.TIMESTAMP_MIN_VALUE) > 0) {
        // We may have finally seen the auction a bid was intended for.
        flushBidsWithoutAuctions();
        TimestampedValue<AuctionBid> result = nextWinningBid(lastTimestamp);
        if (result != null) {
            addResult(result);
            return;
        }
    }
    TimestampedValue<Event> timestampedEvent = nextInput();
    if (timestampedEvent == null) {
        // No more events. Flush any still open auctions.
        TimestampedValue<AuctionBid> result = nextWinningBid(BoundedWindow.TIMESTAMP_MAX_VALUE);
        if (result == null) {
            // We are done.
            allDone();
            return;
        }
        addResult(result);
        return;
    }
    Event event = timestampedEvent.getValue();
    if (event.newPerson != null) {
        // Ignore new person events.
        return;
    }
    lastTimestamp = timestampedEvent.getTimestamp();
    if (event.newAuction != null) {
        // Add this new open auction to our state.
        openAuctions.put(event.newAuction.id, event.newAuction);
    } else {
        if (!captureBestBid(event.bid, true)) {
            // We don't know what to do with this bid yet.
            NexmarkUtils.info("bid not yet accounted for: %s", event.bid);
            bidsWithoutAuctions.add(event.bid);
        }
    }
// Keep looking for winning bids.
}