Examples with Count org.apache.beam.sdk.transforms.Count used on opensource projects

Example 6 with Count

use of org.apache.beam.sdk.transforms.Count in project beam by apache.

the class DynamoDBIOIT method runRead.

/**
 * Read test dataset from DynamoDB.
 */
private void runRead() {
    int rows = env.options().getNumberOfRows();
    PCollection<Map<String, AttributeValue>> records = pipelineRead.apply("Read from DynamoDB", DynamoDBIO.read().withAwsClientsProvider(clientProvider()).withScanRequestFn(in -> buildScanRequest()).items()).apply("Flatten result", Flatten.iterables());
    PAssert.thatSingleton(records.apply("Count All", Count.globally())).isEqualTo((long) rows);
    PCollection<String> consolidatedHashcode = records.apply(MapElements.into(strings()).via(record -> record.get(COL_NAME).getS())).apply("Hash records", Combine.globally(new HashingFn()).withoutDefaults());
    PAssert.that(consolidatedHashcode).containsInAnyOrder(getExpectedHashForRowCount(rows));
    pipelineRead.run().waitUntilFinish();
}

Example 7 with Count

use of org.apache.beam.sdk.transforms.Count in project beam by apache.

the class MongoDBGridFSIOTest method testReadWithParser.

@Test
public void testReadWithParser() {
    PCollection<KV<String, Integer>> output = pipeline.apply(MongoDbGridFSIO.read().withUri("mongodb://localhost:" + port).withDatabase(DATABASE).withBucket("mapBucket").<KV<String, Integer>>withParser((input, callback) -> {
        try (final BufferedReader reader = new BufferedReader(new InputStreamReader(input.getInputStream(), StandardCharsets.UTF_8))) {
            String line = reader.readLine();
            while (line != null) {
                try (Scanner scanner = new Scanner(line.trim())) {
                    scanner.useDelimiter("\\t");
                    long timestamp = scanner.nextLong();
                    String name = scanner.next();
                    int score = scanner.nextInt();
                    callback.output(KV.of(name, score), new Instant(timestamp));
                }
                line = reader.readLine();
            }
        }
    }).withSkew(Duration.millis(3610000L)).withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())));
    PAssert.thatSingleton(output.apply("Count All", Count.globally())).isEqualTo(50100L);
    PAssert.that(output.apply("Max PerElement", Max.integersPerKey())).satisfies(input -> {
        for (KV<String, Integer> element : input) {
            assertEquals(101, element.getValue().longValue());
        }
        return null;
    });
    pipeline.run();
}