Skip to content

Commit 76e4c46

Browse files
Add Sample.Any to the Python SDK to match Java's Sample.any
Python only had Sample.FixedSizeGlobally, which runs a uniform reservoir sample and returns a single list. Add Sample.Any, the equivalent of Java's Sample.any, which returns up to n arbitrary elements as a PCollection without the random sampling cost. If the input has fewer than n elements, all are returned. Includes unit tests on the DirectRunner and a CHANGES.md entry. Fixes #18552
1 parent da7693c commit 76e4c46

3 files changed

Lines changed: 106 additions & 0 deletions

File tree

CHANGES.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070

7171
* (Python) Removed the `envoy-data-plane` (and transitive `betterproto`) dependency; `EnvoyRateLimiter` now uses a small vendored protobuf definition instead, resolving dependency conflicts for downstream projects ([#37854](https://github.com/apache/beam/issues/37854)).
7272
* (Java) Supported acknowledge mode for JmsIO ([#39253](https://github.com/apache/beam/issues/39253)).
73+
* (Python) Added `Sample.Any`, the Python equivalent of Java's `Sample.any`, which returns up to n arbitrary elements from a PCollection ([#18552](https://github.com/apache/beam/issues/18552)).
7374
* X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
7475

7576
## Breaking Changes

sdks/python/apache_beam/transforms/combiners.py

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,32 @@ def display_data(self):
597597
def default_label(self):
598598
return 'FixedSizePerKey(%d)' % self._n
599599

600+
@with_input_types(T)
601+
@with_output_types(T)
602+
class Any(ptransform.PTransform):
603+
"""Returns up to n arbitrary elements from the input PCollection.
604+
605+
This is the Python equivalent of Java's ``Sample.any``. Unlike
606+
``FixedSizeGlobally`` it does not sample uniformly at random, and it returns
607+
the selected elements rather than a single list. If the input has fewer than
608+
n elements, all of them are returned.
609+
"""
610+
def __init__(self, n):
611+
self._n = n
612+
613+
def expand(self, pcoll):
614+
return (
615+
pcoll
616+
| core.CombineGlobally(_SampleAnyCombineFn(
617+
self._n)).without_defaults()
618+
| core.FlatMap(lambda elements: elements))
619+
620+
def display_data(self):
621+
return {'n': self._n}
622+
623+
def default_label(self):
624+
return 'Any(%d)' % self._n
625+
600626

601627
@with_input_types(T)
602628
@with_output_types(list[T])
@@ -636,6 +662,35 @@ def teardown(self):
636662
self._top_combiner.teardown()
637663

638664

665+
@with_input_types(T)
666+
@with_output_types(list[T])
667+
class _SampleAnyCombineFn(core.CombineFn):
668+
"""CombineFn that keeps up to n arbitrary elements (no random sampling)."""
669+
def __init__(self, n):
670+
super().__init__()
671+
self._n = n
672+
673+
def create_accumulator(self):
674+
return []
675+
676+
def add_input(self, accumulator, element):
677+
if len(accumulator) < self._n:
678+
accumulator.append(element)
679+
return accumulator
680+
681+
def merge_accumulators(self, accumulators):
682+
result = []
683+
for accumulator in accumulators:
684+
for element in accumulator:
685+
if len(result) >= self._n:
686+
return result
687+
result.append(element)
688+
return result
689+
690+
def extract_output(self, accumulator):
691+
return accumulator
692+
693+
639694
class _TupleCombineFnBase(core.CombineFn):
640695
def __init__(self, *combiners, merge_accumulators_batch_size=None):
641696
self._combiners = [core.CombineFn.maybe_from_callable(c) for c in combiners]

sdks/python/apache_beam/transforms/combiners_test.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ def individual_test_per_key_dd(sampleFn, n):
253253

254254
individual_test_per_key_dd(combine.Sample.FixedSizePerKey, 5)
255255
individual_test_per_key_dd(combine.Sample.FixedSizeGlobally, 5)
256+
individual_test_per_key_dd(combine.Sample.Any, 5)
256257

257258
def test_combine_globally_display_data(self):
258259
transform = beam.CombineGlobally(combine.Smallest(5))
@@ -359,6 +360,55 @@ def match(actual):
359360

360361
assert_that(result, matcher())
361362

363+
def test_sample_any(self):
364+
with TestPipeline() as pipeline:
365+
pcoll = pipeline | 'start' >> Create([1, 2, 3, 4, 5])
366+
result = pcoll | 'sample-any' >> combine.Sample.Any(3)
367+
368+
def check(actual):
369+
assert len(actual) == 3, actual
370+
for element in actual:
371+
assert element in [1, 2, 3, 4, 5], element
372+
373+
assert_that(result, check)
374+
375+
def test_sample_any_at_most_input_size(self):
376+
with TestPipeline() as pipeline:
377+
pcoll = pipeline | 'start' >> Create([1, 2])
378+
result = pcoll | 'sample-any' >> combine.Sample.Any(5)
379+
assert_that(result, equal_to([1, 2]))
380+
381+
def test_sample_any_windowed(self):
382+
with TestPipeline() as pipeline:
383+
pcoll = (
384+
pipeline
385+
| 'start' >> Create([1, 2, 3, 4])
386+
| 'timestamp' >> Map(lambda x: TimestampedValue(x, x * 10))
387+
| 'window' >> WindowInto(FixedWindows(15)))
388+
result = pcoll | 'sample-any' >> combine.Sample.Any(1)
389+
390+
def check(actual):
391+
# Timestamps 10, 20, 30, 40 fall into fixed windows [0, 15), [15, 30)
392+
# and [30, 45), holding {1}, {2} and {3, 4}. One element is sampled from
393+
# each window that has elements.
394+
assert len(actual) == 3, actual
395+
for element in actual:
396+
assert element in [1, 2, 3, 4], element
397+
398+
assert_that(result, check)
399+
400+
def test_sample_any_empty(self):
401+
with TestPipeline() as pipeline:
402+
pcoll = pipeline | 'start' >> Create([])
403+
result = pcoll | 'sample-any' >> combine.Sample.Any(3)
404+
assert_that(result, equal_to([]))
405+
406+
def test_sample_any_zero(self):
407+
with TestPipeline() as pipeline:
408+
pcoll = pipeline | 'start' >> Create([1, 2, 3])
409+
result = pcoll | 'sample-any' >> combine.Sample.Any(0)
410+
assert_that(result, equal_to([]))
411+
362412
def test_tuple_combine_fn(self):
363413
with TestPipeline() as p:
364414
result = (

0 commit comments

Comments
 (0)