Remove rules_python dependency (#1810)

Author: yongtang

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
         run: |
           set -x -e
           git log --pretty -1
-          sudo python3 -m pip install -U numpy
+          sudo python3 -m pip install -U numpy black pyupgrade
           bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade black clang
 
   lint-docs:

.github/workflows/reviewdog.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: |
-          sudo python3 -m pip install -U numpy pip
+          sudo python3 -m pip install -U numpy pip black pyupgrade
           bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- bazel
       - uses: reviewdog/action-suggester@v1
   black:
@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: |
-          sudo python3 -m pip install -U numpy pip
+          sudo python3 -m pip install -U numpy pip black pyupgrade
           bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- black
       - uses: reviewdog/action-suggester@v1
   clang:
@@ -30,7 +30,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: |
-          sudo python3 -m pip install -U numpy pip
+          sudo python3 -m pip install -U numpy pip black pyupgrade
           bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- clang
       - uses: reviewdog/action-suggester@v1
   pyupgrade:
@@ -39,6 +39,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: |
-          sudo python3 -m pip install -U numpy pip
+          sudo python3 -m pip install -U numpy pip black pyupgrade
           bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- pyupgrade
       - uses: reviewdog/action-suggester@v1

WORKSPACE

@@ -91,27 +91,6 @@ switched_rules_by_language(
     grpc = True,
 )
 
-# Note rules_python is placed earlier as tensorflow's version is older
-http_archive(
-    name = "rules_python",
-    sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
-    urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
-        "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
-    ],
-)
-
-load("@rules_python//python:pip.bzl", "pip_import")
-
-pip_import(
-    name = "lint_dependencies",
-    requirements = "//tools/lint:requirements.txt",
-)
-
-load("@lint_dependencies//:requirements.bzl", "pip_install")
-
-pip_install()
-
 http_archive(
     name = "org_tensorflow",
     sha256 = "e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc",

tensorflow_io/python/experimental/filter_ops.py

@@ -188,8 +188,8 @@ def gabor(
                 1.0
                 / np.pi
                 * np.sqrt(np.log(2) / 2.0)
-                * (2.0 ** b + 1)
-                / (2.0 ** b - 1),
+                * (2.0**b + 1)
+                / (2.0**b - 1),
                 dtype.real_dtype,
             )
             / freq
@@ -220,7 +220,7 @@ def gabor(
     rotx = y * tf.math.sin(theta) + x * tf.math.cos(theta)
     roty = y * tf.math.cos(theta) - x * tf.math.sin(theta)
 
-    g = tf.math.exp(-0.5 * (rotx ** 2 / sigma_x ** 2 + roty ** 2 / sigma_y ** 2))
+    g = tf.math.exp(-0.5 * (rotx**2 / sigma_x**2 + roty**2 / sigma_y**2))
     g = g / (2 * np.pi * sigma_x * sigma_y)
     g = tf.cast(g, dtype) * tf.exp(
         tf.cast(1j, dtype) * tf.cast(2 * np.pi * freq * rotx + offset, dtype)

tensorflow_io/python/experimental/openexr_io_tensor_ops.py

@@ -44,7 +44,7 @@ def __init__(self, filename, internal=False):
             shapes, dtypes, channels = core_ops.io_decode_exr_info(data)
             parts = []
             index = 0
-            for (shape, dtypes, channels) in zip(
+            for shape, dtypes, channels in zip(
                 shapes.numpy(), dtypes.numpy(), channels.numpy()
             ):
                 # Remove trailing 0 from dtypes

tensorflow_io/python/experimental/serialization_ops.py

@@ -19,6 +19,7 @@
 import tensorflow as tf
 from tensorflow_io.python.ops import core_ops
 
+
 # _NamedTensorSpec allows adding a `named` key while traversing,
 # so that it is possible to build up the `/R/Foo` JSON Pointers.
 class _NamedTensorSpec(tf.TensorSpec):

tensorflow_io/python/ops/arrow_io_tensor_ops.py

@@ -83,7 +83,6 @@ def _extract_table_arrays(table):
     # Pad buffer addrs, sizes and array lengths so inputs are rectangular
     num_columns = len(array_buffer_sizes)
     for i in range(num_columns):
-
         # pad chunk list with empty lists that will be padded with null bufs
         if len(array_buffer_sizes[i]) < max_num_chunks:
             array_buffer_sizes[i].extend(
@@ -94,7 +93,6 @@ def _extract_table_arrays(table):
 
         num_chunks = len(array_buffer_sizes[i])
         for j in range(num_chunks):
-
             # pad buffer addr, size, and array length lists
             if len(array_buffer_sizes[i][j]) < max_num_bufs:
                 array_buffer_sizes[i][j].extend(
@@ -140,7 +138,6 @@ def wrap_func(inp):
     def init_resource_op(self, table):
         """Initialize the resource op with a pyarrow.Table"""
         with tf.name_scope("ArrowIOTensor") as scope:
-
             # Serialize the schema to send to the kernel
             schema_buffer = table.schema.serialize()
 
@@ -241,7 +238,6 @@ class ArrowIOTensor(io_tensor_ops._TableIOTensor):  # pylint: disable=protected-
     # Constructor (private)
     # =============================================================================
     def __init__(self, table, spec=None, internal=False):
-
         # Get or build a new ArrowIOResource
         if isinstance(table, ArrowIOResource):
             arrow_resource = table
@@ -292,7 +288,7 @@ def __init__(self, table, spec=None, internal=False):
             ]
 
             shapes = []
-            for (column_index, column) in columns:
+            for column_index, column in columns:
                 shape, _ = core_ops.io_arrow_readable_spec(
                     resource, column_index, column
                 )
@@ -306,7 +302,7 @@ def __init__(self, table, spec=None, internal=False):
             ]
 
             elements = []
-            for ((column_index, column), entry, shape) in zip(columns, entries, shapes):
+            for (column_index, column), entry, shape in zip(columns, entries, shapes):
                 function = _ArrowIOTensorComponentFunction(
                     core_ops.io_arrow_readable_read,
                     resource,

tensorflow_io/python/ops/avro_dataset_ops.py

@@ -75,7 +75,7 @@ def __init__(self, filename, schema, columns=None, internal=True):
                 )
                 columns_function.append(function)
 
-            for (column, function) in zip(columns, columns_function):
+            for column, function in zip(columns, columns_function):
                 column_dataset = tf.compat.v2.data.Dataset.range(
                     0, sys.maxsize, capacity
                 )

tensorflow_io/python/ops/data_ops.py

@@ -16,6 +16,7 @@
 
 import tensorflow as tf
 
+
 # Note: BaseDataset could be used by Dataset implementations
 # that does not utilize DataInput implementation.
 class BaseDataset(tf.compat.v2.data.Dataset):

tensorflow_io/python/ops/json_dataset_ops.py

@@ -75,7 +75,7 @@ def __init__(self, filename, columns=None, mode=None, internal=True):
                 )
                 columns_function.append(function)
 
-            for (column, function) in zip(columns, columns_function):
+            for column, function in zip(columns, columns_function):
                 column_dataset = tf.compat.v2.data.Dataset.range(
                     0, sys.maxsize, capacity
                 )

tensorflow_io/python/ops/orc_dataset_ops.py

@@ -69,7 +69,7 @@ def __init__(self, filename, columns=None, internal=True, **kwargs):
                 )
                 columns_function.append(function)
 
-            for (column, function) in zip(columns, columns_function):
+            for column, function in zip(columns, columns_function):
                 column_dataset = tf.compat.v2.data.Dataset.range(
                     0, sys.maxsize, capacity
                 )

tensorflow_io/python/ops/prometheus_dataset_ops.py

@@ -56,6 +56,7 @@ def __init__(
                             )
                         spec_by_job[instance] = spec_by_instance
                     spec[job] = spec_by_job
+
             # Map spec to entries of 3 tuple (job, instance, name)
             class MetricEntry:
                 def __init__(self, job, instance, name):

tests/test_arrow.py

@@ -378,10 +378,8 @@ def evaluate_result(value):
 
         # Iterate over the dataset
         for results in dataset:
-
             # For batches, iterate over each row in batch or remainder at end
             for result_idx in range(batch_size or 1):
-
                 # Get a single row value
                 if batch_size is None:
                     value = [r.numpy() for r in results]

tests/test_atds_avro/utils/stat_t_test_cli.py

@@ -139,13 +139,13 @@ def run_welchs_ttest(stat1, stat2, alpha, faster):
     df1 = n1 - 1  # degree of freedom of stat1
     df2 = n2 - 1  # degree of freedom of stat2
 
-    sample_v1 = s1 ** 2 / n1  # biased estimated sample variance of stat1
-    sample_v2 = s2 ** 2 / n2  # biased estimated sample variance of stat2
+    sample_v1 = s1**2 / n1  # biased estimated sample variance of stat1
+    sample_v2 = s2**2 / n2  # biased estimated sample variance of stat2
 
     biased_variance = np.sqrt(sample_v1 + sample_v2)
     # degree of freedom
     df = (sample_v1 + sample_v2) ** 2 / (
-        sample_v1 ** 2 / (df1) + sample_v2 ** 2 / (df2)
+        sample_v1**2 / (df1) + sample_v2**2 / (df2)
     )
 
     mean_delta = m1 - m2

tests/test_audio.py

@@ -77,7 +77,7 @@ def test_audio_io_tensor_24(audio_data_24, io_tensor_func):
     for step in [1, 100, 101, 200, 501, 600, 1001, 2000, 5001]:
         indices = list(range(0, 14336, step))
         # TODO: -1 vs. 14336 might need fix
-        for (start, stop) in zip(indices, indices[1:] + [14336]):
+        for start, stop in zip(indices, indices[1:] + [14336]):
             audio_tensor_value = audio_tensor[start:stop]
             audio_value_value = audio_value[start:stop]
             assert audio_tensor_value.shape == audio_value_value.shape
@@ -139,7 +139,7 @@ def test_audio_io_tensor(audio_data, io_tensor_func):
     for step in [1, 100, 101, 200, 501, 600, 1001, 2000, 5001]:
         indices = list(range(0, 5760, step))
         # TODO: -1 vs. 5760 might need fix
-        for (start, stop) in zip(indices, indices[1:] + [5760]):
+        for start, stop in zip(indices, indices[1:] + [5760]):
             audio_tensor_value = audio_tensor[start:stop]
             audio_value_value = audio_value[start:stop]
             assert audio_tensor_value.shape == audio_value_value.shape
@@ -218,7 +218,7 @@ def func(filename, position):
     dataset = dataset.map(func, num_parallel_calls=num_parallel_calls)
 
     item = 0
-    for (data, rate) in dataset:
+    for data, rate in dataset:
         assert audio_rate == rate
         assert data.shape == (100, 1)
         position = 1000 if item == 0 else 2000

tests/test_azure.py

@@ -206,7 +206,6 @@ class AZFSTest(tf.test.TestCase, AZFSTestBase):
     """Run tests for azfs backend using account key authentication."""
 
     def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
-
         self.account = "devstoreaccount1"
         self.container = "aztest"
         self.path_root = "az://" + os.path.join(self.account, self.container)

tests/test_http.py

@@ -77,7 +77,7 @@ def test_gfile_read(local_content, remote_filename):
     assert remote_gfile.size() == len(local_content)
     offset_start = list(range(0, len(local_content), 100))
     offset_stop = offset_start[1:] + [len(local_content)]
-    for (start, stop) in zip(offset_start, offset_stop):
+    for start, stop in zip(offset_start, offset_stop):
         assert remote_gfile.read(100) == local_content[start:stop]
 
 

tests/test_json.py

@@ -61,7 +61,7 @@ def test_io_tensor_json_recods_mode():
     dataset = tf.data.Dataset.zip((feature_dataset, label_dataset))
 
     i = 0
-    for (j_x, j_y) in dataset:
+    for j_x, j_y in dataset:
         v_x = x_test[i]
         v_y = y_test[i]
         for index, x in enumerate(j_x):
@@ -123,7 +123,7 @@ def test_io_tensor_json():
     dataset = tf.data.Dataset.zip((feature_dataset, label_dataset))
 
     i = 0
-    for (j_x, j_y) in dataset:
+    for j_x, j_y in dataset:
         v_x = x_test[i]
         v_y = y_test[i]
         for index, x in enumerate(j_x):
@@ -158,7 +158,7 @@ def test_json_dataset():
     dataset = tf.data.Dataset.zip((feature_dataset, label_dataset))
 
     i = 0
-    for (j_x, j_y) in dataset:
+    for j_x, j_y in dataset:
         v_x = x_test[i]
         v_y = y_test[i]
         for index, x in enumerate(j_x):

tests/test_ossfs.py

@@ -359,7 +359,7 @@ def test_walk_in_order(self):
         all_dirs = []
         all_subdirs = []
         all_files = []
-        for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=True):
+        for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=True):
             all_dirs.append(w_dir)
             all_subdirs.append(w_subdirs)
             all_files.append(w_files)
@@ -388,7 +388,7 @@ def test_walk_post_order(self):
         all_dirs = []
         all_subdirs = []
         all_files = []
-        for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=False):
+        for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=False):
             all_dirs.append(w_dir)
             all_subdirs.append(w_subdirs)
             all_files.append(w_files)
@@ -416,7 +416,7 @@ def test_walk_failure(self):
         all_dirs = []
         all_subdirs = []
         all_files = []
-        for (w_dir, w_subdirs, w_files) in file_io.walk(dir_path, in_order=False):
+        for w_dir, w_subdirs, w_files in file_io.walk(dir_path, in_order=False):
             all_dirs.append(w_dir)
             all_subdirs.append(w_subdirs)
             all_files.append(w_files)

tests/test_parse_avro.py

@@ -223,7 +223,6 @@ def _assert_equal(expected, actual):
             _assert_equal(expected, actual)
 
     def _verify_output(self, expected_data, actual_dataset):
-
         next_data = iter(actual_dataset)
 
         for expected in expected_data:
@@ -286,7 +285,6 @@ def test_inval_num_parallel_calls(self):
         )
 
         with pytest.raises(ValueError):
-
             dataset_a = tfio.experimental.columnar.AvroRecordDataset(
                 filenames=filenames,
                 num_parallel_reads=NUM_PARALLEL_READS,
@@ -295,7 +293,6 @@ def test_inval_num_parallel_calls(self):
             )
 
         with pytest.raises(ValueError):
-
             dataset_b = tfio.experimental.columnar.AvroRecordDataset(
                 filenames=filenames,
                 num_parallel_reads=NUM_PARALLEL_READS,

tests/test_pulsar.py

@@ -66,7 +66,7 @@ def test_pulsar_keyed_messages():
         timeout=default_pulsar_timeout,
     )
     kv = dict()
-    for (msg, key) in dataset:
+    for msg, key in dataset:
         kv.setdefault(key.numpy().decode(), []).append(msg.numpy())
     assert kv["K0"] == [("D" + str(i)).encode() for i in range(0, 6, 2)]
     assert kv["K1"] == [("D" + str(i)).encode() for i in range(1, 6, 2)]
@@ -232,7 +232,7 @@ def test_pulsar_write_keyed_messages():
         timeout=default_pulsar_timeout,
     )
     kv = dict()
-    for (msg, key) in dataset:
+    for msg, key in dataset:
         kv.setdefault(key.numpy().decode(), []).append(msg.numpy())
     assert kv["0"] == [("msg-" + str(i)).encode() for i in range(0, 10, 3)]
     assert kv["1"] == [("msg-" + str(i)).encode() for i in range(1, 10, 3)]

tests/test_text/stdin_test.py

@@ -33,5 +33,5 @@ def f(v):
 
 text_dataset = text_io.TextDataset("file://-").map(f)
 
-for (frame_number_value, ip_dst_value, ip_proto_value) in text_dataset:
+for frame_number_value, ip_dst_value, ip_proto_value in text_dataset:
     print(ip_dst_value.numpy())