Install codegen in bazel workspace to allow docs tests to run.

PiperOrigin-RevId: 159477794

Author: MarkDaoust

tensorflow/tools/docs/BUILD

@@ -37,14 +37,14 @@ py_library(
     srcs = ["parser.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
+    deps = ["@com_github_andreif_codegen"],
 )
 
 py_test(
     name = "parser_test",
     size = "small",
     srcs = ["parser_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["manual"],
     deps = [
         ":parser",
         "//tensorflow/python:platform_test",
@@ -78,13 +78,10 @@ py_test(
     size = "small",
     srcs = ["generate_lib_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["manual"],
     deps = [
         ":generate_lib",
         ":parser",
-        "//tensorflow:tensorflow_py",
         "//tensorflow/python:platform_test",
-        "//tensorflow/python/debug:debug_py",
     ],
 )
 
@@ -105,7 +102,6 @@ py_test(
     srcs = ["build_docs_test.py"],
     data = ["//tensorflow:docs_src"],
     srcs_version = "PY2AND3",
-    tags = ["manual"],
     deps = [
         ":generate_lib",
         "//tensorflow:tensorflow_py",

tensorflow/tools/docs/build_docs_test.py

@@ -19,6 +19,8 @@
 from __future__ import print_function
 
 import os
+import sys
+import textwrap
 
 import tensorflow as tf
 from tensorflow.python import debug as tf_debug
@@ -29,19 +31,40 @@
 
 class Flags(object):
   resource_root = resource_loader.get_root_dir_with_all_resources()
-  src_dir = os.path.join(resource_root, 'third_party/tensorflow/docs_src')
-  base_dir = os.path.join(resource_root, 'third_party/tensorflow/')
+  src_dir = os.path.join(resource_root, 'tensorflow/docs_src')
+  base_dir = os.path.join(resource_root, 'tensorflow/')
   output_dir = googletest.GetTempDir()
 
 
 class BuildDocsTest(googletest.TestCase):
 
   def testBuildDocs(self):
+    if sys.version_info >= (3, 0):
+      print('Warning: Doc generation is not supported from python3.')
+      return
+
     doc_generator = generate_lib.DocGenerator()
 
     doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)])
 
-    status = doc_generator.build(Flags())
+    try:
+      status = doc_generator.build(Flags())
+    except RuntimeError as e:
+      if not e.args[0].startswith('Modules nested too deep'):
+        raise
+
+      msg = textwrap.dedent("""\
+          %s
+
+          ****************************************************************
+          If this test fails here, you have most likely introduced an
+          unsealed module. Make sure to use `remove_undocumented` or similar
+          utilities to avoid leaking symbols. See above for more information
+          on the exact point of failure.
+          ****************************************************************
+          """ % e.args[0])
+
+      raise RuntimeError(msg)
 
     if status:
       self.fail('Found %s Errors!' % status)

tensorflow/tools/docs/generate_lib.py

@@ -20,6 +20,7 @@
 
 import argparse
 import os
+import sys
 
 import six
 
@@ -415,6 +416,8 @@ class DocGenerator(object):
   """Main entry point for generating docs."""
 
   def __init__(self):
+    if sys.version_info >= (3, 0):
+      print('Warning: Doc generation is not supported from python3.')
     self.argument_parser = argparse.ArgumentParser()
     self._py_modules = None
     self._private_map = _get_default_private_map()

tensorflow/tools/docs/generate_lib_test.py

@@ -21,9 +21,6 @@
 import os
 import sys
 
-import tensorflow as tf
-
-from tensorflow.python import debug as tf_debug
 from tensorflow.python.platform import googletest
 from tensorflow.tools.docs import generate_lib
 from tensorflow.tools.docs import parser
@@ -54,22 +51,6 @@ def __init__(self, index, duplicate_of):
 
 class GenerateTest(googletest.TestCase):
 
-  def test_extraction(self):
-    py_modules = [('tf', tf), ('tfdbg', tf_debug)]
-
-    try:
-      generate_lib.extract(py_modules,
-                           generate_lib._get_default_private_map(),
-                           generate_lib._get_default_do_not_descend_map())
-    except RuntimeError:
-      print('*****************************************************************')
-      print('If this test fails, you have most likely introduced an unsealed')
-      print('module. Make sure to use remove_undocumented or similar utilities')
-      print('to avoid leaking symbols. See below for more information on the')
-      print('failure.')
-      print('*****************************************************************')
-      raise
-
   def test_write(self):
     module = sys.modules[__name__]
 

tensorflow/tools/docs/parser_test.py

@@ -491,13 +491,13 @@ def testSaveReferenceResolver(self):
 
 class TestParseFunctionDetails(googletest.TestCase):
 
-  def testParseFunctionDetails(self):
+  def test_parse_function_details(self):
     docstring, function_details = parser._parse_function_details(RELU_DOC)
 
     self.assertEqual(len(function_details), 2)
     args = function_details[0]
     self.assertEqual(args.keyword, 'Args')
-    self.assertEmpty(args.header)
+    self.assertEqual(len(args.header), 0)
     self.assertEqual(len(args.items), 2)
     self.assertEqual(args.items[0][0], 'features')
     self.assertEqual(args.items[1][0], 'name')
@@ -515,5 +515,60 @@ def testParseFunctionDetails(self):
         docstring + ''.join(str(detail) for detail in function_details))
 
 
+class TestGenerateSignature(googletest.TestCase):
+
+  def test_known_object(self):
+    if sys.version_info >= (3, 0):
+      print('Warning: Doc generation is not supported from python3.')
+      return
+
+    known_object = object()
+    reverse_index = {id(known_object): 'location.of.object.in.api'}
+
+    def example_fun(arg=known_object):  # pylint: disable=unused-argument
+      pass
+
+    sig = parser._generate_signature(example_fun, reverse_index)
+    self.assertEqual(sig, ['arg=location.of.object.in.api'])
+
+  def test_literals(self):
+    if sys.version_info >= (3, 0):
+      print('Warning: Doc generation is not supported from python3.')
+      return
+
+    def example_fun(a=5, b=5.0, c=None, d=True, e='hello', f=(1, (2, 3))):  # pylint: disable=g-bad-name, unused-argument
+      pass
+
+    sig = parser._generate_signature(example_fun, reverse_index={})
+    self.assertEqual(
+        sig, ['a=5', 'b=5.0', 'c=None', 'd=True', "e='hello'", 'f=(1, (2, 3))'])
+
+  def test_dotted_name(self):
+    if sys.version_info >= (3, 0):
+      print('Warning: Doc generation is not supported from python3.')
+      return
+
+    # pylint: disable=g-bad-name
+    class a(object):
+
+      class b(object):
+
+        class c(object):
+
+          class d(object):
+
+            def __init__(self, *args):
+              pass
+    # pylint: enable=g-bad-name
+
+    e = {'f': 1}
+
+    def example_fun(arg1=a.b.c.d, arg2=a.b.c.d(1, 2), arg3=e['f']):  # pylint: disable=unused-argument
+      pass
+
+    sig = parser._generate_signature(example_fun, reverse_index={})
+    self.assertEqual(sig, ['arg1=a.b.c.d', 'arg2=a.b.c.d(1, 2)', "arg3=e['f']"])
+
+
 if __name__ == '__main__':
   googletest.main()

tensorflow/workspace.bzl

@@ -297,6 +297,17 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       build_file = str(Label("//third_party:backports_weakref.BUILD")),
   )
 
+  native.new_http_archive(
+      name = "com_github_andreif_codegen",
+      urls = [
+          "http://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz",
+          "https://github.com/andreif/codegen/archive/1.0.tar.gz",
+      ],
+      sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee",
+      strip_prefix = "codegen-1.0",
+      build_file = str(Label("//third_party:codegen.BUILD")),
+  )
+
   filegroup_external(
       name = "org_python_license",
       licenses = ["notice"],  # Python 2.0

third_party/codegen.BUILD

@@ -0,0 +1,16 @@
+# -*- mode: python; -*-
+#
+# Description:
+#   Extension to ast that allow ast -> python code generation.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # New BSD
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "com_github_andreif_codegen",
+    srcs = glob(["codegen.py"]),
+    srcs_version = "PY2AND3",
+)