Automatically build SWIG from source

This change allows Bazel to fetch and build SWIG rather than getting it
from the system. This change also improves the i/o performance of the
SWIG build, makes it hermetically sealed, and ensures tf_py_wrap_cc()
can function correctly across Bazel repositories.

CC: tensorflow#4983
Change: 136783531

Author: jart

configure

@@ -107,20 +107,6 @@ else
   perl -pi -e "s,WITH_HDFS_SUPPORT = (False|True),WITH_HDFS_SUPPORT = False,s" tensorflow/core/platform/default/build_config.bzl
 fi
 
-## Find swig path
-if [ -z "$SWIG_PATH" ]; then
-  SWIG_PATH=`type -p swig 2> /dev/null || true`
-fi
-if [[ ! -e "$SWIG_PATH" ]]; then
-  echo "Can't find swig.  Ensure swig is in \$PATH or set \$SWIG_PATH."
-  exit 1
-fi
-# Convert swig path to Windows style before writing into swig_path
-if is_windows; then
-  SWIG_PATH="$(cygpath -m "$SWIG_PATH")"
-fi
-echo "$SWIG_PATH" > tensorflow/tools/swig/swig_path
-
 # Invoke python_config and set up symlinks to python includes
 ./util/python/python_config.sh --setup "$PYTHON_BIN_PATH"
 

tensorflow/BUILD

@@ -63,12 +63,6 @@ package_group(
     packages = ["//tensorflow/..."],
 )
 
-sh_binary(
-    name = "swig",
-    srcs = ["tools/swig/swig.sh"],
-    data = glob(["tools/swig/**"]),
-)
-
 filegroup(
     name = "all_files",
     srcs = glob(

tensorflow/g3doc/get_started/os_setup.md

@@ -568,9 +568,9 @@ binary path.
 
 ```bash
 # For Python 2.7:
-$ sudo apt-get install python-numpy swig python-dev python-wheel
+$ sudo apt-get install python-numpy python-dev python-wheel
 # For Python 3.x:
-$ sudo apt-get install python3-numpy swig python3-dev python3-wheel
+$ sudo apt-get install python3-numpy python3-dev python3-wheel
 ```
 
 #### Optional: Install CUDA (GPUs on Linux)
@@ -617,20 +617,16 @@ sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn*
 
 ### Prepare environment for Mac OS X
 
-We recommend using [homebrew](http://brew.sh) to install the bazel and SWIG
-dependencies, and installing python dependencies using easy_install or pip.
-
-Of course you can also install Swig from source without using homebrew. In that
-case, be sure to install its dependency [PCRE](http://www.pcre.org) and not
-PCRE2.
+We recommend using [homebrew](http://brew.sh) to install the bazel dependency,
+and installing python dependencies using easy_install or pip.
 
 #### Dependencies
 
 Follow instructions [here](http://bazel.io/docs/install.html) to install the
-dependencies for bazel. You can then use homebrew to install bazel and SWIG:
+dependencies for bazel. You can then use homebrew to install bazel:
 
 ```bash
-$ brew install bazel swig
+$ brew install bazel
 ```
 
 You can install the python dependencies using easy_install or pip. Using

tensorflow/python/lib/io/file_io.i

@@ -275,5 +275,8 @@ string ReadFromStream(tensorflow::io::BufferedInputStream* stream,
 %unignoreall
 
 %include "tensorflow/c/tf_status_helper.h"
+
+%ignore tensorflow::io::internal::JoinPathImpl;
 %include "tensorflow/core/lib/io/path.h"
-%include "tensorflow/core/platform/file_statistics.h"
+
+%include "tensorflow/core/platform/file_statistics.h"

tensorflow/tensorflow.bzl

@@ -557,33 +557,30 @@ def _py_wrap_cc_impl(ctx):
   if len(srcs) != 1:
     fail("Exactly one SWIG source file label must be specified.", "srcs")
   module_name = ctx.attr.module_name
-  cc_out = ctx.outputs.cc_out
-  py_out = ctx.outputs.py_out
   src = ctx.files.srcs[0]
-  args = ["-c++", "-python"]
-  args += ["-module", module_name]
-  args += ["-l" + f.path for f in ctx.files.swig_includes]
-  cc_include_dirs = set()
-  cc_includes = set()
+  inputs = set([src])
+  inputs += ctx.files.swig_includes
   for dep in ctx.attr.deps:
-    cc_include_dirs += [h.dirname for h in dep.cc.transitive_headers]
-    cc_includes += dep.cc.transitive_headers
-  args += ["-I" + x for x in cc_include_dirs]
-  args += ["-I" + ctx.label.workspace_root]
-  args += ["-o", cc_out.path]
-  args += ["-outdir", py_out.dirname]
+    inputs += dep.cc.transitive_headers
+  inputs += ctx.files._swiglib
+  swig_include_dirs = set([f.root.path for f in inputs if f.root.path])
+  swig_include_dirs += sorted([f.dirname for f in ctx.files._swiglib])
+  args = ["-c++",
+          "-python",
+          "-module", module_name,
+          "-o", ctx.outputs.cc_out.path,
+          "-outdir", ctx.outputs.py_out.dirname]
+  args += ["-l" + f.path for f in ctx.files.swig_includes]
+  args += ["-I" + i for i in swig_include_dirs]
   args += [src.path]
-  outputs = [cc_out, py_out]
-  # TODO(pcloudy): Move args to arguments after
-  # https://github.com/bazelbuild/bazel/issues/1926 is fixed
-  ctx.action(command=" ".join(["tensorflow/tools/swig/swig.sh"] + args),
-             arguments=[],
-             mnemonic="PythonSwig",
-             inputs=sorted(set([src]) + cc_includes + ctx.files.swig_includes +
-                         ctx.attr.swig_deps.files),
+  outputs = [ctx.outputs.cc_out,
+             ctx.outputs.py_out]
+  ctx.action(executable=ctx.executable._swig,
+             arguments=args,
+             inputs=list(inputs),
              outputs=outputs,
-             use_default_shell_env=True,
-             progress_message="SWIGing {input}".format(input=src.path))
+             mnemonic="PythonSwig",
+             progress_message="SWIGing " + src.path)
   return struct(files=set(outputs))
 
 _py_wrap_cc = rule(
@@ -600,11 +597,17 @@ _py_wrap_cc = rule(
             allow_files = True,
             providers = ["cc"],
         ),
-        "swig_deps": attr.label(default = Label(
-            "//tensorflow:swig",  # swig_templates
-        )),
         "module_name": attr.string(mandatory = True),
         "py_module_name": attr.string(mandatory = True),
+        "_swig": attr.label(
+            default = Label("@swig//:swig"),
+            executable = True,
+            cfg = "host",
+        ),
+        "_swiglib": attr.label(
+            default = Label("@swig//:templates"),
+            allow_files = True,
+        ),
     },
     outputs = {
         "cc_out": "%{module_name}.cc",

tensorflow/tools/docker/Dockerfile.devel

@@ -14,7 +14,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python-dev \
         rsync \
         software-properties-common \
-        swig \
         unzip \
         zip \
         zlib1g-dev \

tensorflow/tools/docker/Dockerfile.devel-gpu

@@ -15,7 +15,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python-dev \
         rsync \
         software-properties-common \
-        swig \
         unzip \
         zip \
         zlib1g-dev \

tensorflow/tools/swig/swig.sh

@@ -126,6 +126,22 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     actual = str(Label("//util/python:python_headers")),
   )
 
+  native.new_http_archive(
+    name = "pcre",
+    sha256 = "ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7",
+    url = "http://ftp.cs.stanford.edu/pub/exim/pcre/pcre-8.39.tar.gz",
+    strip_prefix = "pcre-8.39",
+    build_file = str(Label("//third_party:pcre.BUILD")),
+  )
+
+  native.new_http_archive(
+    name = "swig",
+    sha256 = "a2669657cabcedc371f63c0457407a183e0b6b2ef4e7e303c1ec9a3964cc7813",
+    url = "http://ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.2/swig-3.0.2.tar.gz",
+    strip_prefix = "swig-3.0.2",
+    build_file = str(Label("//third_party:swig.BUILD")),
+  )
+
   # grpc expects //external:protobuf_clib and //external:protobuf_compiler
   # to point to the protobuf's compiler library.
   native.bind(

tensorflow/workspace.bzl

@@ -0,0 +1 @@
+licenses(["notice"])  # Apache 2.0

third_party/BUILD

@@ -0,0 +1,80 @@
+licenses(["notice"])  # BSD
+
+exports_files(["LICENSE"])
+
+cc_library(
+    name = "pcre",
+    srcs = [
+        "pcre_byte_order.c",
+        "pcre_chartables.c",
+        "pcre_compile.c",
+        "pcre_config.c",
+        "pcre_dfa_exec.c",
+        "pcre_exec.c",
+        "pcre_fullinfo.c",
+        "pcre_get.c",
+        "pcre_globals.c",
+        "pcre_internal.h",
+        "pcre_jit_compile.c",
+        "pcre_maketables.c",
+        "pcre_newline.c",
+        "pcre_ord2utf8.c",
+        "pcre_refcount.c",
+        "pcre_string_utils.c",
+        "pcre_study.c",
+        "pcre_tables.c",
+        "pcre_ucd.c",
+        "pcre_valid_utf8.c",
+        "pcre_version.c",
+        "pcre_xclass.c",
+        "ucp.h",
+    ],
+    hdrs = [
+        "pcre.h",
+        "pcreposix.h",
+    ],
+    copts = [
+        "-DHAVE_BCOPY=1",
+        "-DHAVE_INTTYPES_H=1",
+        "-DHAVE_MEMMOVE=1",
+        "-DHAVE_STDINT_H=1",
+        "-DHAVE_STRERROR=1",
+        "-DHAVE_SYS_STAT_H=1",
+        "-DHAVE_SYS_TYPES_H=1",
+        "-DHAVE_UNISTD_H=1",
+        "-DLINK_SIZE=2",
+        "-DMATCH_LIMIT=10000000",
+        "-DMATCH_LIMIT_RECURSION=1000",
+        "-DMAX_NAME_COUNT=10000",
+        "-DMAX_NAME_SIZE=32",
+        "-DNEWLINE=10",
+        "-DNO_RECURSE",
+        "-DPARENS_NEST_LIMIT=50",
+        "-DPCRE_STATIC=1",
+        "-DPOSIX_MALLOC_THRESHOLD=10",
+        "-DSTDC_HEADERS=1",
+        "-DSUPPORT_UCP",
+        "-DSUPPORT_UTF",
+    ],
+    includes = ["."],
+    visibility = ["@swig//:__pkg__"],  # Please use RE2
+    alwayslink = 1,
+)
+
+genrule(
+    name = "pcre_h",
+    srcs = ["pcre.h.in"],
+    outs = ["pcre.h"],
+    cmd = "sed -e s/@PCRE_MAJOR@/8/" +
+          "    -e s/@PCRE_MINOR@/39/" +
+          "    -e s/@PCRE_PRERELEASE@//" +
+          "    -e s/@PCRE_DATE@/redacted/" +
+          "    $< >$@",
+)
+
+genrule(
+    name = "pcre_chartables_c",
+    srcs = ["pcre_chartables.c.dist"],
+    outs = ["pcre_chartables.c"],
+    cmd = "cp $< $@",
+)