Skip to content

Commit 25eb7c3

Browse files
qihqipytorchmergebot
qihqi
authored and
pytorchmergebot
committed
Clean up dependancy for flatbuffer_loader (pytorch#86041)
Test Plan: waitforsandcastle Differential Revision: D38445936 Pull Request resolved: pytorch#86041 Approved by: https://github.com/cccclai
1 parent 3789204 commit 25eb7c3

23 files changed

+231
-316
lines changed

‎BUILD.bazel

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1648,6 +1648,17 @@ cu_library(
16481648
deps = [":torch_headers"],
16491649
)
16501650

1651+
torch_sources = ({
1652+
k: "" for k in (
1653+
libtorch_core_sources +
1654+
libtorch_distributed_sources +
1655+
torch_cpp_srcs +
1656+
libtorch_extra_sources +
1657+
jit_core_sources +
1658+
lazy_tensor_ts_sources +
1659+
GENERATED_AUTOGRAD_CPP)
1660+
}).keys()
1661+
16511662
cc_library(
16521663
name = "torch",
16531664
srcs = if_cuda(glob(
@@ -1657,11 +1668,7 @@ cc_library(
16571668
"torch/csrc/cuda/nccl.cpp",
16581669
"torch/csrc/distributed/c10d/quantization/quantization_gpu.cu",
16591670
],
1660-
)) + libtorch_core_sources + libtorch_distributed_sources + torch_cpp_srcs + libtorch_extra_sources + jit_core_sources + lazy_tensor_ts_sources + GENERATED_AUTOGRAD_CPP + [
1661-
"torch/csrc/jit/serialization/flatbuffer_serializer.cpp",
1662-
"torch/csrc/jit/mobile/flatbuffer_loader.cpp",
1663-
"torch/csrc/jit/serialization/flatbuffer_serializer_jit.cpp",
1664-
],
1671+
)) + torch_sources,
16651672
copts = TORCH_COPTS,
16661673
defines = [
16671674
"CAFFE2_NIGHTLY_VERSION=20200115",

‎buckbuild.bzl

Lines changed: 32 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ load(
1717
"aten_cpu_source_list",
1818
"aten_native_source_list",
1919
"core_sources_common",
20-
"core_sources_full_mobile_no_backend_interface",
20+
"core_sources_full_mobile_no_backend_interface_xplat",
2121
"core_trainer_sources",
2222
"jit_core_headers",
2323
"jit_core_sources",
@@ -755,14 +755,13 @@ def get_pt_operator_registry_dict(
755755
"pt_operator_registry",
756756
],
757757
deps = [
758-
# need absolute path here
759-
ROOT + ":torch_mobile_core",
760-
ROOT + ":aten_cpu",
761-
ROOT + ":aten_metal_prepack_header",
762-
third_party("glog"),
763-
C10,
764-
] + ([ROOT + ":torch_mobile_train"] if train else []) +
765-
([ROOT + ":flatbuffers_mobile"] if enable_flatbuffer else []),
758+
# need absolute path here
759+
ROOT + ":torch_mobile_core",
760+
ROOT + ":aten_cpu",
761+
ROOT + ":aten_metal_prepack_header",
762+
third_party("glog"),
763+
C10,
764+
] + ([ROOT + ":torch_mobile_train"] if train else []),
766765
**kwargs
767766
)
768767

@@ -1303,12 +1302,14 @@ def define_buck_targets(
13031302
name = "torch_mobile_deserialize",
13041303
srcs = [
13051304
"torch/csrc/jit/mobile/import.cpp",
1305+
"torch/csrc/jit/mobile/flatbuffer_loader.cpp",
13061306
],
13071307
compiler_flags = get_pt_compiler_flags(),
1308-
exported_preprocessor_flags = get_pt_preprocessor_flags(),
1308+
exported_preprocessor_flags = get_pt_preprocessor_flags() + (["-DFB_XPLAT_BUILD"] if not IS_OSS else []),
13091309
header_namespace = "",
13101310
exported_headers = [
13111311
"torch/csrc/jit/mobile/import.h",
1312+
"torch/csrc/jit/mobile/flatbuffer_loader.h",
13121313
],
13131314
# torch_mobile_deserialize brings in sources neccessary to read a module
13141315
# which depends on mobile module definition
@@ -1331,6 +1332,7 @@ def define_buck_targets(
13311332
":torch_mobile_module",
13321333
":torch_mobile_observer",
13331334
":torch_mobile_deserialize_common",
1335+
":mobile_bytecode",
13341336
C10,
13351337
],
13361338
)
@@ -1390,11 +1392,7 @@ def define_buck_targets(
13901392

13911393
pt_xplat_cxx_library(
13921394
name = "torch_core",
1393-
srcs = core_sources_full_mobile_no_backend_interface + [
1394-
"torch/csrc/api/src/jit.cpp",
1395-
"torch/csrc/jit/serialization/export_bytecode.cpp",
1396-
"torch/csrc/jit/serialization/export_module.cpp",
1397-
],
1395+
srcs = core_sources_full_mobile_no_backend_interface_xplat,
13981396
compiler_flags = get_pt_compiler_flags(),
13991397
exported_preprocessor_flags = get_pt_preprocessor_flags(),
14001398
visibility = [
@@ -1443,6 +1441,7 @@ def define_buck_targets(
14431441
":torch_core",
14441442
":torch_mobile_deserialize",
14451443
":torch_mobile_train",
1444+
":jit_module_saving",
14461445
C10,
14471446
],
14481447
)
@@ -1474,6 +1473,7 @@ def define_buck_targets(
14741473
":generated-autograd-headers",
14751474
":torch_headers",
14761475
":torch_mobile_deserialize",
1476+
":flatbuffers_serializer_mobile",
14771477
C10,
14781478
],
14791479
)
@@ -1563,15 +1563,16 @@ def define_buck_targets(
15631563
"torch/csrc/jit/serialization/export_module.cpp",
15641564
],
15651565
compiler_flags = get_pt_compiler_flags(),
1566-
exported_preprocessor_flags = get_pt_preprocessor_flags(),
1566+
exported_preprocessor_flags = get_pt_preprocessor_flags() +
1567+
(["-DFB_XPLAT_BUILD"] if not IS_OSS else []),
15671568
exported_headers = [
15681569
"torch/csrc/jit/serialization/export.h",
1569-
"torch/csrc/jit/serialization/flatbuffer_serializer_jit.h",
15701570
],
15711571
visibility = ["PUBLIC"],
15721572
deps = [
15731573
":torch",
15741574
":torch_mobile_core",
1575+
":flatbuffers_serializer_mobile",
15751576
],
15761577
)
15771578

@@ -1618,6 +1619,7 @@ def define_buck_targets(
16181619
]),
16191620
)
16201621

1622+
#TODO(qihan) delete
16211623
pt_xplat_cxx_library(
16221624
name = "torch_mobile_core_flatbuffer",
16231625
srcs = [],
@@ -1639,9 +1641,7 @@ def define_buck_targets(
16391641
exported_deps = [
16401642
":aten_cpu",
16411643
":torch_common",
1642-
] + ([] if IS_OSS else [
1643-
"//xplat/caffe2/fb/runtime:torch_mobile_deserialize_flatbuffer",
1644-
]),
1644+
],
16451645
)
16461646

16471647
fb_xplat_cxx_library(
@@ -1739,7 +1739,7 @@ def define_buck_targets(
17391739
# the internals of the loader/serializer layer.
17401740
visibility = [
17411741
"{}:flatbuffer_loader".format(ROOT),
1742-
"{}:flatbuffer_serializer_mobile".format(ROOT),
1742+
"{}:flatbuffers_serializer_mobile".format(ROOT),
17431743
],
17441744
exported_deps = [
17451745
third_party("flatbuffers-api"),
@@ -1766,14 +1766,15 @@ def define_buck_targets(
17661766
C10,
17671767
],
17681768
exported_deps = [
1769-
":torch_mobile_train",
1769+
":torch_mobile_deserialize",
1770+
":mobile_bytecode",
17701771
],
17711772
)
17721773

1774+
# TODO (qihan) delete
17731775
pt_xplat_cxx_library(
17741776
name = "flatbuffer_loader",
17751777
srcs = [
1776-
"torch/csrc/jit/mobile/flatbuffer_loader.cpp",
17771778
],
17781779
exported_headers = [
17791780
"torch/csrc/jit/mobile/flatbuffer_loader.h",
@@ -1803,24 +1804,26 @@ def define_buck_targets(
18031804
":mobile_bytecode",
18041805
],
18051806
exported_deps = [
1806-
":torch_mobile_deserialize",
18071807
C10,
18081808
],
18091809
)
18101810

1811+
# TODO(qihan) delete
18111812
fb_xplat_cxx_library(
18121813
name = "flatbuffers_serializer_jit",
1813-
srcs = ["torch/csrc/jit/serialization/flatbuffer_serializer_jit.cpp"],
1814-
exported_headers = [
1815-
"torch/csrc/jit/serialization/flatbuffer_serializer_jit.h",
1816-
],
18171814
compiler_flags = [
18181815
"-g0",
18191816
"-O3",
18201817
"-fexceptions",
18211818
"-frtti",
18221819
"-Wno-deprecated-declarations",
18231820
],
1821+
headers = [
1822+
"torch/csrc/jit/serialization/flatbuffer_serializer_jit.h",
1823+
],
1824+
srcs = [
1825+
"torch/csrc/jit/serialization/flatbuffer_serializer_jit.cpp",
1826+
],
18241827
linker_flags = [
18251828
"-Wl,--no-as-needed",
18261829
],
@@ -1850,6 +1853,7 @@ def define_buck_targets(
18501853
exported_deps = [
18511854
":flatbuffer_loader",
18521855
":flatbuffers_serializer_mobile",
1856+
":torch_mobile_train",
18531857
],
18541858
)
18551859

‎build_variables.bzl

Lines changed: 47 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,28 @@ core_trainer_sources = [
177177
"torch/csrc/jit/serialization/type_name_uniquer.cpp",
178178
]
179179

180-
core_sources_full_mobile_no_backend_interface = [
180+
torch_mobile_core = [
181+
# backend_debug_info.cpp provides
182+
# __torch__.torch.classes.backend.BackendDebugInfo class
183+
# This should not be needed eventually.
184+
# TODO: Remove this dependency
185+
"torch/csrc/jit/backends/backend_debug_info.cpp",
186+
"torch/csrc/jit/mobile/compatibility/model_compatibility.cpp",
187+
"torch/csrc/jit/mobile/function.cpp",
188+
"torch/csrc/jit/mobile/import.cpp",
189+
"torch/csrc/jit/mobile/flatbuffer_loader.cpp",
190+
"torch/csrc/jit/mobile/interpreter.cpp",
191+
"torch/csrc/jit/mobile/module.cpp",
192+
"torch/csrc/jit/mobile/observer.cpp",
193+
"torch/csrc/jit/mobile/parse_bytecode.cpp",
194+
"torch/csrc/jit/mobile/parse_operators.cpp",
195+
"torch/csrc/jit/mobile/quantization.cpp",
196+
"torch/csrc/jit/mobile/upgrader_mobile.cpp",
197+
"torch/csrc/jit/runtime/register_prim_ops.cpp",
198+
"torch/csrc/jit/runtime/register_special_ops.cpp",
199+
]
200+
201+
core_sources_full_mobile_no_backend_interface_xplat = [
181202
"torch/csrc/jit/api/function_impl.cpp",
182203
"torch/csrc/jit/api/module.cpp",
183204
"torch/csrc/jit/api/object.cpp",
@@ -385,6 +406,26 @@ core_sources_full_mobile_no_backend_interface = [
385406
"torch/csrc/utils/variadic.cpp",
386407
]
387408

409+
core_sources_full_mobile_no_backend_interface = core_sources_full_mobile_no_backend_interface_xplat + [
410+
# backend_debug_info.cpp provides
411+
# __torch__.torch.classes.backend.BackendDebugInfo class
412+
# This should not be needed eventually.
413+
# TODO: Remove this dependency
414+
"torch/csrc/jit/backends/backend_debug_info.cpp",
415+
"torch/csrc/jit/mobile/compatibility/model_compatibility.cpp",
416+
"torch/csrc/jit/mobile/function.cpp",
417+
"torch/csrc/jit/mobile/import.cpp",
418+
"torch/csrc/jit/mobile/flatbuffer_loader.cpp",
419+
"torch/csrc/jit/mobile/interpreter.cpp",
420+
"torch/csrc/jit/mobile/module.cpp",
421+
"torch/csrc/jit/mobile/observer.cpp",
422+
"torch/csrc/jit/mobile/parse_bytecode.cpp",
423+
"torch/csrc/jit/mobile/parse_operators.cpp",
424+
"torch/csrc/jit/mobile/quantization.cpp",
425+
"torch/csrc/jit/mobile/upgrader_mobile.cpp",
426+
]
427+
428+
388429
core_sources_full_mobile = core_sources_full_mobile_no_backend_interface + [
389430
"torch/csrc/jit/backends/backend_debug_info.cpp",
390431
"torch/csrc/jit/backends/backend_interface.cpp",
@@ -563,28 +604,6 @@ torch_mobile_tracer_sources = [
563604
"torch/csrc/jit/mobile/model_tracer/BuildFeatureTracer.cpp",
564605
]
565606

566-
torch_mobile_core = [
567-
# backend_debug_info.cpp provides
568-
# __torch__.torch.classes.backend.BackendDebugInfo class
569-
# This should not be needed eventually.
570-
# TODO: Remove this dependency
571-
"torch/csrc/jit/backends/backend_debug_info.cpp",
572-
"torch/csrc/jit/mobile/compatibility/model_compatibility.cpp",
573-
# TODO: This line needs to be uncommented to build mobile in OSS with flatbuffers
574-
# "torch/csrc/jit/mobile/flatbuffer_loader.cpp",
575-
"torch/csrc/jit/mobile/function.cpp",
576-
"torch/csrc/jit/mobile/import.cpp",
577-
"torch/csrc/jit/mobile/interpreter.cpp",
578-
"torch/csrc/jit/mobile/module.cpp",
579-
"torch/csrc/jit/mobile/observer.cpp",
580-
"torch/csrc/jit/mobile/parse_bytecode.cpp",
581-
"torch/csrc/jit/mobile/parse_operators.cpp",
582-
"torch/csrc/jit/mobile/quantization.cpp",
583-
"torch/csrc/jit/mobile/upgrader_mobile.cpp",
584-
"torch/csrc/jit/runtime/register_prim_ops.cpp",
585-
"torch/csrc/jit/runtime/register_special_ops.cpp",
586-
]
587-
588607
libtorch_lite_eager_symbolication = [
589608
"torch/csrc/jit/frontend/source_range.cpp",
590609
"torch/csrc/jit/ir/scope.cpp",
@@ -621,6 +640,7 @@ libtorch_extra_sources = libtorch_core_jit_sources + [
621640
# when it is built in libtorch
622641
"torch/csrc/jit/mobile/debug_info.cpp",
623642
"torch/csrc/jit/mobile/function.cpp",
643+
"torch/csrc/jit/mobile/flatbuffer_loader.cpp",
624644
"torch/csrc/jit/mobile/import.cpp",
625645
"torch/csrc/jit/mobile/import_data.cpp",
626646
"torch/csrc/jit/mobile/interpreter.cpp",
@@ -638,24 +658,16 @@ libtorch_extra_sources = libtorch_core_jit_sources + [
638658
"torch/csrc/jit/serialization/export.cpp",
639659
"torch/csrc/jit/serialization/export_bytecode.cpp",
640660
"torch/csrc/jit/serialization/export_module.cpp",
661+
"torch/csrc/jit/serialization/flatbuffer_serializer.cpp",
641662
"torch/csrc/jit/serialization/import_legacy.cpp",
642663
"torch/csrc/utils/byte_order.cpp",
643664
"torch/csrc/utils/out_types.cpp",
644665
]
645666

646667
def libtorch_sources(gencode_pattern = ":generate-code[{}]"):
647-
enable_flatbuffer = bool(native.read_config("fbcode", "caffe2_enable_flatbuffer", None))
648-
flatbuffer_serializer_sources = [
649-
"torch/csrc/jit/serialization/flatbuffer_serializer.cpp",
650-
"torch/csrc/jit/serialization/flatbuffer_serializer_jit.cpp",
651-
]
652-
if enable_flatbuffer:
653-
return (
654-
libtorch_generated_sources(gencode_pattern) + libtorch_core_sources + libtorch_distributed_sources + libtorch_extra_sources +
655-
flatbuffer_serializer_sources
656-
)
657-
else:
658-
return libtorch_generated_sources(gencode_pattern) + libtorch_core_sources + libtorch_distributed_sources + libtorch_extra_sources
668+
return (
669+
libtorch_generated_sources(gencode_pattern) + libtorch_core_sources + libtorch_distributed_sources + libtorch_extra_sources
670+
)
659671

660672
libtorch_cuda_core_sources = [
661673
"torch/csrc/CudaIPCTypes.cpp",

‎caffe2/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,7 @@ if(BUILD_LITE_INTERPRETER)
492492
set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
493493
else()
494494
append_filelist("libtorch_cmake_sources" LIBTORCH_CMAKE_SRCS)
495+
list(APPEND LIBTORCH_CMAKE_SRCS ${LITE_EAGER_SYMOBLICATION_SRCS})
495496
if(BUILD_LAZY_TS_BACKEND)
496497
append_filelist("lazy_tensor_ts_sources" LIBTORCH_CMAKE_SRCS)
497498
endif()
@@ -564,6 +565,7 @@ if(NOT INTERN_DISABLE_MOBILE_INTERP)
564565
${TORCH_SRC_DIR}/csrc/jit/mobile/train/random.cpp
565566
${TORCH_SRC_DIR}/csrc/jit/mobile/train/sequential.cpp
566567
${TORCH_SRC_DIR}/csrc/jit/mobile/upgrader_mobile.cpp
568+
${TORCH_SRC_DIR}/csrc/jit/serialization/flatbuffer_serializer.cpp
567569
)
568570
list(APPEND TORCH_SRCS ${MOBILE_SRCS})
569571
list(APPEND TORCH_SRCS ${LITE_EAGER_SYMOBLICATION_SRCS})
@@ -599,7 +601,6 @@ if(NOT INTERN_BUILD_MOBILE AND NOT BUILD_LITE_INTERPRETER)
599601
${TORCH_SRC_DIR}/csrc/jit/serialization/export_bytecode.cpp
600602
${TORCH_SRC_DIR}/csrc/jit/serialization/export_module.cpp
601603
${TORCH_SRC_DIR}/csrc/jit/serialization/flatbuffer_serializer.cpp
602-
${TORCH_SRC_DIR}/csrc/jit/serialization/flatbuffer_serializer_jit.cpp
603604
${TORCH_SRC_DIR}/csrc/jit/codegen/fuser/cpu/fused_kernel.cpp
604605
${TORCH_SRC_DIR}/csrc/jit/api/module_save.cpp
605606
${TORCH_SRC_DIR}/csrc/utils/byte_order.cpp

‎test/cpp/jit/test_lite_interpreter.cpp

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
#include <torch/csrc/jit/mobile/parse_operators.h>
1818
#include <torch/csrc/jit/mobile/upgrader_mobile.h>
1919
#include <torch/csrc/jit/serialization/export.h>
20-
#include <torch/csrc/jit/serialization/flatbuffer_serializer_jit.h>
2120
#include <torch/csrc/jit/serialization/import.h>
2221
#include <torch/custom_class.h>
2322
#include <torch/torch.h>
@@ -680,7 +679,6 @@ void backportAllVersionCheck(
680679

681680
#if !defined FB_XPLAT_BUILD
682681
TEST(LiteInterpreterTest, BackPortByteCodeModelAllVersions) {
683-
torch::jit::register_flatbuffer_all();
684682
torch::jit::Module module("m");
685683
// NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers)
686684
module.register_parameter("weight", torch::ones({20, 1, 5, 5}), false);

0 commit comments

Comments
 (0)