Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 39 additions & 20 deletions graph_net/paddle/validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import numpy as np
import graph_net
import os
import re
import ast
import paddle


Expand All @@ -29,29 +29,49 @@ def _get_sha_hash(content):
return m.hexdigest()


def _save_to_model_path(dump_dir, hash_text):
file_path = f"{dump_dir}/graph_hash.txt"
with open(file_path, "w") as f:
f.write(hash_text)
def _extract_forward_source(model_path):
source = None
with open(f"{model_path}/model.py", "r") as f:
source = f.read()

tree = ast.parse(source)
forward_code = None

def extract_from_forward_regex(text, case_sensitive=True):
pattern = r"forward.*"
flags = 0 if case_sensitive else re.IGNORECASE
for node in tree.body:
if isinstance(node, ast.ClassDef) and node.name == "GraphModule":
for fn in node.body:
if isinstance(fn, ast.FunctionDef) and fn.name == "forward":
return ast.unparse(fn)
return None

match = re.search(pattern, text, flags)
if match:
return match.group(0)

def check_graph_hash(args):
model_path = args.model_path
file_path = f"{model_path}/graph_hash.txt"
if args.dump_graph_hash_key:
model_str = _extract_forward_source(model_path)
assert model_str is not None, f"model_str of {args.model_path} is None."
new_hash_text = _get_sha_hash(model_str)

old_hash_text = None
if os.path.exists(file_path):
with open(file_path, "r") as f:
old_hash_text = f.read()

if old_hash_text is None or new_hash_text != old_hash_text:
print(f"Writing to {file_path}.")
with open(file_path, "w") as f:
f.write(new_hash_text)
if old_hash_text is not None:
assert (
new_hash_text == old_hash_text
), f"Hash value for {model_path} is not consistent."
else:
raise ValueError("Erroneous case occurs.")
assert os.path.exists(file_path), f"{file_path} does not exist."


def main(args):
model_path = args.model_path
with open(f"{model_path}/model.py", "r") as fp:
model_str = fp.read()
model_str = extract_from_forward_regex(model_str)
_save_to_model_path(model_path, _get_sha_hash(model_str))
check_graph_hash(args)

model_path = args.model_path
model_class = load_class_from_file(
Expand Down Expand Up @@ -100,17 +120,16 @@ def main(args):
required=True,
help="Path to folder e.g '../test_dataset'",
)

parser.add_argument(
"--no-check-redundancy",
action="store_true",
default=False,
help="whether check model graph redundancy",
)

parser.add_argument(
"--dump-graph-hash-key",
action="store_true",
default=False,
default=True,
help="Dump graph hash key",
)
parser.add_argument(
Expand Down
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/bert-base-cased/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
f2b5a332b1b19703e7ccfb450de96c9c12244144c7b9d305d20587f772fb6672
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/bert-base-chinese/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
f2b5a332b1b19703e7ccfb450de96c9c12244144c7b9d305d20587f772fb6672
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
f2b5a332b1b19703e7ccfb450de96c9c12244144c7b9d305d20587f772fb6672
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
f2b5a332b1b19703e7ccfb450de96c9c12244144c7b9d305d20587f772fb6672
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/bert-base-uncased/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
f2b5a332b1b19703e7ccfb450de96c9c12244144c7b9d305d20587f772fb6672
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/bert-large-cased/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1bad8e4fab570ff456bad864ef45a755f07b2e466cced7983a8383abccc8fc7a
02fa10efca360c8ba7818c367cdeb9979e2af8c72cf489913396a1f241bbad07
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/bert-large-uncased/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1bad8e4fab570ff456bad864ef45a755f07b2e466cced7983a8383abccc8fc7a
02fa10efca360c8ba7818c367cdeb9979e2af8c72cf489913396a1f241bbad07
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
f2b5a332b1b19703e7ccfb450de96c9c12244144c7b9d305d20587f772fb6672
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/convbert-base/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
13585d923bee689fb22489ff8e5dde8cfeb28cce83f6caf8b12ecaf8c5194de8
482fd9e9f201b45c2ce0b22b3037878aa3d139cc203fb35c781fd470140ec962
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c79ca3158ca0a5441bd8ba65f292c2f272451af037decd112e852ba87bfb30be
e013c0a1d9173f7db5ed91398ad65fa43154e3bc8ce2e15c2d5a6637ddec61d8
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/convbert-small/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c79ca3158ca0a5441bd8ba65f292c2f272451af037decd112e852ba87bfb30be
55b1fcce22aee360f71154396a1f528446cae70ebd991927c0abf6c06016d201
Original file line number Diff line number Diff line change
@@ -1 +1 @@
18ee8d7a8c41e47428b3f7a1e25c3066445106e38fa8a4e828dc318c6bd602e9
dd86f56770fe419ac8c9d7bd8426e60faa05682c72c30cfdfc7d01a1a3da21c4
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-1.0-base-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
40fde6163a995d989050cf8b78b44132b4b62ce218f604dad67aff1f4f5a56f0
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1bad8e4fab570ff456bad864ef45a755f07b2e466cced7983a8383abccc8fc7a
3a7feb40180804f8f08663997cafa9e489e00e701df32aa996bfb102b8fbd830
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-1.0/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
2ea1f7f9bb52a294ff9fb5fd9876b9e9ed8b4af2fdb6cce93985eedfe50c7a94
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-2.0-base-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
40fde6163a995d989050cf8b78b44132b4b62ce218f604dad67aff1f4f5a56f0
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-2.0-large-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1bad8e4fab570ff456bad864ef45a755f07b2e466cced7983a8383abccc8fc7a
3a7feb40180804f8f08663997cafa9e489e00e701df32aa996bfb102b8fbd830
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-3.0-base-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
18ee8d7a8c41e47428b3f7a1e25c3066445106e38fa8a4e828dc318c6bd602e9
dd86f56770fe419ac8c9d7bd8426e60faa05682c72c30cfdfc7d01a1a3da21c4
Original file line number Diff line number Diff line change
@@ -1 +1 @@
9764f79bedc8e7c456ee5c7b25cbb9abdea2729f61e438aa8dbffe315df8fe32
48344d251c95a3f7f38838cf0ab9657eac5cb033f7b03132c1934adddde24f5d
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-3.0-micro-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
cc4e3aafe4d3ee7b6dcf72ffc2d76236ccf37e7be99eb034918fc289da0d0456
245f9c8ca479848f19fd1a5d3ff83eaf92645a86cb53968ece04fe3191bf511a
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-3.0-mini-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
9764f79bedc8e7c456ee5c7b25cbb9abdea2729f61e438aa8dbffe315df8fe32
58e65c27f593dcea8ddd01589ed498ffa8d2d8640a1e1097cfde261a3fe232c3
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-3.0-nano-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
cc4e3aafe4d3ee7b6dcf72ffc2d76236ccf37e7be99eb034918fc289da0d0456
076fbe76fccfe420c82cfec8b7b3a439d5950e5e8968a0d66398c4e26807d3aa
Original file line number Diff line number Diff line change
@@ -1 +1 @@
18ee8d7a8c41e47428b3f7a1e25c3066445106e38fa8a4e828dc318c6bd602e9
dd86f56770fe419ac8c9d7bd8426e60faa05682c72c30cfdfc7d01a1a3da21c4
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
1d1c6e22f774dfa1dcc245efdf6538aa8d111dc9ba366e91de0ad0b1987f28a4
Original file line number Diff line number Diff line change
@@ -1 +1 @@
9764f79bedc8e7c456ee5c7b25cbb9abdea2729f61e438aa8dbffe315df8fe32
48344d251c95a3f7f38838cf0ab9657eac5cb033f7b03132c1934adddde24f5d
Original file line number Diff line number Diff line change
@@ -1 +1 @@
af4e387d77d1ff4ded1d7a28b14cfd8b001f9f528f91a5620248888096316840
6dda1d769139a2cb61941e6f069bd42f5350ea24d70d8d237dcc3fbe95d28824
Original file line number Diff line number Diff line change
@@ -1 +1 @@
cc4e3aafe4d3ee7b6dcf72ffc2d76236ccf37e7be99eb034918fc289da0d0456
245f9c8ca479848f19fd1a5d3ff83eaf92645a86cb53968ece04fe3191bf511a
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1de71358d130138fb30b80908decdb96cf5864b7d6f20f0100f99e9932b474bf
8c7bd4ff0aadb54fd17dcac3b7fc5c530fab0ac7ef931ee435f82836f7f56e1a
Original file line number Diff line number Diff line change
@@ -1 +1 @@
9764f79bedc8e7c456ee5c7b25cbb9abdea2729f61e438aa8dbffe315df8fe32
58e65c27f593dcea8ddd01589ed498ffa8d2d8640a1e1097cfde261a3fe232c3
Original file line number Diff line number Diff line change
@@ -1 +1 @@
af4e387d77d1ff4ded1d7a28b14cfd8b001f9f528f91a5620248888096316840
dc0063011bb320a7f5931fea39b1ac41c89b18bb48c901cbc49ebbd4f4daf247
Original file line number Diff line number Diff line change
@@ -1 +1 @@
cc4e3aafe4d3ee7b6dcf72ffc2d76236ccf37e7be99eb034918fc289da0d0456
076fbe76fccfe420c82cfec8b7b3a439d5950e5e8968a0d66398c4e26807d3aa
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1de71358d130138fb30b80908decdb96cf5864b7d6f20f0100f99e9932b474bf
57fc54f7e6af8a83fd3fc5d76dfd850fcb09c0031f50a2b12d9fe305fa8ebb27
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1f334ce1cc6eed8751a5037242055135405e8b39896c0df7a27b91dd6054eaef
e7e1a39e89be40667a96c4328ee450e998bcdbfd63561a1124ff6bcda7acddfd
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-3.0-xbase-zh/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
86d96e779e9dcd1a787f60c3531d0b5df8a888561d06718439411fed2701e29f
fab73a89eb89acf2cc5148bcd7dba425ac19d782b823482d72c7b7a896e6c946
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-m-base/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
ad67e1fb3b084a9365843c9924a2902b07256db3c25afdeaa947b060cd80d603
bc1692f95f97cd0e08a9a8d9d9024df027c12c0a80b8a4980e0916daf7d7a2dc
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-m-large/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c92ad5128f98551c09e66473e78c43f18d92c4a5215e031f6eba588520447750
f5a2b5a3ed88293b1643f5ff48d787655038e15be4e19c51cab914e3f75c74dc
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c1e7e52eab55414cee7c44a9e8c4f81bbd59e3837b185e179e6317efa04f69ec
58f6f8319a02432f55efca078df4c74b372dd0ba432afaa4ce769ca3862fd927
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1bad8e4fab570ff456bad864ef45a755f07b2e466cced7983a8383abccc8fc7a
a0eb1177220779b3dfce38fa401f5f980dbbc62a121ed740b1e52192a0f9736e
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ernie-tiny/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1f334ce1cc6eed8751a5037242055135405e8b39896c0df7a27b91dd6054eaef
6dedb596800536d3d576bd324fe499585d5ea55379abce2325c472fc742e7a74
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/facebook_llama-7b/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
b50836afe93fe05789323270e59637a643d8249457d8f86b73fab4b09c9dea70
d426eda7b3e49ffe4fc7dab17e51c07ac39c7fd99f06d5d7431a7d1b34b98dbc
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/gpt2-medium-en/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
75761733f5bcdb34dd40225df5a9290f48687b3c07b60818b866cfffe75063b0
7f1c8957fca37619a68f2a59e3a8967f7dba7c14dbea4ee0df259e0f2318f143
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/nezha-base-chinese/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
5a0582c526dfbc3f2d25583463af74a3851e2d5f1aef0277592daa48a5ef972c
222830e57f3442cd04d555c7f785735dc98739f4463f0faf4bb57ca27b361d9a
Original file line number Diff line number Diff line change
@@ -1 +1 @@
5a0582c526dfbc3f2d25583463af74a3851e2d5f1aef0277592daa48a5ef972c
222830e57f3442cd04d555c7f785735dc98739f4463f0faf4bb57ca27b361d9a
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0a5f4e8f2f08d6d432de26d35efc9813d7f72661c10076db8451f109275a2f6c
08ac06b39c04a5a0d4eb4d6d5d4783eb99fc5998dac8e993e01bacdb53b465d7
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0a5f4e8f2f08d6d432de26d35efc9813d7f72661c10076db8451f109275a2f6c
08ac06b39c04a5a0d4eb4d6d5d4783eb99fc5998dac8e993e01bacdb53b465d7
2 changes: 1 addition & 1 deletion paddle_samples/PaddleNLP/ppminilm-6l-768h/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
af4e387d77d1ff4ded1d7a28b14cfd8b001f9f528f91a5620248888096316840
a0f0737d524f8e4ca4f1e056e556480a1b89c8d7b138c32cd9797d398a04a9ff
Original file line number Diff line number Diff line change
@@ -1 +1 @@
d86297d73ef9dc4f94aa9469ce74b3907d2acccb9cf55c4159cbc65cd4f8f85d
0f79e348a95fcebb8d8879f8c542c184d72c1ac615c928c17e6699e289b3a914
Original file line number Diff line number Diff line change
@@ -1 +1 @@
d86297d73ef9dc4f94aa9469ce74b3907d2acccb9cf55c4159cbc65cd4f8f85d
effb9e0c852b7bcd7d668832c3b1569603ef15ecce91c6befeabab7e652f3efc
Original file line number Diff line number Diff line change
@@ -1 +1 @@
ebeb915a78b15d8213af7bd64296cf76ef62f4a8b483758b9d39e7f2f53268ae
743dc9654e25ceb1d1af7938d15f50f2516b335e5d85e66d1c44c4376c7443e5
Original file line number Diff line number Diff line change
@@ -1 +1 @@
d86297d73ef9dc4f94aa9469ce74b3907d2acccb9cf55c4159cbc65cd4f8f85d
ac7c29f0c10115309e4ac782586e1a5a0fd80293790c956e696119a24cd5dc54
Original file line number Diff line number Diff line change
@@ -1 +1 @@
d86297d73ef9dc4f94aa9469ce74b3907d2acccb9cf55c4159cbc65cd4f8f85d
ac7c29f0c10115309e4ac782586e1a5a0fd80293790c956e696119a24cd5dc54
Original file line number Diff line number Diff line change
@@ -1 +1 @@
ebeb915a78b15d8213af7bd64296cf76ef62f4a8b483758b9d39e7f2f53268ae
d5a06418c2f1d40774413b6a84ed47630ccffeea660744a4d750e6345dac1d7d
Original file line number Diff line number Diff line change
@@ -1 +1 @@
ebeb915a78b15d8213af7bd64296cf76ef62f4a8b483758b9d39e7f2f53268ae
d5a06418c2f1d40774413b6a84ed47630ccffeea660744a4d750e6345dac1d7d
Original file line number Diff line number Diff line change
@@ -1 +1 @@
ebeb915a78b15d8213af7bd64296cf76ef62f4a8b483758b9d39e7f2f53268ae
a2b91ca0dcfe90f676cebdd8ce70ed25ab4a19a0e5d3d7f6d3afbe824b8f5940
Original file line number Diff line number Diff line change
@@ -1 +1 @@
7e07783670d7000424d3aa58b38b90330cc4c7e43dcc84f5ce318e21ce9bfd98
0337f9ba075f621057b51aa2473d06aa2a767fe5a38a69020fecee4788e01897
Original file line number Diff line number Diff line change
@@ -1 +1 @@
7e07783670d7000424d3aa58b38b90330cc4c7e43dcc84f5ce318e21ce9bfd98
ea25b56bfbf5b928d72465c4b78f7fd85dbf465d0d7e561fc82d62dac602051d
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1bad8e4fab570ff456bad864ef45a755f07b2e466cced7983a8383abccc8fc7a
14b2d1e4140658134efe653f341f7afe466787ae6d6aa309d9669566846ee682
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1bad8e4fab570ff456bad864ef45a755f07b2e466cced7983a8383abccc8fc7a
3f6835de7964f219d2277cfacbe9da8e681a61e90a50d626f4811d49b4366418
Original file line number Diff line number Diff line change
@@ -1 +1 @@
6f3dcd3f048c2f76bf01f0b9d90372d402b7ed0431ad5102bd9f54d53cbada4c
cc80681add836df428c67ac97235458fb43a7b287cc481c428a59a798f0405e1
Original file line number Diff line number Diff line change
@@ -1 +1 @@
8b51d7c27aa8386c19c24188ab1b30250189db4262db16ec473fd8581b36358d
6a200693d12f8fd9ed7f82ed145f098358c6110c9eea5a325db33858fd9a970c
2 changes: 1 addition & 1 deletion paddle_samples/PaddleX/AutoEncoder_ad/graph_hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
fba682c4e5997ff32e9ba905886524cfe92e6d474b10ab52e7cbc96952bb163a
2c49ff97f987487128eb08aba6769445168406254145db3293651d1a6063fa63
Loading