HFValidationError Traceback (most recent call last)
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/configuration_utils.py:628, in PretrainedConfig._get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
626 try:
627 # Load from local folder or from cache or download from model Hub and cache
--> 628 resolved_config_file = cached_file(
629 pretrained_model_name_or_path,
630 configuration_file,
631 cache_dir=cache_dir,
632 force_download=force_download,
633 proxies=proxies,
634 resume_download=resume_download,
635 local_files_only=local_files_only,
636 use_auth_token=use_auth_token,
637 user_agent=user_agent,
638 revision=revision,
639 subfolder=subfolder,
640 _commit_hash=commit_hash,
641 )
642 commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/utils/hub.py:409, in cached_file(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, use_auth_token, revision, local_files_only, subfolder, user_agent, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash)
407 try:
408 # Load from URL or cache if already cached
--> 409 resolved_file = hf_hub_download(
410 path_or_repo_id,
411 filename,
412 subfolder=None if len(subfolder) == 0 else subfolder,
413 revision=revision,
414 cache_dir=cache_dir,
415 user_agent=user_agent,
416 force_download=force_download,
417 proxies=proxies,
418 resume_download=resume_download,
419 use_auth_token=use_auth_token,
420 local_files_only=local_files_only,
421 )
423 except RepositoryNotFoundError:
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:106, in validate_hf_hub_args.._inner_fn(*args, **kwargs)
105 if arg_name in ["repo_id", "from_id", "to_id"]:
--> 106 validate_repo_id(arg_value)
108 elif arg_name == "token" and arg_value is not None:
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:154, in validate_repo_id(repo_id)
153 if repo_id.count("/") > 1:
--> 154 raise HFValidationError(
155 "Repo id must be in the form 'repo_name' or 'namespace/repo_name':"
156 f" '{repo_id}'. Use repo_type argument if needed."
157 )
159 if not REPO_ID_REGEX.match(repo_id):
HFValidationError: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/netscratch/duynguyen/Research/Nghiem_LLaVA-Med/LVLM-Med/clip-vit-large-patch14'. Use repo_type argument if needed.
During handling of the above exception, another exception occurred:
OSError Traceback (most recent call last)
Cell In[3], line 17
14 # 1) Load model via eval_model-style loader
15 model_dir = "/mnt/sdz/abka03_data/S-Chain/architectures/Exgra-Med/checkpoints/exgra-med-gpt-schain"
---> 17 model, tokenizer, image_processor, image_token_len, mm_use_im_start_end, device = load_exgra_med_schain(
18 model_name=model_dir,
19 mm_dense_connector_type="none",
20 num_l=6,
21 contrastive=False,
22 mm_projector=None,
23 device="cuda:1",
24 )
26 print("Using device:", device)
28 # 2) Prepare image tensor
Cell In[2], line 74
71 setattr(vision_tower, "config", vision_config)
72 else:
73 # ✅ this is the branch for exgra-med-gpt-schain
---> 74 model = LlavaLlamaForCausalLM.from_pretrained(
75 model_name,
76 torch_dtype=torch.float16,
77 use_cache=True,
78 ).to(device)
80 model.model.contrastive = contrastive
81 model.model.mm_dense_connector_type = mm_dense_connector_type
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/modeling_utils.py:2493, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
2490 init_contexts.append(init_empty_weights())
2492 with ContextManagers(init_contexts):
-> 2493 model = cls(config, *model_args, **model_kwargs)
2495 # Check first if we are from_pt
2496 if use_keep_in_fp32_modules:
File /mnt/sdz/abka03_data/S-Chain/architectures/Exgra-Med/llava/model/llava.py:353, in LlavaLlamaForCausalLM.init(self, config)
351 def init(self, config): # config is taken from file /netscratch/trnguyen/llava_med_checkpoints_llama/LLaVA-7b-v0/config.json
352 super(LlamaForCausalLM, self).init(config)
--> 353 self.model = LlavaLlamaModel(config)
355 self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
357 # Initialize weights and apply final processing
File /mnt/sdz/abka03_data/S-Chain/architectures/Exgra-Med/llava/model/llava.py:91, in LlavaLlamaModel.init(self, config, mm_vision_tower, mm_hidden_size)
89 self.vision_tower_name = "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
90 else:
---> 91 self.vision_tower = [CLIPVisionModel.from_pretrained(config.mm_vision_tower)]
94 #if hasattr(config, "use_mm_proj"):
95 if hasattr(config, "mm_projector_type"):
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/modeling_utils.py:2170, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
2168 if not isinstance(config, PretrainedConfig):
2169 config_path = config if config is not None else pretrained_model_name_or_path
-> 2170 config, model_kwargs = cls.config_class.from_pretrained(
2171 config_path,
2172 cache_dir=cache_dir,
2173 return_unused_kwargs=True,
2174 force_download=force_download,
2175 resume_download=resume_download,
2176 proxies=proxies,
2177 local_files_only=local_files_only,
2178 use_auth_token=use_auth_token,
2179 revision=revision,
2180 subfolder=subfolder,
2181 _from_auto=from_auto_class,
2182 _from_pipeline=from_pipeline,
2183 **kwargs,
2184 )
2185 else:
2186 model_kwargs = kwargs
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/models/clip/configuration_clip.py:233, in CLIPVisionConfig.from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
231 @classmethod
232 def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
--> 233 config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
235 # get the vision config dict if we are loading from CLIPConfig
236 if config_dict.get("model_type") == "clip":
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/configuration_utils.py:573, in PretrainedConfig.get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
571 original_kwargs = copy.deepcopy(kwargs)
572 # Get config dict associated with the base config file
--> 573 config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
574 if "_commit_hash" in config_dict:
575 original_kwargs["_commit_hash"] = config_dict["_commit_hash"]
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/configuration_utils.py:649, in PretrainedConfig._get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
646 raise
647 except Exception:
648 # For any other exception, we throw a generic error.
--> 649 raise EnvironmentError(
650 f"Can't load the configuration of '{pretrained_model_name_or_path}'. If you were trying to load it"
651 " from 'https://huggingface.co/models', make sure you don't have a local directory with the same"
652 f" name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory"
653 f" containing a {configuration_file} file"
654 )
656 try:
657 # Load config dict
658 config_dict = cls._dict_from_json_file(resolved_config_file)
OSError: Can't load the configuration of '/netscratch/duynguyen/Research/Nghiem_LLaVA-Med/LVLM-Med/clip-vit-large-patch14'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure '/netscratch/duynguyen/Research/Nghiem_LLaVA-Med/LVLM-Med/clip-vit-large-patch14' is the correct path to a directory containing a config.json file
HFValidationError Traceback (most recent call last)
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/configuration_utils.py:628, in PretrainedConfig._get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
626 try:
627 # Load from local folder or from cache or download from model Hub and cache
--> 628 resolved_config_file = cached_file(
629 pretrained_model_name_or_path,
630 configuration_file,
631 cache_dir=cache_dir,
632 force_download=force_download,
633 proxies=proxies,
634 resume_download=resume_download,
635 local_files_only=local_files_only,
636 use_auth_token=use_auth_token,
637 user_agent=user_agent,
638 revision=revision,
639 subfolder=subfolder,
640 _commit_hash=commit_hash,
641 )
642 commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/utils/hub.py:409, in cached_file(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, use_auth_token, revision, local_files_only, subfolder, user_agent, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash)
407 try:
408 # Load from URL or cache if already cached
--> 409 resolved_file = hf_hub_download(
410 path_or_repo_id,
411 filename,
412 subfolder=None if len(subfolder) == 0 else subfolder,
413 revision=revision,
414 cache_dir=cache_dir,
415 user_agent=user_agent,
416 force_download=force_download,
417 proxies=proxies,
418 resume_download=resume_download,
419 use_auth_token=use_auth_token,
420 local_files_only=local_files_only,
421 )
423 except RepositoryNotFoundError:
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:106, in validate_hf_hub_args.._inner_fn(*args, **kwargs)
105 if arg_name in ["repo_id", "from_id", "to_id"]:
--> 106 validate_repo_id(arg_value)
108 elif arg_name == "token" and arg_value is not None:
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:154, in validate_repo_id(repo_id)
153 if repo_id.count("/") > 1:
--> 154 raise HFValidationError(
155 "Repo id must be in the form 'repo_name' or 'namespace/repo_name':"
156 f" '{repo_id}'. Use
repo_typeargument if needed."157 )
159 if not REPO_ID_REGEX.match(repo_id):
HFValidationError: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/netscratch/duynguyen/Research/Nghiem_LLaVA-Med/LVLM-Med/clip-vit-large-patch14'. Use
repo_typeargument if needed.During handling of the above exception, another exception occurred:
OSError Traceback (most recent call last)
Cell In[3], line 17
14 # 1) Load model via eval_model-style loader
15 model_dir = "/mnt/sdz/abka03_data/S-Chain/architectures/Exgra-Med/checkpoints/exgra-med-gpt-schain"
---> 17 model, tokenizer, image_processor, image_token_len, mm_use_im_start_end, device = load_exgra_med_schain(
18 model_name=model_dir,
19 mm_dense_connector_type="none",
20 num_l=6,
21 contrastive=False,
22 mm_projector=None,
23 device="cuda:1",
24 )
26 print("Using device:", device)
28 # 2) Prepare image tensor
Cell In[2], line 74
71 setattr(vision_tower, "config", vision_config)
72 else:
73 # ✅ this is the branch for exgra-med-gpt-schain
---> 74 model = LlavaLlamaForCausalLM.from_pretrained(
75 model_name,
76 torch_dtype=torch.float16,
77 use_cache=True,
78 ).to(device)
80 model.model.contrastive = contrastive
81 model.model.mm_dense_connector_type = mm_dense_connector_type
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/modeling_utils.py:2493, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
2490 init_contexts.append(init_empty_weights())
2492 with ContextManagers(init_contexts):
-> 2493 model = cls(config, *model_args, **model_kwargs)
2495 # Check first if we are
from_pt2496 if use_keep_in_fp32_modules:
File /mnt/sdz/abka03_data/S-Chain/architectures/Exgra-Med/llava/model/llava.py:353, in LlavaLlamaForCausalLM.init(self, config)
351 def init(self, config): # config is taken from file /netscratch/trnguyen/llava_med_checkpoints_llama/LLaVA-7b-v0/config.json
352 super(LlamaForCausalLM, self).init(config)
--> 353 self.model = LlavaLlamaModel(config)
355 self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
357 # Initialize weights and apply final processing
File /mnt/sdz/abka03_data/S-Chain/architectures/Exgra-Med/llava/model/llava.py:91, in LlavaLlamaModel.init(self, config, mm_vision_tower, mm_hidden_size)
89 self.vision_tower_name = "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
90 else:
---> 91 self.vision_tower = [CLIPVisionModel.from_pretrained(config.mm_vision_tower)]
94 #if hasattr(config, "use_mm_proj"):
95 if hasattr(config, "mm_projector_type"):
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/modeling_utils.py:2170, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
2168 if not isinstance(config, PretrainedConfig):
2169 config_path = config if config is not None else pretrained_model_name_or_path
-> 2170 config, model_kwargs = cls.config_class.from_pretrained(
2171 config_path,
2172 cache_dir=cache_dir,
2173 return_unused_kwargs=True,
2174 force_download=force_download,
2175 resume_download=resume_download,
2176 proxies=proxies,
2177 local_files_only=local_files_only,
2178 use_auth_token=use_auth_token,
2179 revision=revision,
2180 subfolder=subfolder,
2181 _from_auto=from_auto_class,
2182 _from_pipeline=from_pipeline,
2183 **kwargs,
2184 )
2185 else:
2186 model_kwargs = kwargs
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/models/clip/configuration_clip.py:233, in CLIPVisionConfig.from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
231 @classmethod
232 def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
--> 233 config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
235 # get the vision config dict if we are loading from CLIPConfig
236 if config_dict.get("model_type") == "clip":
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/configuration_utils.py:573, in PretrainedConfig.get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
571 original_kwargs = copy.deepcopy(kwargs)
572 # Get config dict associated with the base config file
--> 573 config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
574 if "_commit_hash" in config_dict:
575 original_kwargs["_commit_hash"] = config_dict["_commit_hash"]
File /mnt/sdz/abka03_data/env/schain/lib/python3.10/site-packages/transformers/configuration_utils.py:649, in PretrainedConfig._get_config_dict(cls, pretrained_model_name_or_path, **kwargs)
646 raise
647 except Exception:
648 # For any other exception, we throw a generic error.
--> 649 raise EnvironmentError(
650 f"Can't load the configuration of '{pretrained_model_name_or_path}'. If you were trying to load it"
651 " from 'https://huggingface.co/models', make sure you don't have a local directory with the same"
652 f" name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory"
653 f" containing a {configuration_file} file"
654 )
656 try:
657 # Load config dict
658 config_dict = cls._dict_from_json_file(resolved_config_file)
OSError: Can't load the configuration of '/netscratch/duynguyen/Research/Nghiem_LLaVA-Med/LVLM-Med/clip-vit-large-patch14'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure '/netscratch/duynguyen/Research/Nghiem_LLaVA-Med/LVLM-Med/clip-vit-large-patch14' is the correct path to a directory containing a config.json file