error torch.OutOfMemoryError: CUDA out of memory.

[benjamin920101]

torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 36.71 GiB. GPU 0 has a total capacity of 14.83 GiB of which 1.43 GiB is free. Of the allocated memory 13.20 GiB is allocated by PyTorch, and 52.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
Traceback:
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\chandra\scripts\app.py", line 120, in
result, layout_image = ocr_layout(
^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\chandra\scripts\app.py", line 61, in ocr_layout
result = model.generate([batch])[0]
^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\chandra\model_init_.py", line 35, in generate
results = generate_hf(
^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\chandra\model\hf.py", line 34, in generate_hf
generated_ids = model.generate(**inputs, max_new_tokens=max_output_tokens)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\utils_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\generation\utils.py", line 2564, in generate
result = decoding_method(
^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\generation\utils.py", line 2784, in _sample
outputs = self(**model_inputs, return_dict=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\accelerate\hooks.py", line 175, in new_forward
output = module._old_forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\utils\generic.py", line 1064, in wrapper
outputs = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 1344, in forward
outputs = self.model(
^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\utils\generic.py", line 1064, in wrapper
outputs = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 1138, in forward
image_embeds, deepstack_image_embeds = self.get_image_features(pixel_values, image_grid_thw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 1061, in get_image_features
image_embeds, deepstack_image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\accelerate\hooks.py", line 175, in new_forward
output = module._old_forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 739, in forward
hidden_states = blk(
^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\modeling_layers.py", line 94, in call
return super().call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\accelerate\hooks.py", line 175, in new_forward
output = module._old_forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 267, in forward
hidden_states = hidden_states + self.attn(
^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\accelerate\hooks.py", line 175, in new_forward
output = module._old_forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 231, in forward
attention_interface(
File "C:\Users\4321-1.conda\envs\chandra-ocr\Lib\site-packages\transformers\integrations\sdpa_attention.py", line 96, in sdpa_attention_forward
attn_output = torch.nn.functional.scaled_dot_product_attention(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

[demic-dev]

I got the same error with chandra cli (version 0.1.8) and torch (version 2.9.1), on Linux (and with enough memory on GPU).