看到新聞寫聯發科釋出新的LLM模型，還可以做圖像處理或文件處理，
找到在 huggingface 上的model 和簡易的使用方法
https://huggingface.co/MediaTek-Research/Llama-Breeze2-3B-Instruct
https://huggingface.co/MediaTek-Research/Llama-Breeze2-8B-Instruct
但是實際上還需要補上不少的python 套件
如果是另外新建立 anaconda 的環境，
會需要再安裝 jupiter-notebook
#conda install jupiter-notebook
#conda install anaconda-navigator
#conda install conda-forge::jupyter anaconda::ipywidgets
文件說明需要 transformers
#conda install transformers
所有安裝指令
==================
#conda install conda-forge::pytorch-gpu
#conda install -c conda-forge libstdcxx-ng
#pip install mtkresearch
#conda install pytorch torchvision torchaudio cudatoolkit=11.8 -c pytorch
#conda install pytorch::pytorch-cuda
#conda install timm einops accelerate
==================
逐步解說
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
Cell In[2], line 3
1 from transformers import AutoModel, AutoTokenizer
2 from transformers import GenerationConfig
----> 3 import torch
4 from mtkresearch.llm.prompt import MRPromptV3
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torch/__init__.py:367
365 if USE_GLOBAL_DEPS:
366 _load_global_deps()
--> 367 from torch._C import * # noqa: F403
370 class SymInt:
371 """
372 Like an int (including magic methods), but redirects all operations on the
373 wrapped node. This is used in particular to symbolically record operations
374 in the symbolic shape workflow.
375 """
ImportError: libtorch_cpu.so: cannot enable executable stack as shared object requires: Invalid argument
https://anaconda.org/conda-forge/pytorch-gpu
#conda install conda-forge::pytorch-gpu
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
Cell In[3], line 3
1 from transformers import AutoModel, AutoTokenizer
2 from transformers import GenerationConfig
----> 3 import torch
4 from mtkresearch.llm.prompt import MRPromptV3
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torch/__init__.py:290
289 def _load_global_deps() -> None:
--> 290 if _running_with_deploy() or platform.system() == "Windows":
291 return
293 # Determine the file extension based on the platform
ImportError: /home/rexct/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/zmq/backend/cython/../../../../.././libstdc++.so.6: version GLIBCXX_3.4.30' not found (required by /home/rexct/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torch/../../../libtorch_python.so)
#conda install -c conda-forge libstdcxx-ng
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[2], line 4
2 from transformers import GenerationConfig
3 import torch
----> 4 from mtkresearch.llm.prompt import MRPromptV3
ModuleNotFoundError: No module named 'mtkresearch'
#pip install mtkresearch
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[3], line 4
2 from transformers import GenerationConfig
3 import torch
----> 4 from mtkresearch.llm.prompt import MRPromptV3
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/mtkresearch/llm/prompt.py:9
6 from datetime import datetime
7 from PIL import Image
----> 9 from mtkresearch.llm.image_process_v3 import load_image
12 def _removeprefix(content, prefix):
13 if sys.version_info[0] >= 3 and sys.version_info[1] >= 9:
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/mtkresearch/llm/image_process_v3.py:3
1 from PIL import Image
2 import torch
----> 3 import torchvision.transforms as T
4 from torchvision.transforms.functional import InterpolationMode
6 IMAGENET_MEAN = (0.485, 0.456, 0.406)
ModuleNotFoundError: No module named 'torchvision'
到下列網頁看最新cudatoolkit 版本，
目前是11.8
https://anaconda.org/conda-forge/cudatoolkit
#conda install pytorch torchvision torchaudio cudatoolkit=11.8 -c pytorch如果出現以下錯誤，
表示 pytorch 和 torchvision 不合
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Cell In[4], line 4
2 from transformers import GenerationConfig
3 import torch
----> 4 from mtkresearch.llm.prompt import MRPromptV3
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/mtkresearch/llm/prompt.py:9
6 from datetime import datetime
7 from PIL import Image
----> 9 from mtkresearch.llm.image_process_v3 import load_image
12 def _removeprefix(content, prefix):
13 if sys.version_info[0] >= 3 and sys.version_info[1] >= 9:
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/mtkresearch/llm/image_process_v3.py:3
1 from PIL import Image
2 import torch
----> 3 import torchvision.transforms as T
4 from torchvision.transforms.functional import InterpolationMode
6 IMAGENET_MEAN = (0.485, 0.456, 0.406)
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torchvision/__init__.py:10
7 # Don't re-order these, we need to load the _C extension (done when importing
8 # .extensions) before entering _meta_registrations.
9 from .extension import _HAS_OPS # usort:skip
---> 10 from torchvision import _meta_registrations, datasets, io, models, ops, transforms, utils # usort:skip
12 try:
13 from .version import __version__ # noqa: F401
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torchvision/_meta_registrations.py:163
153 torch._check(
154 grad.dtype == rois.dtype,
155 lambda: (
(...)
158 ),
159 )
160 return grad.new_empty((batch_size, channels, height, width))
--> 163 @torch.library.register_fake("torchvision::nms")
164 def meta_nms(dets, scores, iou_threshold):
165 torch._check(dets.dim() == 2, lambda: f"boxes should be a 2d tensor, got {dets.dim()}D")
166 torch._check(dets.size(1) == 4, lambda: f"boxes should have 4 elements in dimension 1, got {dets.size(1)}")
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torch/library.py:654, in register_fake.<locals>.register(func)
652 else:
653 use_lib = lib
--> 654 use_lib._register_fake(op_name, func, _stacklevel=stacklevel + 1)
655 return func
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torch/library.py:154, in Library._register_fake(self, op_name, fn, _stacklevel)
151 else:
152 func_to_register = fn
--> 154 handle = entry.abstract_impl.register(func_to_register, source)
155 self._registration_handles.append(handle)
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torch/_library/abstract_impl.py:31, in AbstractImplHolder.register(self, func, source)
25 if self.kernel is not None:
26 raise RuntimeError(
27 f"register_fake(...): the operator {self.qualname} "
28 f"already has an fake impl registered at "
29 f"{self.kernel.source}."
30 )
---> 31 if torch._C._dispatch_has_kernel_for_dispatch_key(self.qualname, "Meta"):
32 raise RuntimeError(
33 f"register_fake(...): the operator {self.qualname} "
34 f"already has an DispatchKey::Meta implementation via a "
(...)
37 f"register_fake."
38 )
40 if torch._C._dispatch_has_kernel_for_dispatch_key(
41 self.qualname, "CompositeImplicitAutograd"
42 ):
RuntimeError: operator torchvision::nms does not exist
先移除 torchvision
#conda uninstall torchvision
#conda uninstall pytorch
安裝指定版 torchvision
#conda install pytorch::torchvision=0.17.2
#conda install pytorch torchvision torchaudio conda-forge::cudatoolkit=11.8 -c pytorch
執行檢查是否使用CUDA
import torch
torch.cuda.is_available()
有使用會輸出 True
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
Cell In[8], line 4
2 from transformers import GenerationConfig
3 import torch
----> 4 from mtkresearch.llm.prompt import MRPromptV3
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/mtkresearch/llm/prompt.py:9
6 from datetime import datetime
7 from PIL import Image
----> 9 from mtkresearch.llm.image_process_v3 import load_image
12 def _removeprefix(content, prefix):
13 if sys.version_info[0] >= 3 and sys.version_info[1] >= 9:
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/mtkresearch/llm/image_process_v3.py:3
1 from PIL import Image
2 import torch
----> 3 import torchvision.transforms as T
4 from torchvision.transforms.functional import InterpolationMode
6 IMAGENET_MEAN = (0.485, 0.456, 0.406)
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torchvision/__init__.py:6
0 <Error retrieving source code with stack_data see ipython/ipython#13598>
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torchvision/_meta_registrations.py:25
20 return fn
22 return wrapper
---> 25 @register_meta("roi_align")
26 def meta_roi_align(input, rois, spatial_scale, pooled_height, pooled_width, sampling_ratio, aligned):
27 torch._check(rois.size(1) == 5, lambda: "rois must have shape as Tensor[K, 5]")
28 torch._check(
29 input.dtype == rois.dtype,
30 lambda: (
(...)
33 ),
34 )
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/torchvision/_meta_registrations.py:18, in register_meta.<locals>.wrapper(fn)
17 def wrapper(fn):
---> 18 if torchvision.extension._has_ops():
19 get_meta_lib().impl(getattr(getattr(torch.ops.torchvision, op_name), overload_name), fn)
20 return fn
AttributeError: partially initialized module 'torchvision' has no attribute 'extension' (most likely due to a circular import)
可能是因為之前一直測試重複import，
重啟後這個問題就消失了
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
Cell In[2], line 2
1 model_id = 'MediaTek-Research/Llama-Breeze2-3B-Instruct-v0_1'
----> 2 model = AutoModel.from_pretrained(
3 model_id,
4 torch_dtype=torch.bfloat16,
5 low_cpu_mem_usage=True,
6 trust_remote_code=True,
7 device_map='auto',
8 img_context_token_id=128212
9 ).eval()
11 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, use_fast=False)
13 generation_config = GenerationConfig(
14 max_new_tokens=2048,
15 do_sample=True,
(...)
19 eos_token_id=128009
20 )
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py:553, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
551 if has_remote_code and trust_remote_code:
552 class_ref = config.auto_map[cls.__name__]
--> 553 model_class = get_class_from_dynamic_module(
554 class_ref, pretrained_model_name_or_path, code_revision=code_revision, **hub_kwargs, **kwargs
555 )
556 _ = hub_kwargs.pop("code_revision", None)
557 cls.register(config.__class__, model_class, exist_ok=True)
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/transformers/dynamic_module_utils.py:540, in get_class_from_dynamic_module(class_reference, pretrained_model_name_or_path, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, repo_type, code_revision, **kwargs)
538 code_revision = revision
539 # And lastly we get the class inside our newly created module
--> 540 final_module = get_cached_module_file(
541 repo_id,
542 module_file + ".py",
543 cache_dir=cache_dir,
544 force_download=force_download,
545 resume_download=resume_download,
546 proxies=proxies,
547 token=token,
548 revision=code_revision,
549 local_files_only=local_files_only,
550 repo_type=repo_type,
551 )
552 return get_class_in_module(class_name, final_module, force_reload=force_download)
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/transformers/dynamic_module_utils.py:403, in get_cached_module_file(pretrained_model_name_or_path, module_file, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, repo_type, _commit_hash, **deprecated_kwargs)
401 for module_needed in modules_needed:
402 if not (submodule_path / f"{module_needed}.py").exists():
--> 403 get_cached_module_file(
404 pretrained_model_name_or_path,
405 f"{module_needed}.py",
406 cache_dir=cache_dir,
407 force_download=force_download,
408 resume_download=resume_download,
409 proxies=proxies,
410 token=token,
411 revision=revision,
412 local_files_only=local_files_only,
413 _commit_hash=commit_hash,
414 )
415 new_files.append(f"{module_needed}.py")
417 if len(new_files) > 0 and revision is None:
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/transformers/dynamic_module_utils.py:365, in get_cached_module_file(pretrained_model_name_or_path, module_file, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, repo_type, _commit_hash, **deprecated_kwargs)
362 raise
364 # Check we have all the requirements in our environment
--> 365 modules_needed = check_imports(resolved_module_file)
367 # Now we move the module inside our cached dynamic modules.
368 full_submodule = TRANSFORMERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/transformers/dynamic_module_utils.py:197, in check_imports(filename)
194 raise
196 if len(missing_packages) > 0:
--> 197 raise ImportError(
198 "This modeling file requires the following packages that were not found in your environment: "
199 f"{', '.join(missing_packages)}. Run `pip install {' '.join(missing_packages)}`"
200 )
202 return get_relative_imports(filename)
ImportError: This modeling file requires the following packages that were not found in your environment: timm, einops. Run `pip install timm einops`
#conda install timm einops
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
Cell In[3], line 2
1 model_id = 'MediaTek-Research/Llama-Breeze2-3B-Instruct-v0_1'
----> 2 model = AutoModel.from_pretrained(
3 model_id,
4 torch_dtype=torch.bfloat16,
5 low_cpu_mem_usage=True,
6 trust_remote_code=True,
7 device_map='auto',
8 img_context_token_id=128212
9 ).eval()
11 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, use_fast=False)
13 generation_config = GenerationConfig(
14 max_new_tokens=2048,
15 do_sample=True,
(...)
19 eos_token_id=128009
20 )
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py:559, in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
557 else:
558 cls.register(config.__class__, model_class, exist_ok=True)
--> 559 return model_class.from_pretrained(
560 pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
561 )
562 elif type(config) in cls._model_mapping.keys():
563 model_class = _get_model_class(config, cls._model_mapping)
File ~/.conda/envs/mtk-breeze2/lib/python3.12/site-packages/transformers/modeling_utils.py:3318, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, *model_args, **kwargs)
3314 raise ValueError(
3315 "DeepSpeed Zero-3 is not compatible with `low_cpu_mem_usage=True` or with passing a `device_map`."
3316 )
3317 elif not is_accelerate_available():
-> 3318 raise ImportError(
3319 "Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`"
3320 )
3322 # handling bnb config from kwargs, remove after `load_in_{4/8}bit` deprecation.
3323 if load_in_4bit or load_in_8bit:
ImportError: Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`
#conda install accelerate
FlashAttention2 is not installed.
#conda install pytorch::pytorch-cuda
可能會遇到相依性問題，
找出樹狀圖中的 cuda-toolkit 版本，
到下列網址中複製對應的版本執行
https://anaconda.org/nvidia/cuda-toolkit
#conda install nvidia/label/cuda-12.4.1::cuda-toolkit
#conda install pytorch torchvision torchaudio cudatoolkit=11.8 -c pytorch
參考：
https://ithelp.ithome.com.tw/articles/10282119
https://stackoverflow.com/questions/48152674/how-do-i-check-if-pytorch-is-using-the-gpu
