python ../convert-hf-to-gguf.py phi-2/
../quantize phi-2/ggml-model-f16.gguf phi-2/ggml-model-q4_0.gguf q4_0
../main phi-2/ggml-model-q4_0.gguf -p "what about 1+1"

error loading model: create_tensor: tensor 'token_embd.weight' not found
llama_load_model_from_file: failed to load model
llama_init_from_gpt_params: error: failed to load model 'models/nekomata-14b-q4_0.gguf'

Administrator in ~ via  v21.4.0
❯ pip install vllm
Collecting vllm
  Using cached vllm-0.2.6.tar.gz (167 kB)
  Installing build dependencies ... done
  Getting requirements to build wheel ... error
  error: subprocess-exited-with-error

  × Getting requirements to build wheel did not run successfully.
  │ exit code: 1
  ╰─> [22 lines of output]
      C:\Users\Owner\AppData\Local\Temp\pip-build-env-h1dt3he5\overlay\Lib\site-packages\torch\nn\modules\transformer.py:20: UserWarning: Failed to initialize NumPy: No module named 'numpy' (Triggered internally at ..\torch\csrc\utils\tensor_numpy.cpp:84.)
        device: torch.device = torch.device(torch._C._get_default_device()),  # torch.device('cpu'),
      No CUDA runtime is found, using CUDA_HOME='C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2'
      Traceback (most recent call last):
        File "C:\ProgramData\anaconda3\Lib\site-packages\pip\_vendor\pyproject_hooks\_in_process\_in_process.py", line 353, in <module>
          main()
        File "C:\ProgramData\anaconda3\Lib\site-packages\pip\_vendor\pyproject_hooks\_in_process\_in_process.py", line 335, in main
          json_out['return_val'] = hook(**hook_input['kwargs'])
                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        File "C:\ProgramData\anaconda3\Lib\site-packages\pip\_vendor\pyproject_hooks\_in_process\_in_process.py", line 118, in get_requires_for_build_wheel
          return hook(config_settings)
                 ^^^^^^^^^^^^^^^^^^^^^
        File "C:\Users\Owner\AppData\Local\Temp\pip-build-env-h1dt3he5\overlay\Lib\site-packages\setuptools\build_meta.py", line 325, in get_requires_for_build_wheel
          return self._get_build_requires(config_settings, requirements=['wheel'])
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        File "C:\Users\Owner\AppData\Local\Temp\pip-build-env-h1dt3he5\overlay\Lib\site-packages\setuptools\build_meta.py", line 295, in _get_build_requires
          self.run_setup()
        File "C:\Users\Owner\AppData\Local\Temp\pip-build-env-h1dt3he5\overlay\Lib\site-packages\setuptools\build_meta.py", line 311, in run_setup
          exec(code, locals())
        File "<string>", line 298, in <module>
        File "<string>", line 268, in get_vllm_version
      NameError: name 'nvcc_cuda_version' is not defined. Did you mean: 'cuda_version'?
      [end of output]

  note: This error originates from a subprocess, and is likely not a problem with pip.
error: subprocess-exited-with-error

× Getting requirements to build wheel did not run successfully.
│ exit code: 1
╰─> See above for output.

note: This error originates

pip install sentencepiece accelerate protobuf

root@DESKTOP-2TQ96U5:/mnt/c/Users/Owner/work/private/ai/LLM/llm-jp-eval2# python swallow-sample.py 
Traceback (most recent call last):
  File "/mnt/c/Users/Owner/work/private/ai/LLM/llm-jp-eval2/swallow-sample.py", line 6, in <module>
    tokenizer = AutoTokenizer.from_pretrained("./Swallow-7b-instruct-hf")
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/.anyenv/envs/pyenv/versions/3.11.4/lib/python3.11/site-packages/transformers/models/auto/tokenization_auto.py", line 787, in from_pretrained
    return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/.anyenv/envs/pyenv/versions/3.11.4/lib/python3.11/site-packages/transformers/tokenization_utils_base.py", line 2028, in from_pretrained
    return cls._from_pretrained(
           ^^^^^^^^^^^^^^^^^^^^^
  File "/root/.anyenv/envs/pyenv/versions/3.11.4/lib/python3.11/site-packages/transformers/tokenization_utils_base.py", line 2260, in _from_pretrained
    tokenizer = cls(*init_inputs, **init_kwargs)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/.anyenv/envs/pyenv/versions/3.11.4/lib/python3.11/site-packages/transformers/models/llama/tokenization_llama_fast.py", line 124, in __init__
    super().__init__(
  File "/root/.anyenv/envs/pyenv/versions/3.11.4/lib/python3.11/site-packages/transformers/tokenization_utils_fast.py", line 120, in __init__
    raise ValueError(
ValueError: Couldn't instantiate the backend tokenizer from one of: 
(1) a `tokenizers` library serialization file, 
(2) a slow tokenizer instance to convert or 
(3) an equivalent slow tokenizer class to instantiate and convert. 
You need to have sentencepiece installed to convert a slow tokenizer to a fast one.

root@DESKTOP-2TQ96U5:/mnt/c/Users/Owner/work/private/ai/LLM/llm-jp-eval2# CUDA_VISIBLE_DEVICES=0 python scripts/evaluate_llm.py -cn config.yaml model.pretrained_model_name_or_path=Swallow-7b-instruct-hf tokenizer.pretrained_model_name_or_path=Swallow-7b-instruct-hf dataset_dir=sample/1.1.0/evaluation/test/

git clone git@hf.co:tokyotech-llm/Swallow-7b-instruct-hf
python swallow-sample.py
python scripts/preprocess_dataset.py --dataset-name all --output-dir sample

root@DESKTOP-2TQ96U5:/mnt/c/Users/Owner/work/private/ai/LLM/llm-jp-eval2# CUDA_VISIBLE_DEVICES=0 python scripts/evaluate_llm.py -cn config.yaml model.pretrained_model_name_or_path=Swallow-7b-instruct-hf tokenizer.pretrained_model_name_or_path=Swallow-7b-instruct-hf dataset_dir=sample/1.1.0/evaluation/test/
[2023-12-30 07:34:52,696][accelerate.utils.modeling][INFO] - We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [04:32<00:00, 90.99s/it]
jamp:   0%|                                                                                                                                                                                                                                         | 0/100 [00:00<?, ?it/s]wandb: (1) Create a W&B account
wandb: (2) Use an existing W&B account
wandb: (3) Don't visualize my results
wandb: Enter your choice: 3
wandb: You chose "Don't visualize my results"
wandb: W&B syncing is set to `offline` in this directory.  
wandb: Run `wandb online` or set WANDB_MODE=online to enable cloud syncing.
wandb: Streaming LangChain activity to W&B at 
wandb: `WandbTracer` is currently in beta.
wandb: Please report any issues to https://github.com/wandb/wandb/issues with the tag `langchain`.
Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.
jamp:   1%|██▎                                                                                                                                                                                                                              | 1/100 [00:18<30:46, 18.65s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.
jamp:   2%|████▌                                                  

BitsAndBytesConfigなどの理解は必要だけどね

## 出てきた謎のエラーたち
* FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.
* The model is automatically converting to bf16 for faster inference. If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to "AutoModelForCausalLM.from_pretrained".
* ValueError: `.to` is not supported for `4-bit` or `8-bit` bitsandbytes models. Please use the model as it is, since the model has already been set to the correct devices and casted to the correct `dtype`.
* You shouldn't move a model when it is dispatched on multiple devices.
* The model is automatically converting to bf16 for faster inference. If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to "AutoModelForCausalLM.from_pretrained".
* FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.