298977887/ComfyUI
1
0
Fork 0
mirror of https://github.com/comfyanonymous/ComfyUI.git synced 2025-04-20 03:13:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix hard crash when the spiece tokenizer path is bad.

Browse Source
This commit is contained in:
comfyanonymous 2025-04-19 15:55:43 -04:00
parent dc300a4569
commit 636d4bfb89
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
comfy/text_encoders/spiece_tokenizer.py
View File

@ -1,4 +1,5 @@
import torch
import os
class SPieceTokenizer:
@staticmethod
@ -15,6 +16,8 @@ class SPieceTokenizer:
if isinstance(tokenizer_path, bytes):
self.tokenizer = sentencepiece.SentencePieceProcessor(model_proto=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos)
else:
if not os.path.isfile(tokenizer_path):
raise ValueError("invalid tokenizer")
self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos)
def get_vocab(self):