298977887/ComfyUI
1
0
Fork 0
mirror of https://github.com/comfyanonymous/ComfyUI.git synced 2025-01-25 15:55:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename LLAMATokenizer to SPieceTokenizer.

Browse Source
This commit is contained in:
comfyanonymous 2024-07-22 12:21:45 -04:00
parent b2c995f623
commit 14764aa2e2
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
comfy/text_encoders/aura_t5.py
View File

@ -1,5 +1,5 @@
from comfy import sd1_clip
from .llama_tokenizer import LLAMATokenizer
from .spiece_tokenizer import SPieceTokenizer
import comfy.text_encoders.t5
import os
@ -11,7 +11,7 @@ class PT5XlModel(sd1_clip.SDClipModel):
class PT5XlTokenizer(sd1_clip.SDTokenizer):
def __init__(self, embedding_directory=None):
tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model")
super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LLAMATokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=SPieceTokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):
def __init__(self, embedding_directory=None):

4
comfy/text_encoders/llama_tokenizer.py → comfy/text_encoders/spiece_tokenizer.py
View File

@ -1,9 +1,9 @@
import os
class LLAMATokenizer:
class SPieceTokenizer:
@staticmethod
def from_pretrained(path):
return LLAMATokenizer(path)
return SPieceTokenizer(path)
def __init__(self, tokenizer_path):
import sentencepiece