add

modeling_moss_tts_nano.py

@@ -1026,23 +1026,47 @@ class MossTTSNanoForCausalLM(MossTTSNanoPreTrainedModel):
             return True
         return False
 
-    def _resolve_text_tokenizer_path(self, raw_path: Union[str, Path]) -> Path:
-        candidate_path = Path(raw_path)
-        if candidate_path.is_file() and candidate_path.suffix == ".model":
-            return candidate_path
+    @staticmethod
+    def _looks_like_hf_repo_id(candidate: str) -> bool:
+        stripped = candidate.strip()
+        if not stripped:
+            return False
+        if stripped.startswith((os.sep, ".", "~")):
+            return False
+        if "\\" in stripped:
+            return False
+        parts = stripped.split("/")
+        return len(parts) == 2 and all(part.strip() for part in parts)
+
+    @staticmethod
+    def _existing_local_path(raw_path: Union[str, Path]) -> Optional[Path]:
+        candidate_path = Path(raw_path).expanduser()
         if not candidate_path.exists():
-            raise FileNotFoundError(f"Tokenizer path does not exist: {candidate_path}")
+            return None
+        return candidate_path.resolve()
+
+    def _resolve_text_tokenizer_path(self, raw_path: Union[str, Path]) -> str:
+        local_candidate_path = self._existing_local_path(raw_path)
+        if local_candidate_path is None:
+            raw_source = str(raw_path).strip()
+            if self._looks_like_hf_repo_id(raw_source):
+                return raw_source
+            raise FileNotFoundError(f"Tokenizer path does not exist: {raw_source}")
+
+        candidate_path = local_candidate_path
+        if candidate_path.is_file() and candidate_path.suffix == ".model":
+            return str(candidate_path)
         if candidate_path.is_dir():
             if (candidate_path / "tokenizer.model").is_file():
-                return candidate_path
+                return str(candidate_path)
             if self._looks_like_hf_tokenizer_dir(candidate_path):
-                return candidate_path
+                return str(candidate_path)
             hf_dir = candidate_path / "hf_tokenizer"
             if self._looks_like_hf_tokenizer_dir(hf_dir):
-                return hf_dir
+                return str(hf_dir)
             sentencepiece_model = candidate_path / "sentencepiece" / "mossttsnano_spm_bpe.model"
             if sentencepiece_model.is_file():
-                return sentencepiece_model
+                return str(sentencepiece_model)
             final_summary_path = candidate_path / "final_summary.json"
             if final_summary_path.is_file():
                 final_summary = json.loads(final_summary_path.read_text(encoding="utf-8"))
@@ -1050,26 +1074,33 @@ class MossTTSNanoForCausalLM(MossTTSNanoPreTrainedModel):
                 if latest_hf_dir:
                     latest_hf_path = Path(str(latest_hf_dir))
                     if self._looks_like_hf_tokenizer_dir(latest_hf_path):
-                        return latest_hf_path
+                        return str(latest_hf_path.resolve())
         raise ValueError(
             "Could not resolve a tokenizer from the provided path. Expected a tokenizer dir, experiment dir, or SentencePiece .model file."
         )
 
-    def _load_resolved_text_tokenizer(self, resolved_path: Path, cache_dir: str):
-        if resolved_path.is_file() and resolved_path.suffix == ".model":
-            return MossTTSNanoSentencePieceTokenizer(vocab_file=str(resolved_path))
+    def _load_resolved_text_tokenizer(self, resolved_path: str, cache_dir: str):
+        local_path = self._existing_local_path(resolved_path)
+        load_source = str(local_path) if local_path is not None else str(resolved_path)
+        if local_path is not None and local_path.is_file() and local_path.suffix == ".model":
+            return MossTTSNanoSentencePieceTokenizer(vocab_file=str(local_path))
         try:
+            load_kwargs: dict[str, object] = {
+                "trust_remote_code": True,
+                "use_fast": bool(self.config.tokenizer_use_fast),
+                "cache_dir": cache_dir,
+            }
+            if local_path is not None:
+                load_kwargs["local_files_only"] = True
             return AutoTokenizer.from_pretrained(
-                str(resolved_path),
-                trust_remote_code=True,
-                use_fast=bool(self.config.tokenizer_use_fast),
-                local_files_only=True,
-                cache_dir=cache_dir,
+                load_source,
+                **load_kwargs,
             )
         except Exception:
-            model_path = resolved_path / "tokenizer.model"
-            if model_path.is_file():
-                return MossTTSNanoSentencePieceTokenizer(vocab_file=str(model_path))
+            if local_path is not None:
+                model_path = local_path / "tokenizer.model"
+                if model_path.is_file():
+                    return MossTTSNanoSentencePieceTokenizer(vocab_file=str(model_path))
             raise
 
     @staticmethod
@@ -1087,33 +1118,39 @@ class MossTTSNanoForCausalLM(MossTTSNanoPreTrainedModel):
         os.environ["HF_MODULES_CACHE"] = modules_cache_dir
         dynamic_module_utils.HF_MODULES_CACHE = modules_cache_dir
 
-    def _resolve_default_text_tokenizer_path(self) -> Path:
-        candidates: list[Path] = []
+    def _resolve_default_text_tokenizer_path(self) -> str:
+        candidates: list[Union[str, Path]] = []
 
         raw_name_or_path = getattr(self.config, "_name_or_path", None)
         if raw_name_or_path:
-            candidates.append(Path(str(raw_name_or_path)).expanduser())
+            candidates.append(str(raw_name_or_path).strip())
 
         raw_model_name_or_path = getattr(self, "name_or_path", None)
         if raw_model_name_or_path:
-            candidates.append(Path(str(raw_model_name_or_path)).expanduser())
+            candidates.append(str(raw_model_name_or_path).strip())
 
         candidates.append(Path(__file__).resolve().parent)
 
         checked: set[str] = set()
         for candidate in candidates:
-            resolved_candidate = candidate.resolve()
-            key = str(resolved_candidate)
-            if key in checked:
+            raw_candidate = str(candidate).strip()
+            if not raw_candidate:
+                continue
+            try:
+                resolved_candidate = self._resolve_text_tokenizer_path(raw_candidate)
+            except (FileNotFoundError, ValueError):
                 continue
-            checked.add(key)
+            if resolved_candidate in checked:
+                continue
+            checked.add(resolved_candidate)
+            return resolved_candidate
 
-            if (resolved_candidate / "tokenizer.model").is_file():
-                return resolved_candidate
-            if self._looks_like_hf_tokenizer_dir(resolved_candidate):
-                return resolved_candidate
+        for candidate in candidates:
+            raw_candidate = str(candidate).strip()
+            if raw_candidate:
+                return raw_candidate
 
-        return candidates[0].resolve()
+        return str(Path(__file__).resolve().parent)
 
     def _load_text_tokenizer(self, text_tokenizer=None, text_tokenizer_path: Optional[str] = None):
         if text_tokenizer is not None:
@@ -1124,7 +1161,7 @@ class MossTTSNanoForCausalLM(MossTTSNanoPreTrainedModel):
             if text_tokenizer_path is not None
             else self._resolve_default_text_tokenizer_path()
         )
-        normalized_path = str(resolved_path.resolve())
+        normalized_path = str(resolved_path)
         cached = getattr(self, "_cached_text_tokenizer", None)
         cached_path = getattr(self, "_cached_text_tokenizer_path", None)
         if cached is not None and cached_path == normalized_path: