Browse Source

tinygrad select model size

Alex Cheema 10 months ago
parent
commit
31641d1023
1 changed files with 1 additions and 1 deletions
  1. 1 1
      exo/inference/tinygrad/inference.py

+ 1 - 1
exo/inference/tinygrad/inference.py

@@ -92,6 +92,6 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
       return
       return
 
 
     model_path = await self.shard_downloader.ensure_shard(shard)
     model_path = await self.shard_downloader.ensure_shard(shard)
-    self.model = build_transformer(model_path, shard, model_size="8B")
+    self.model = build_transformer(model_path, shard, model_size="8B" if "8b" in shard.model_id.lower() else "70B")
     self.tokenizer = AutoTokenizer.from_pretrained(str((model_path if model_path.is_dir() else model_path.parent)))
     self.tokenizer = AutoTokenizer.from_pretrained(str((model_path if model_path.is_dir() else model_path.parent)))
     self.shard = shard
     self.shard = shard