|
@@ -156,8 +156,11 @@ async def shutdown(signal, loop):
|
|
async def select_best_inference_engine(node: StandardNode):
|
|
async def select_best_inference_engine(node: StandardNode):
|
|
supported_engines = node.get_supported_inference_engines()
|
|
supported_engines = node.get_supported_inference_engines()
|
|
await node.broadcast_supported_engines(supported_engines)
|
|
await node.broadcast_supported_engines(supported_engines)
|
|
-
|
|
|
|
|
|
+ logger.error('ABOVE and ALL')
|
|
|
|
+ logger.error("Topology inference engines pool: %s", node.get_topology_inference_engines())
|
|
|
|
+ logger.error(f'result:{node.get_topology_inference_engines()}')
|
|
if node.get_topology_inference_engines():
|
|
if node.get_topology_inference_engines():
|
|
|
|
+ logger.info("Topology inference engines pool: %s", node.get_topology_inference_engines())
|
|
topology_inference_engines_pool = node.get_topology_inference_engines()
|
|
topology_inference_engines_pool = node.get_topology_inference_engines()
|
|
if any("tinygrad" in engines and len(engines) == 1 for engines in topology_inference_engines_pool):
|
|
if any("tinygrad" in engines and len(engines) == 1 for engines in topology_inference_engines_pool):
|
|
return "tinygrad"
|
|
return "tinygrad"
|
|
@@ -207,14 +210,13 @@ async def main():
|
|
loop.add_signal_handler(s, handle_exit)
|
|
loop.add_signal_handler(s, handle_exit)
|
|
|
|
|
|
await node.start(wait_for_peers=args.wait_for_peers)
|
|
await node.start(wait_for_peers=args.wait_for_peers)
|
|
- await select_best_inference_engine(node)
|
|
|
|
-
|
|
|
|
|
|
|
|
if args.command == "run" or args.run_model:
|
|
if args.command == "run" or args.run_model:
|
|
model_name = args.model_name or args.run_model
|
|
model_name = args.model_name or args.run_model
|
|
if not model_name:
|
|
if not model_name:
|
|
print("Error: Model name is required when using 'run' command or --run-model")
|
|
print("Error: Model name is required when using 'run' command or --run-model")
|
|
return
|
|
return
|
|
|
|
+ await select_best_inference_engine(node)
|
|
await run_model_cli(node, inference_engine, model_name, args.prompt)
|
|
await run_model_cli(node, inference_engine, model_name, args.prompt)
|
|
else:
|
|
else:
|
|
asyncio.create_task(api.run(port=args.chatgpt_api_port)) # Start the API server as a non-blocking task
|
|
asyncio.create_task(api.run(port=args.chatgpt_api_port)) # Start the API server as a non-blocking task
|