try/except for decode, #75

decode_discovery.py

@@ -0,0 +1,42 @@
+import struct
+
+def decode_mikrotik_discovery(data):
+    result = {}
+    # Skip the first 4 bytes
+    index = 4
+    #print(f"Total data length: {len(data)}")
+    while index < len(data):
+        #print(f"Current index: {index}")
+        if index + 4 > len(data):
+            #print("Not enough data for header")
+            break
+        type_field, length = struct.unpack('!HH', data[index:index+4])
+        #print(f"Type: {type_field:02x}, Length: {length}")
+        index += 4
+        if index + length > len(data):
+            #print(f"Not enough data for value (need {length}, have {len(data) - index})")
+            break
+        value = data[index:index+length]
+        try:
+            if type_field in [0x05, 0x07, 0x08, 0x0b, 0x0c, 0x10]:
+                result[type_field] = value.decode('ascii', errors='ignore')
+            elif type_field in [0x00, 0x0a, 0x0e]:
+                result[type_field] = struct.unpack('!I', value.rjust(4, b'\x00'))[0]
+            elif type_field == 0x11:
+                result[type_field] = '.'.join(map(str, value))
+            else:
+                result[type_field] = value.hex()
+        except Exception as e:
+            print(f"Error processing field {type_field:02x}: {e}")
+            result[type_field] = value.hex()
+        index += length
+    return result
+
+# Your byte string
+data = b'\xbf\xa2\x04\x00\x00\x01\x00\x06,\xc8\x1b\x13\x98\xe0\x00\x05\x00\x08MikroTik\x00\x07\x00\x0f6.49.1 (stable)\x00\x08\x00\x08MikroTik\x00\n\x00\x04\xd7|\xf3\x00\x00\x0b\x00\tX4N0-HRA8\x00\x0c\x00\x08RB760iGS\x00\x0e\x00\x01\x00\x00\x10\x00\rbridge/ether5\x00\x11\x00\x04\xc0\xa8X\x01'
+
+decoded = decode_mikrotik_discovery(data)
+
+# Print in a more readable format
+for key, value in decoded.items():
+    print(f"Field {key:02x}: {value}")

exo/networking/grpc/grpc_discovery.py

@@ -112,8 +112,13 @@ class GRPCDiscovery(Discovery):
             if DEBUG_DISCOVERY >= 2: print(f"Received invalid JSON data from {addr}: {decoded_data[:100]}")
             return
 
-        decoder = json.JSONDecoder(strict=False)
-        message = decoder.decode(decoded_data)
+        try:
+            decoder = json.JSONDecoder(strict=False)
+            message = decoder.decode(decoded_data)
+        except json.JSONDecodeError as e:
+            if DEBUG_DISCOVERY >= 2: print(f"Error decoding JSON data from {addr}: {e}")
+            return
+
         if DEBUG_DISCOVERY >= 2: print(f"received from peer {addr}: {message}")
 
         if message['type'] == 'discovery' and message['node_id'] != self.node_id:

out.txt

@@ -0,0 +1,164 @@
+Processing /media/xeb/GreyArea/projects/exo
+  Preparing metadata (setup.py): started
+  Preparing metadata (setup.py): finished with status 'done'
+Collecting tinygrad@ git+https://github.com/tinygrad/tinygrad.git@a9f5a764dc640a5e5cbaaeeee21df7c8ca37da38 (from exo==0.0.1)
+  Using cached tinygrad-0.9.1-py3-none-any.whl
+Collecting aiohttp==3.9.5 (from exo==0.0.1)
+  Using cached aiohttp-3.9.5-cp313-cp313-linux_x86_64.whl
+Collecting aiohttp_cors==0.7.0 (from exo==0.0.1)
+  Using cached aiohttp_cors-0.7.0-py3-none-any.whl.metadata (20 kB)
+Collecting blobfile==2.1.1 (from exo==0.0.1)
+  Using cached blobfile-2.1.1-py3-none-any.whl.metadata (15 kB)
+Collecting grpcio==1.64.1 (from exo==0.0.1)
+  Using cached grpcio-1.64.1.tar.gz (12.2 MB)
+  Preparing metadata (setup.py): started
+  Preparing metadata (setup.py): finished with status 'done'
+Collecting grpcio-tools==1.64.1 (from exo==0.0.1)
+  Using cached grpcio_tools-1.64.1.tar.gz (5.0 MB)
+  Preparing metadata (setup.py): started
+  Preparing metadata (setup.py): finished with status 'done'
+Collecting huggingface-hub==0.23.4 (from exo==0.0.1)
+  Using cached huggingface_hub-0.23.4-py3-none-any.whl.metadata (12 kB)
+Collecting Jinja2==3.1.4 (from exo==0.0.1)
+  Using cached jinja2-3.1.4-py3-none-any.whl.metadata (2.6 kB)
+Collecting numpy==2.0.0 (from exo==0.0.1)
+  Using cached numpy-2.0.0-cp313-cp313-linux_x86_64.whl
+Collecting prometheus-client==0.20.0 (from exo==0.0.1)
+  Using cached prometheus_client-0.20.0-py3-none-any.whl.metadata (1.8 kB)
+Collecting protobuf==5.27.1 (from exo==0.0.1)
+  Using cached protobuf-5.27.1-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes)
+Collecting psutil==6.0.0 (from exo==0.0.1)
+  Using cached psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)
+Collecting pynvml==11.5.3 (from exo==0.0.1)
+  Using cached pynvml-11.5.3-py3-none-any.whl.metadata (8.8 kB)
+Collecting requests==2.32.3 (from exo==0.0.1)
+  Using cached requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)
+Collecting rich==13.7.1 (from exo==0.0.1)
+  Using cached rich-13.7.1-py3-none-any.whl.metadata (18 kB)
+Collecting safetensors==0.4.3 (from exo==0.0.1)
+  Using cached safetensors-0.4.3.tar.gz (69 kB)
+  Installing build dependencies: started
+  Installing build dependencies: finished with status 'done'
+  Getting requirements to build wheel: started
+  Getting requirements to build wheel: finished with status 'done'
+  Preparing metadata (pyproject.toml): started
+  Preparing metadata (pyproject.toml): finished with status 'done'
+Collecting tiktoken==0.7.0 (from exo==0.0.1)
+  Using cached tiktoken-0.7.0.tar.gz (33 kB)
+  Installing build dependencies: started
+  Installing build dependencies: finished with status 'done'
+  Getting requirements to build wheel: started
+  Getting requirements to build wheel: finished with status 'done'
+  Preparing metadata (pyproject.toml): started
+  Preparing metadata (pyproject.toml): finished with status 'done'
+Collecting tokenizers==0.19.1 (from exo==0.0.1)
+  Using cached tokenizers-0.19.1.tar.gz (321 kB)
+  Installing build dependencies: started
+  Installing build dependencies: finished with status 'done'
+  Getting requirements to build wheel: started
+  Getting requirements to build wheel: finished with status 'done'
+  Preparing metadata (pyproject.toml): started
+  Preparing metadata (pyproject.toml): finished with status 'done'
+Collecting tqdm==4.66.4 (from exo==0.0.1)
+  Using cached tqdm-4.66.4-py3-none-any.whl.metadata (57 kB)
+Collecting transformers==4.41.2 (from exo==0.0.1)
+  Using cached transformers-4.41.2-py3-none-any.whl.metadata (43 kB)
+Collecting uuid==1.30 (from exo==0.0.1)
+  Using cached uuid-1.30-py3-none-any.whl
+Collecting aiosignal>=1.1.2 (from aiohttp==3.9.5->exo==0.0.1)
+  Using cached aiosignal-1.3.1-py3-none-any.whl.metadata (4.0 kB)
+Collecting attrs>=17.3.0 (from aiohttp==3.9.5->exo==0.0.1)
+  Using cached attrs-23.2.0-py3-none-any.whl.metadata (9.5 kB)
+Collecting frozenlist>=1.1.1 (from aiohttp==3.9.5->exo==0.0.1)
+  Using cached frozenlist-1.4.1-py3-none-any.whl.metadata (12 kB)
+Collecting multidict<7.0,>=4.5 (from aiohttp==3.9.5->exo==0.0.1)
+  Using cached multidict-6.0.5-py3-none-any.whl.metadata (4.2 kB)
+Collecting yarl<2.0,>=1.0 (from aiohttp==3.9.5->exo==0.0.1)
+  Using cached yarl-1.9.4-py3-none-any.whl.metadata (31 kB)
+Collecting pycryptodomex~=3.8 (from blobfile==2.1.1->exo==0.0.1)
+  Using cached pycryptodomex-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.4 kB)
+Collecting urllib3<3,>=1.25.3 (from blobfile==2.1.1->exo==0.0.1)
+  Using cached urllib3-2.2.2-py3-none-any.whl.metadata (6.4 kB)
+Collecting lxml~=4.9 (from blobfile==2.1.1->exo==0.0.1)
+  Using cached lxml-4.9.4.tar.gz (3.6 MB)
+  Preparing metadata (setup.py): started
+  Preparing metadata (setup.py): finished with status 'done'
+Collecting filelock~=3.0 (from blobfile==2.1.1->exo==0.0.1)
+  Using cached filelock-3.15.4-py3-none-any.whl.metadata (2.9 kB)
+Requirement already satisfied: setuptools in ./venv/lib/python3.13/site-packages (from grpcio-tools==1.64.1->exo==0.0.1) (71.1.0)
+Collecting fsspec>=2023.5.0 (from huggingface-hub==0.23.4->exo==0.0.1)
+  Using cached fsspec-2024.6.1-py3-none-any.whl.metadata (11 kB)
+Collecting packaging>=20.9 (from huggingface-hub==0.23.4->exo==0.0.1)
+  Using cached packaging-24.1-py3-none-any.whl.metadata (3.2 kB)
+Collecting pyyaml>=5.1 (from huggingface-hub==0.23.4->exo==0.0.1)
+  Using cached PyYAML-6.0.1-cp313-cp313-linux_x86_64.whl
+Collecting typing-extensions>=3.7.4.3 (from huggingface-hub==0.23.4->exo==0.0.1)
+  Using cached typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB)
+Collecting MarkupSafe>=2.0 (from Jinja2==3.1.4->exo==0.0.1)
+  Using cached MarkupSafe-2.1.5-cp313-cp313-linux_x86_64.whl
+Collecting charset-normalizer<4,>=2 (from requests==2.32.3->exo==0.0.1)
+  Using cached charset_normalizer-3.3.2-py3-none-any.whl.metadata (33 kB)
+Collecting idna<4,>=2.5 (from requests==2.32.3->exo==0.0.1)
+  Using cached idna-3.7-py3-none-any.whl.metadata (9.9 kB)
+Collecting certifi>=2017.4.17 (from requests==2.32.3->exo==0.0.1)
+  Using cached certifi-2024.7.4-py3-none-any.whl.metadata (2.2 kB)
+Collecting markdown-it-py>=2.2.0 (from rich==13.7.1->exo==0.0.1)
+  Using cached markdown_it_py-3.0.0-py3-none-any.whl.metadata (6.9 kB)
+Collecting pygments<3.0.0,>=2.13.0 (from rich==13.7.1->exo==0.0.1)
+  Using cached pygments-2.18.0-py3-none-any.whl.metadata (2.5 kB)
+Collecting regex>=2022.1.18 (from tiktoken==0.7.0->exo==0.0.1)
+  Using cached regex-2024.5.15-cp313-cp313-linux_x86_64.whl
+Collecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich==13.7.1->exo==0.0.1)
+  Using cached mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)
+Using cached aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)
+Using cached blobfile-2.1.1-py3-none-any.whl (73 kB)
+Using cached huggingface_hub-0.23.4-py3-none-any.whl (402 kB)
+Using cached jinja2-3.1.4-py3-none-any.whl (133 kB)
+Using cached prometheus_client-0.20.0-py3-none-any.whl (54 kB)
+Using cached protobuf-5.27.1-cp38-abi3-manylinux2014_x86_64.whl (309 kB)
+Using cached psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (290 kB)
+Using cached pynvml-11.5.3-py3-none-any.whl (53 kB)
+Using cached requests-2.32.3-py3-none-any.whl (64 kB)
+Using cached rich-13.7.1-py3-none-any.whl (240 kB)
+Using cached tqdm-4.66.4-py3-none-any.whl (78 kB)
+Using cached transformers-4.41.2-py3-none-any.whl (9.1 MB)
+Using cached aiosignal-1.3.1-py3-none-any.whl (7.6 kB)
+Using cached attrs-23.2.0-py3-none-any.whl (60 kB)
+Using cached certifi-2024.7.4-py3-none-any.whl (162 kB)
+Using cached charset_normalizer-3.3.2-py3-none-any.whl (48 kB)
+Using cached filelock-3.15.4-py3-none-any.whl (16 kB)
+Using cached frozenlist-1.4.1-py3-none-any.whl (11 kB)
+Using cached fsspec-2024.6.1-py3-none-any.whl (177 kB)
+Using cached idna-3.7-py3-none-any.whl (66 kB)
+Using cached markdown_it_py-3.0.0-py3-none-any.whl (87 kB)
+Using cached multidict-6.0.5-py3-none-any.whl (9.7 kB)
+Using cached packaging-24.1-py3-none-any.whl (53 kB)
+Using cached pycryptodomex-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)
+Using cached pygments-2.18.0-py3-none-any.whl (1.2 MB)
+Using cached typing_extensions-4.12.2-py3-none-any.whl (37 kB)
+Using cached urllib3-2.2.2-py3-none-any.whl (121 kB)
+Using cached yarl-1.9.4-py3-none-any.whl (31 kB)
+Using cached mdurl-0.1.2-py3-none-any.whl (10.0 kB)
+Building wheels for collected packages: exo, grpcio, grpcio-tools, safetensors, tiktoken, tokenizers, lxml
+  Building wheel for exo (setup.py): started
+  Building wheel for exo (setup.py): finished with status 'done'
+  Created wheel for exo: filename=exo-0.0.1-py3-none-any.whl size=59254 sha256=90f791e9120765638995e8c2601ba07f86338d492b6a07a423a118774de375a2
+  Stored in directory: /tmp/pip-ephem-wheel-cache-sk105743/wheels/dc/26/a7/9ffda4701ecc427d85b816380d68106c7a252648fa3f64b5cc
+  Building wheel for grpcio (setup.py): started
+  Building wheel for grpcio (setup.py): still running...
+  Building wheel for grpcio (setup.py): finished with status 'error'
+  Running setup.py clean for grpcio
+  Building wheel for grpcio-tools (setup.py): started
+  Building wheel for grpcio-tools (setup.py): finished with status 'error'
+  Running setup.py clean for grpcio-tools
+  Building wheel for safetensors (pyproject.toml): started
+  Building wheel for safetensors (pyproject.toml): finished with status 'error'
+  Building wheel for tiktoken (pyproject.toml): started
+  Building wheel for tiktoken (pyproject.toml): finished with status 'error'
+  Building wheel for tokenizers (pyproject.toml): started
+  Building wheel for tokenizers (pyproject.toml): finished with status 'error'
+  Building wheel for lxml (setup.py): started
+  Building wheel for lxml (setup.py): finished with status 'error'
+  Running setup.py clean for lxml
+Successfully built exo
+Failed to build grpcio grpcio-tools safetensors tiktoken tokenizers lxml

run.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+python main.py --chatgpt-api-port=8007 --disable-tui