add support for image upload to tinychat for vision models

tinychat/examples/tinychat/index.css

@@ -309,4 +309,74 @@ p {
   outline: none;
   border-color: #007bff;
   box-shadow: 0 0 0 2px rgba(0,123,255,.25);
+}
+
+/* Image upload button styles */
+.image-input-button {
+  background-color: var(--secondary-color);
+  color: var(--foreground-color);
+  border: none;
+  border-radius: 50%;
+  width: 40px;
+  height: 40px;
+  font-size: 18px;
+  cursor: pointer;
+  transition: all 0.3s ease;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  margin-right: 10px;
+}
+
+.image-input-button:hover {
+  background-color: var(--secondary-color-transparent);
+  transform: scale(1.1);
+}
+
+.image-input-button:focus {
+  outline: none;
+  box-shadow: 0 0 0 3px rgba(var(--secondary-color-rgb), 0.5);
+}
+
+.image-input-button i {
+  transition: all 0.3s ease;
+}
+
+.image-input-button:hover i {
+  transform: scale(1.2);
+}
+
+/* Hidden file input styles */
+#image-upload {
+  display: none;
+}
+
+.image-preview-container {
+  position: relative;
+  display: inline-block;
+  margin-right: 10px;
+}
+
+.image-preview {
+  max-width: 100px;
+  max-height: 100px;
+  object-fit: cover;
+  border-radius: 5px;
+}
+
+.remove-image-button {
+  position: absolute;
+  top: -5px;
+  right: -5px;
+  background-color: rgba(255, 255, 255, 0.8);
+  border: none;
+  border-radius: 50%;
+  padding: 2px 5px;
+  cursor: pointer;
+}
+
+.message > p > img {
+  max-width: 100%;
+  max-height: 100%;
+  object-fit: contain;
 }

tinychat/examples/tinychat/index.html

@@ -44,6 +44,7 @@
         <option value="mistral-nemo">Mistral Nemo</option>
         <option value="mistral-large">Mistral Large</option>
         <option value="deepseek-coder-v2-lite">Deepseek Coder V2 Lite</option>
+        <option value="llava-1.5-7b-hf">LLaVa 1.5 7B (Vision Model)</option>
       </select>
     </div>
     <div class="home centered" x-show="home === 0" x-transition x-effect="
@@ -159,6 +160,16 @@
         </span>
       </div>
       <div class="input">
+        <button x-show="cstate.selectedModel === 'llava-1.5-7b-hf'" class="image-input-button" @click="$refs.imageUpload.click()">
+          <i class="fas fa-image"></i>
+        </button>
+        <input x-ref="imageUpload" type="file" id="image-upload" accept="image/*" @change="$data.handleImageUpload($event)" style="display: none;">
+        <div x-show="imagePreview" class="image-preview-container">
+          <img :src="imagePreview" alt="Uploaded Image" class="image-preview">
+          <button @click="imagePreview = null; imageUrl = null;" class="remove-image-button">
+            <i class="fas fa-times"></i>
+          </button>
+        </div>
         <textarea x-ref="inputForm" id="input-form" class="input-form" autofocus rows=1 x-autosize
           :placeholder="generating ? 'Generating...' : 'Say something'" :disabled="generating" @input="
             home = (home === 0) ? 1 : home
@@ -187,4 +198,4 @@
   </main>
 </body>
 
-</html>
+</html>

tinychat/examples/tinychat/index.js

@@ -4,6 +4,7 @@ document.addEventListener("alpine:init", () => {
     cstate: {
       time: null,
       messages: [],
+      selectedModel: 'llama-3.1-8b',
     },
 
     // historical state
@@ -18,6 +19,9 @@ document.addEventListener("alpine:init", () => {
     tokens_per_second: 0,
     total_tokens: 0,
 
+    // image handling
+    imagePreview: null,
+
     removeHistory(cstate) {
       const index = this.histories.findIndex((state) => {
         return state.time === cstate.time;
@@ -28,10 +32,28 @@ document.addEventListener("alpine:init", () => {
       }
     },
 
+    async handleImageUpload(event) {
+      const file = event.target.files[0];
+      if (file) {
+        const reader = new FileReader();
+        reader.onload = (e) => {
+          this.imagePreview = e.target.result;
+          this.imageUrl = e.target.result; // Store the image URL
+          // Add image preview to the chat
+          this.cstate.messages.push({
+            role: "user",
+            content: `![Uploaded Image](${this.imagePreview})`,
+          });
+        };
+        reader.readAsDataURL(file);
+      }
+    },
+
+
     async handleSend() {
       const el = document.getElementById("input-form");
       const value = el.value.trim();
-      if (!value) return;
+      if (!value && !this.imagePreview) return;
 
       if (this.generating) return;
       this.generating = true;
@@ -41,7 +63,9 @@ document.addEventListener("alpine:init", () => {
       window.history.pushState({}, "", "/");
 
       // add message to list
-      this.cstate.messages.push({ role: "user", content: value });
+      if (value) {
+        this.cstate.messages.push({ role: "user", content: value });
+      }
 
       // clear textarea
       el.value = "";
@@ -54,10 +78,51 @@ document.addEventListener("alpine:init", () => {
       let tokens = 0;
       this.tokens_per_second = 0;
 
+      // prepare messages for API request
+      const apiMessages = this.cstate.messages.map(msg => {
+        if (msg.content.startsWith('![Uploaded Image]')) {
+          return {
+            role: "user",
+            content: [
+              {
+                type: "image_url",
+                image_url: {
+                  url: this.imageUrl
+                }
+              }
+            ]
+          };
+        } else {
+          return {
+            role: msg.role,
+            content: [
+              {
+                type: "text",
+                text: msg.content
+              }
+            ]
+          };
+        }
+      });
+
+      // If there's an image URL, add it to all messages
+      if (this.imageUrl) {
+        apiMessages.forEach(msg => {
+          if (!msg.content.some(content => content.type === "image_url")) {
+            msg.content.push({
+              type: "image_url",
+              image_url: {
+                url: this.imageUrl
+              }
+            });
+          }
+        });
+      }
+
       // start receiving server sent events
       let gottenFirstChunk = false;
       for await (
-        const chunk of this.openaiChatCompletion(this.cstate.selectedModel, this.cstate.messages)
+        const chunk of this.openaiChatCompletion(this.cstate.selectedModel, apiMessages)
       ) {
         if (!gottenFirstChunk) {
           this.cstate.messages.push({ role: "assistant", content: "" });