feat: support 3k+ collections (#847)

* virtual tree

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* make tree height dynmaic

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* finsih batchRefreshCollections

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* disable interative if the node is loading

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* adjust style if schema is not loaded

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix schema page error

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix homepage

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

---------

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

client/package.json

@@ -25,6 +25,7 @@
     "@mui/material": "^5.17.1",
     "@mui/styles": "^5.17.1",
     "@mui/x-tree-view": "^7.28.0",
+    "@tanstack/react-virtual": "^3.13.6",
     "axios": "^1.8.2",
     "codemirror": "^6.0.1",
     "d3": "^7.9.0",

client/src/components/grid/Grid.tsx

@@ -129,6 +129,7 @@ const AttuGrid: FC<AttuGridType> = props => {
     orderBy,
     showPagination = true,
     hideOnDisable,
+    rowDecorator = () => ({}),
   } = props;
 
   const _isSelected = (row: { [x: string]: any }) => {
@@ -261,6 +262,7 @@ const AttuGrid: FC<AttuGridType> = props => {
           handleSort={handleSort}
           order={order}
           orderBy={orderBy}
+          rowDecorator={rowDecorator}
         ></Table>
         {rowCount && showPagination ? (
           <TablePagination

client/src/components/grid/Table.tsx

@@ -119,6 +119,7 @@ const EnhancedTable: FC<TableType> = props => {
     handleSort,
     order,
     orderBy,
+    rowDecorator = () => ({}),
   } = props;
   const classes = useStyles({ tableCellMaxWidth });
   const { t: commonTrans } = useTranslation();
@@ -171,6 +172,7 @@ const EnhancedTable: FC<TableType> = props => {
                             ? classes.selected
                             : undefined,
                       }}
+                      sx={rowDecorator(row)}
                     >
                       {openCheckBox && (
                         <TableCell

client/src/components/grid/Types.ts

@@ -2,6 +2,7 @@ import React, { ReactElement, Ref } from 'react';
 import { LabelDisplayedRowsArgs } from '@mui/material';
 import { IconsType } from '../icons/Types';
 import { SearchType } from '../customInput/Types';
+import type { SxProps, Theme } from '@mui/material';
 
 export type IconConfigType = {
   [x: string]: JSX.Element;
@@ -85,6 +86,7 @@ export type TableType = {
   primaryKey: string;
   openCheckBox?: boolean;
   disableSelect?: boolean;
+  rowDecorator?: (row: any) => SxProps<Theme> | React.CSSProperties;
   noData?: string;
   showHoverStyle?: boolean;
   isLoading?: boolean;
@@ -154,6 +156,7 @@ export type AttuGridType = ToolBarType & {
   rowHeight?: number;
   hideOnDisable?: boolean;
   pagerHeight?: number;
+  rowDecorator?: (row: any) => SxProps<Theme> | React.CSSProperties;
 };
 
 export type ActionBarType = {

client/src/context/Data.tsx

@@ -48,6 +48,7 @@ export const dataContext = createContext<DataContextType>({
   fetchCollection: async () => {
     return {} as CollectionFullObject;
   },
+  batchRefreshCollections: async () => {},
   createCollection: async () => {
     return {} as CollectionFullObject;
   },
@@ -145,7 +146,7 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
   // Websocket Callback: update single collection
   const updateCollections = useCallback(
     (props: { collections: CollectionFullObject[]; database?: string }) => {
-      const { collections, database: remote } = props;
+      const { collections = [], database: remote } = props;
       if (
         remote !== database &&
         database !== undefined &&
@@ -220,7 +221,7 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
       // set collections
       setCollections([]);
       // fetch collections
-      const res = await CollectionService.getCollections();
+      const res = await CollectionService.getAllCollections();
       // Only process the response if this is the latest request
       if (currentRequestId === requestIdRef.current) {
         // check state
@@ -249,6 +250,71 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
     return res;
   };
 
+  const _fetchCollections = async (collectionNames: string[]) => {
+    const res = await CollectionService.getCollections({
+      db_name: database,
+      collections: collectionNames,
+    });
+    // update collections
+    updateCollections({ collections: res });
+  };
+
+  // Batch refresh collections with debounce
+  const refreshCollectionsDebounceMapRef = useRef<
+    Map<
+      string,
+      { timer: NodeJS.Timeout | null; names: string[]; pending: Set<string> }
+    >
+  >(new Map());
+
+  const batchRefreshCollections = useCallback(
+    (collectionNames: string[], key: string = 'default') => {
+      let ref = refreshCollectionsDebounceMapRef.current.get(key);
+      if (!ref) {
+        ref = { timer: null, names: [], pending: new Set() };
+        refreshCollectionsDebounceMapRef.current.set(key, ref);
+      }
+
+      const filteredCollectionNames = collectionNames.filter(name => {
+        const collection = collections.find(v => v.collection_name === name);
+        return collection && !collection.schema && !ref!.pending.has(name);
+      });
+
+      ref.names = filteredCollectionNames;
+
+      if (ref.timer) {
+        clearTimeout(ref.timer);
+      }
+
+      ref.timer = setTimeout(async () => {
+        if (ref!.names.length === 0) return;
+        try {
+          const batchSize = 2;
+          for (let i = 0; i < ref!.names.length; i += batchSize) {
+            let batch = ref!.names.slice(i, i + batchSize);
+
+            // recheck if the collection is still pending
+            batch = batch.filter(name => {
+              const collection = collections.find(
+                v => v.collection_name === name
+              );
+              return collection && !collection.schema;
+            });
+
+            batch.forEach(name => ref!.pending.add(name));
+            await _fetchCollections(batch);
+            batch.forEach(name => ref!.pending.delete(name));
+          }
+        } catch (error) {
+          console.error('Failed to refresh collections:', error);
+        }
+        ref!.names = [];
+        ref!.timer = null;
+      }, 200);
+    },
+    [collections, _fetchCollections]
+  );
+
   // API: create collection
   const createCollection = async (data: any) => {
     // create collection
@@ -507,6 +573,7 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
         fetchDatabases,
         fetchCollections,
         fetchCollection,
+        batchRefreshCollections,
         createCollection,
         loadCollection,
         releaseCollection,

client/src/context/Types.ts

@@ -10,7 +10,7 @@ import type {
   DatabaseObject,
   AuthReq,
   AuthObject,
-  ResStatus
+  ResStatus,
 } from '@server/types';
 
 export type RootContextType = {
@@ -111,6 +111,7 @@ export type DataContextType = {
   setDatabase: Dispatch<SetStateAction<string>>;
   databases: DatabaseObject[];
   setDatabaseList: Dispatch<SetStateAction<DatabaseObject[]>>;
+  batchRefreshCollections: (collectionNames: string[], key?: string) => void;
 
   // APIs
   // databases

client/src/http/BaseModel.ts

@@ -32,12 +32,6 @@ export default class BaseModel {
     return response.data?.data as T;
   }
 
-  /**
-   * Search with flexible parameters
-   */
-  static async search<T>(options: RequestParams): Promise<T> {
-    return this.find<T>(options);
-  }
 
   /**
    * Create a new resource

client/src/http/Collection.service.ts

@@ -18,25 +18,32 @@ import type {
 } from '@/pages/databases/collections/schema/Types';
 
 export class CollectionService extends BaseModel {
-  static getCollections(data?: {
+  static getAllCollections(data?: {
     type: ShowCollectionsType;
   }): Promise<CollectionObject[]> {
     return super.find({ path: `/collections`, params: data || {} });
   }
 
+  static getCollections(data?: {
+    db_name?: string;
+    collections: string[];
+  }): Promise<CollectionFullObject[]> {
+    return super.query({ path: `/collections/details`, data });
+  }
+
   static getCollectionsNames(data?: { db_name: string }): Promise<string[]> {
     return super.find({ path: `/collections/names`, params: data || {} });
   }
 
   static describeCollectionUnformatted(collectionName: string) {
-    return super.search({
+    return super.find({
       path: `/collections/${collectionName}/unformatted`,
       params: {},
     });
   }
 
   static getCollection(collectionName: string) {
-    return super.search<CollectionFullObject>({
+    return super.find<CollectionFullObject>({
       path: `/collections/${collectionName}`,
       params: {},
     });
@@ -93,14 +100,14 @@ export class CollectionService extends BaseModel {
   }
 
   static getStatistics() {
-    return super.search<StatisticsObject>({
+    return super.find<StatisticsObject>({
       path: `/collections/statistics`,
       params: {},
     });
   }
 
   static count(collectionName: string) {
-    return super.search<CountObject>({
+    return super.find<CountObject>({
       path: `/collections/${collectionName}/count`,
       params: {},
     });

client/src/http/Database.service.ts

@@ -17,7 +17,7 @@ export interface AlterDatabaseRequest {
 
 export class DatabaseService extends BaseModel {
   static listDatabases() {
-    return super.search<DatabaseObject[]>({
+    return super.find<DatabaseObject[]>({
       path: `/databases`,
       params: {},
     });
@@ -32,7 +32,7 @@ export class DatabaseService extends BaseModel {
   }
 
   static describeDatabase(db_name: string) {
-    return super.search<DatabaseObject>({
+    return super.find<DatabaseObject>({
       path: `/databases/${db_name}`,
       params: {},
     });

client/src/http/Milvus.service.ts

@@ -15,18 +15,18 @@ export class MilvusService extends BaseModel {
   }
 
   static getVersion() {
-    return super.search({ path: `/milvus/version`, params: {} });
+    return super.find({ path: `/milvus/version`, params: {} });
   }
 
   static check(address: string) {
-    return super.search({
+    return super.find({
       path: `/milvus/check`,
       params: { address },
     }) as Promise<{ connected: boolean }>;
   }
 
   static getMetrics() {
-    return super.search({
+    return super.find({
       path: `/milvus/metrics`,
       params: {},
     });

client/src/http/Segment.service.ts

@@ -6,14 +6,14 @@ import type {
 
 export class SegmentService extends BaseModel {
   static getQSegments(collectionName: string) {
-    return super.search<QuerySegmentObjects>({
+    return super.find<QuerySegmentObjects>({
       path: `/collections/${collectionName}/qsegments`,
       params: {},
     });
   }
 
   static getPSegments(collectionName: string) {
-    return super.search<PersistentSegmentObjects>({
+    return super.find<PersistentSegmentObjects>({
       path: `/collections/${collectionName}/psegments`,
       params: {},
     });

client/src/http/User.service.ts

@@ -20,12 +20,12 @@ import type {
 export class UserService extends BaseModel {
   // get user data
   static getUsers() {
-    return super.search<UserWithRoles[]>({ path: `/users`, params: {} });
+    return super.find<UserWithRoles[]>({ path: `/users`, params: {} });
   }
 
   // get all roles
   static getRoles() {
-    return super.search<RolesWithPrivileges[]>({
+    return super.find<RolesWithPrivileges[]>({
       path: `/users/roles`,
       params: {},
     });
@@ -82,7 +82,7 @@ export class UserService extends BaseModel {
 
   // get RBAC info
   static getAllPrivilegeGroups() {
-    return super.search({
+    return super.find({
       path: `/users/privilegeGroups`,
       params: {},
     }) as Promise<PrivilegeGroup[]>;
@@ -90,14 +90,14 @@ export class UserService extends BaseModel {
 
   // get RBAC info
   static getRBAC() {
-    return super.search({
+    return super.find({
       path: `/users/rbac`,
       params: {},
     }) as Promise<RBACOptions>;
   }
   // get privilege groups
   static getPrivilegeGroups() {
-    return super.search<PrivilegeGroupsRes>({
+    return super.find<PrivilegeGroupsRes>({
       path: `/users/privilege-groups`,
       params: {},
     });

client/src/pages/databases/collections/Collections.tsx

@@ -1,4 +1,4 @@
-import { useContext, useMemo, useState } from 'react';
+import { useContext, useMemo, useState, useEffect } from 'react';
 import { Link, useSearchParams } from 'react-router-dom';
 import { Theme } from '@mui/material';
 import { useTranslation } from 'react-i18next';
@@ -73,8 +73,14 @@ const useStyles = makeStyles((theme: Theme) => ({
 
 const Collections = () => {
   const { isManaged } = useContext(authContext);
-  const { collections, database, loading, fetchCollections, fetchCollection } =
-    useContext(dataContext);
+  const {
+    collections,
+    database,
+    loading,
+    fetchCollections,
+    fetchCollection,
+    batchRefreshCollections,
+  } = useContext(dataContext);
 
   const [searchParams] = useSearchParams();
   const [search, setSearch] = useState<string>(
@@ -494,6 +500,17 @@ const Collections = () => {
 
   const CollectionIcon = icons.navCollection;
 
+  // lazy fetch collections that don't have schema
+  useEffect(() => {
+    const names = collectionList
+      .filter(c => !c.schema)
+      .map(c => c.collection_name);
+
+    if (names.length > 0) {
+      batchRefreshCollections(names, 'collection-grid');
+    }
+  }, [collectionList, batchRefreshCollections]);
+
   return (
     <section className={classes.root}>
       {collections.length > 0 || loading ? (
@@ -519,6 +536,16 @@ const Collections = () => {
           labelDisplayedRows={getLabelDisplayedRows(
             commonTrans('grid.collections')
           )}
+          rowDecorator={(row: CollectionObject) => {
+            if (!row.schema) {
+              return {
+                pointerEvents: 'none',
+                opacity: 0.5,
+                backgroundColor: 'rgba(0,0,0,0.04)',
+              };
+            }
+            return {};
+          }}
         />
       ) : (
         <>

client/src/pages/databases/collections/schema/Schema.tsx

@@ -48,10 +48,10 @@ const Overview = () => {
     c => c.collection_name === collectionName
   );
 
-  // check if collection is mmap enabled
-  const isCollectionMmapEnabled = collection?.properties!.some((p: any) => {
-    return p.key === 'mmap.enabled' && p.value === 'true';
-  });
+  // fetch collection if not loaded
+  if (collection && !collection.schema) {
+    fetchCollection(collectionName);
+  }
 
   // get fields
   const fields = collection?.schema?.fields || [];
@@ -308,6 +308,11 @@ const Overview = () => {
     f => f.autoID === true
   );
 
+  // check if collection is mmap enabled
+  const isCollectionMmapEnabled = collection?.properties?.some((p: any) => {
+    return p.key === 'mmap.enabled' && p.value === 'true';
+  });
+
   // get loading state label
   return (
     <section className={classes.wrapper}>

client/src/pages/databases/tree/index.tsx

@@ -1,52 +1,60 @@
-import { useState, useEffect } from 'react';
+import React, {
+  useState,
+  useEffect,
+  useRef,
+  useCallback,
+  useMemo,
+  useContext,
+} from 'react';
 import { useTranslation } from 'react-i18next';
-import { SimpleTreeView, TreeItem } from '@mui/x-tree-view';
 import icons from '@/components/icons/Icons';
-import { Tooltip, Typography, Grow, Popover } from '@mui/material';
+import {
+  Tooltip,
+  Typography,
+  Grow,
+  Popover,
+  Box,
+  IconButton,
+} from '@mui/material'; // Added Box, IconButton
 import { useNavigate } from 'react-router-dom';
 import { CollectionObject } from '@server/types';
 import clcx from 'clsx';
 import { formatNumber } from '@/utils';
 import { useStyles } from './style';
 import {
-  DatabaseTreeItem,
+  DatabaseTreeItem as OriginalDatabaseTreeItem, // Rename original type
   TreeNodeType,
-  DatabaseToolProps,
+  DatabaseTreeProps,
   ContextMenu,
   TreeNodeObject,
 } from './types';
 import { TreeContextMenu } from './TreeContextMenu';
+import { useVirtualizer } from '@tanstack/react-virtual'; // Import virtualizer
+import { dataContext } from '@/context';
 
-// get expanded nodes from data
-const getExpanded = (nodes: DatabaseTreeItem[]) => {
-  const expanded: string[] = [];
-  nodes.forEach(node => {
-    if (node.expanded) {
-      expanded.push(node.id);
-    }
-    if (node.children && node.children.length > 0) {
-      expanded.push(...getExpanded(node.children));
-    }
-  });
-  return expanded;
-};
+// Define a type for the flattened list item
+interface FlatTreeItem {
+  id: string;
+  name: string;
+  depth: number;
+  type: TreeNodeType;
+  data: TreeNodeObject | null; // Allow null for DB node
+  isExpanded: boolean; // Track if the node itself is expanded
+  hasChildren: boolean; // Does it have children?
+  originalNode: OriginalDatabaseTreeItem; // Keep original node ref if needed
+}
 
+// ... existing CollectionNode component (can be reused or integrated) ...
 const CollectionNode: React.FC<{ data: CollectionObject }> = ({ data }) => {
   // i18n collectionTrans
   const { t: commonTrans } = useTranslation();
-
-  // styles
   const classes = useStyles();
-
-  // class
   const loadClass = clcx(classes.dot, {
     [classes.loaded]: data.loaded,
     [classes.unloaded]: !data.loaded,
     [classes.loading]: data.status === 'loading',
     [classes.noIndex]: !data.schema || !data.schema.hasVectorIndex,
   });
-
-  //  status tooltip
   const hasIndex = data.schema && data.schema.hasVectorIndex;
   const loadStatus = hasIndex
     ? data.loaded
@@ -57,20 +65,7 @@ const CollectionNode: React.FC<{ data: CollectionObject }> = ({ data }) => {
   return (
     <div className={classes.collectionNode}>
       <div className={classes.collectionName}>
-        <Tooltip
-          title={data.collection_name}
-          placement="top"
-          PopperProps={{
-            modifiers: [
-              {
-                name: 'offset',
-                options: {
-                  offset: [0, -10],
-                },
-              },
-            ],
-          }}
-        >
+        <Tooltip title={data.collection_name} placement="top">
           <Typography noWrap className="collectionName">
             {data.collection_name}
           </Typography>
@@ -86,41 +81,118 @@ const CollectionNode: React.FC<{ data: CollectionObject }> = ({ data }) => {
   );
 };
 
-const DatabaseTree: React.FC<DatabaseToolProps> = props => {
-  // state
+const DatabaseTree: React.FC<DatabaseTreeProps> = props => {
+  const {
+    database,
+    collections,
+    params,
+    treeHeight = 'calc(100vh - 64px)',
+  } = props;
+  const classes = useStyles();
+  const navigate = useNavigate();
+
+  // context
+  const { batchRefreshCollections } = useContext(dataContext);
+
+  // State
   const [contextMenu, setContextMenu] = useState<ContextMenu | null>(null);
-  // props
-  const { database, collections, params } = props;
+  const [expandedItems, setExpandedItems] = useState<Set<string>>(
+    new Set([database])
+  ); // Start with DB expanded
+  const [selectedItemId, setSelectedItemId] = useState<string | null>(
+    params.collectionName ? `c_${params.collectionName}` : database
+  );
+
+  // Ref for the scrollable element
+  const parentRef = useRef<HTMLDivElement>(null);
+
+  // Icons
+  const DatabaseIcon = icons.database;
+  const CollectionIcon = icons.navCollection;
+  const ExpandIcon = icons.rightArrow;
+
+  // --- Tree Flattening Logic ---
+  const flattenTree = useCallback(
+    (
+      node: OriginalDatabaseTreeItem,
+      depth: number,
+      expanded: Set<string>
+    ): FlatTreeItem[] => {
+      const isExpanded = expanded.has(node.id);
+      const hasChildren = node.children && node.children.length > 0;
 
-  // format tree data
-  const children = collections.map(c => {
-    return {
+      const flatNode: FlatTreeItem = {
+        id: node.id,
+        name: node.name,
+        depth: depth,
+        type: node.type,
+        data: node.data || null,
+        isExpanded: isExpanded,
+        hasChildren: Boolean(hasChildren),
+        originalNode: node,
+      };
+
+      let childrenFlat: FlatTreeItem[] = [];
+      if (hasChildren && isExpanded) {
+        childrenFlat = node
+          .children!.map(child => flattenTree(child, depth + 1, expanded))
+          .reduce((acc, val) => acc.concat(val), []);
+      }
+
+      return [flatNode, ...childrenFlat];
+    },
+    []
+  );
+
+  // --- Memoized Flattened Data ---
+  const flattenedNodes = useMemo(() => {
+    // Rebuild the tree structure in the format needed for flattenTree
+    const children = collections.map(c => ({
       id: `c_${c.collection_name}`,
       name: c.collection_name,
       type: 'collection' as TreeNodeType,
       data: c,
+      children: [], // Collections don't have children in this structure
+      expanded: false, // Not relevant for leaf nodes here
+    }));
+
+    const tree: OriginalDatabaseTreeItem = {
+      id: database,
+      name: database,
+      expanded: expandedItems.has(database), // Use state here
+      type: 'db',
+      children: children,
+      data: undefined, // DB node has no specific data object here
     };
-  });
 
-  // tree data
-  const tree: DatabaseTreeItem = {
-    id: database,
-    name: database,
-    expanded: children.length > 0,
-    type: 'db',
-    children: children,
-  };
+    return flattenTree(tree, 0, expandedItems);
+  }, [database, collections, expandedItems, flattenTree]); // Dependencies
 
-  // Icons
-  const DatabaseIcon = icons.database;
-  const CollectionIcon = icons.navCollection;
+  // --- Virtualizer Instance ---
+  const rowVirtualizer = useVirtualizer({
+    count: flattenedNodes.length,
+    getScrollElement: () => parentRef.current,
+    estimateSize: () => 30, // Adjust this based on your average item height
+    overscan: 5, // Render items slightly outside the viewport
+  });
 
-  // hooks
-  const navigate = useNavigate();
-  const classes = useStyles();
+  // --- Event Handlers ---
+  const handleToggleExpand = (nodeId: string) => {
+    setExpandedItems(prev => {
+      const newSet = new Set(prev);
+      if (newSet.has(nodeId)) {
+        newSet.delete(nodeId);
+      } else {
+        newSet.add(nodeId);
+      }
+      return newSet;
+    });
+    // Close context menu on expand/collapse
+    setContextMenu(null);
+  };
 
-  // on node click
-  const onNodeClick = (node: DatabaseTreeItem) => {
+  const handleNodeClick = (node: FlatTreeItem) => {
+    setSelectedItemId(node.id);
     navigate(
       node.type === 'db'
         ? `/databases/${database}/${params.databasePage || 'collections'}`
@@ -128,155 +200,227 @@ const DatabaseTree: React.FC<DatabaseToolProps> = props => {
             params.collectionPage || 'schema'
           }`
     );
-    // close context menu
     setContextMenu(null);
   };
 
   const handleContextMenu = (
-    event: any,
-    nodeId: string,
-    nodeType: string,
-    object: TreeNodeObject
+    event: React.MouseEvent, // Use React.MouseEvent
+    node: FlatTreeItem
   ) => {
-    // prevent default
     event.preventDefault();
     event.stopPropagation();
-
     setContextMenu({
       mouseX: event.clientX - 2,
       mouseY: event.clientY - 4,
-      nodeId,
-      nodeType: nodeType as TreeNodeType,
-      object: object,
+      nodeId: node.id,
+      nodeType: node.type,
+      object: node.data, // Pass the data associated with the flat node
     });
   };
 
-  const handleClose = () => {
+  const handleCloseContextMenu = () => {
     setContextMenu(null);
   };
 
-  // render children
-  const renderTree = (nodes: DatabaseTreeItem[]) => {
-    return nodes.map(node => {
-      if (node.children && node.children.length > 0) {
-        return (
-          <TreeItem
-            key={node.id}
-            itemId={node.id}
-            slots={{
-              icon: CollectionIcon,
-            }}
-            label={node.name}
-            className={clcx(classes.treeItem, {
-              ['right-selected-on']: contextMenu?.nodeId === node.id,
-            })}
-            onClick={(event: any) => {
-              event.stopPropagation();
-              if (onNodeClick) {
-                onNodeClick(node);
-              }
-            }}
-          >
-            {renderTree(node.children)}
-          </TreeItem>
-        );
-      }
+  // Effect for closing context menu on outside click
+  useEffect(() => {
+    document.addEventListener('click', handleCloseContextMenu);
+    return () => {
+      document.removeEventListener('click', handleCloseContextMenu);
+    };
+  }, []);
 
-      return (
-        <TreeItem
-          key={node.id}
-          itemId={node.id}
-          slots={{
-            icon: CollectionIcon,
-          }}
-          label={<CollectionNode data={node.data!} />}
-          onContextMenu={event =>
-            handleContextMenu(event, node.id, node.type, node.data!)
+  // Track visible items for refreshing
+  useEffect(() => {
+    if (!collections.length) return;
+
+    // Track whether we're currently scrolling
+    let isScrolling = false;
+    let scrollTimeoutId: NodeJS.Timeout | null = null;
+
+    // Save references to stable values to avoid dependency changes
+    const currentFlattenedNodes = flattenedNodes;
+
+    const refreshVisibleCollections = () => {
+      // Early return if the component is unmounted
+      if (!parentRef.current) return;
+
+      const visibleItems = rowVirtualizer.getVirtualItems();
+      const visibleCollectionNames = visibleItems
+        .map(item => {
+          if (item.index >= currentFlattenedNodes.length) return null;
+          const node = currentFlattenedNodes[item.index];
+          if (node && node.type === 'collection' && node.name) {
+            return node.name;
           }
-          className={clcx(classes.treeItem, {
-            ['right-selected-on']: contextMenu?.nodeId === node.id,
-          })}
-          onClick={(event: any) => {
-            event.stopPropagation();
-            if (onNodeClick) {
-              onNodeClick(node);
-            }
-          }}
-        />
-      );
-    });
-  };
+          return null;
+        })
+        .filter(Boolean) as string[];
 
-  // useEffect
-  useEffect(() => {
-    // register click event on document, close context menu if click outside
-    document.addEventListener('click', handleClose);
+      if (visibleCollectionNames.length > 0) {
+        batchRefreshCollections(visibleCollectionNames, 'collection-tree');
+      }
+    };
+
+    // This will be called when scrolling starts
+    const handleScrollStart = () => {
+      if (!isScrolling) {
+        isScrolling = true;
+        // Execute on scroll start
+        refreshVisibleCollections();
+      }
+
+      // Clear any existing timeout
+      if (scrollTimeoutId) {
+        clearTimeout(scrollTimeoutId);
+      }
+
+      // Set a new timeout for scroll end detection
+      scrollTimeoutId = setTimeout(() => {
+        isScrolling = false;
+        // Execute on scroll end
+        refreshVisibleCollections();
+        scrollTimeoutId = null;
+      }, 500); // Wait for scrolling to stop for 300ms
+    };
+
+    // Initial refresh when component mounts - with delay to ensure UI is ready
+    const initialRefreshTimeout = setTimeout(() => {
+      refreshVisibleCollections();
+    }, 100);
+
+    // Setup scroll listener
+    const scrollElement = parentRef.current;
+    if (scrollElement) {
+      scrollElement.addEventListener('scroll', handleScrollStart);
+
+      return () => {
+        scrollElement.removeEventListener('scroll', handleScrollStart);
+        // Clear timeout on cleanup
+        if (scrollTimeoutId) {
+          clearTimeout(scrollTimeoutId);
+        }
+        clearTimeout(initialRefreshTimeout);
+      };
+    }
 
     return () => {
-      document.removeEventListener('click', handleClose);
+      if (scrollTimeoutId) {
+        clearTimeout(scrollTimeoutId);
+      }
+      clearTimeout(initialRefreshTimeout);
     };
-  }, []);
+  }, [collections.length, batchRefreshCollections, rowVirtualizer]);
 
+  // --- Rendering ---
   return (
     <>
-      <SimpleTreeView
-        expandedItems={[database]}
-        multiSelect={false}
-        disableSelection={false}
-        selectedItems={
-          params.collectionName
-            ? `c_${params.collectionName}`
-            : params.databaseName
-        }
-        className={classes.root}
+      <Box
+        ref={parentRef}
+        className={classes.root} // Apply root styles (ensure height and overflow)
+        sx={{
+          height: treeHeight, // Adjust this height based on your layout requirements
+          overflow: 'auto',
+        }}
       >
-        {
-          <TreeItem
-            key={tree.id}
-            itemId={tree.id}
-            label={
-              <Tooltip
-                title={tree.name}
-                placement="top"
-                PopperProps={{
-                  modifiers: [
-                    {
-                      name: 'offset',
-                      options: {
-                        offset: [0, -10],
-                      },
-                    },
-                  ],
+        <Box
+          sx={{
+            height: `${rowVirtualizer.getTotalSize()}px`,
+            width: '100%',
+            position: 'relative',
+            overflow: 'hidden',
+          }}
+        >
+          {rowVirtualizer.getVirtualItems().map(virtualItem => {
+            const node = flattenedNodes[virtualItem.index];
+            if (!node) return null; // Should not happen
+
+            const isSelected = node.id === selectedItemId;
+            const isContextMenuTarget = contextMenu?.nodeId === node.id;
+            const isCollectionNoSchema =
+              node.type === 'collection' &&
+              (!node.data || !(node.data as CollectionObject).schema);
+
+            return (
+              <Box
+                key={node.id}
+                sx={{
+                  position: 'absolute',
+                  top: 0,
+                  left: 0,
+                  width: `calc(100% - ${node.depth * 20}px)`, // Adjust for padding
+                  height: `${virtualItem.size}px`,
+                  transform: `translateY(${virtualItem.start}px)`,
+                  display: 'flex',
+                  flex: 1,
+                  alignItems: 'center',
+                  cursor: 'pointer',
+                  paddingLeft: `${node.depth * 20}px`, // Indent based on depth
+                  // Apply selection/hover styles based on the original classes
+                  backgroundColor: isSelected
+                    ? 'rgba(10, 206, 130, 0.28)'
+                    : isContextMenuTarget
+                      ? 'rgba(10, 206, 130, 0.08)'
+                      : 'transparent',
+                  '&:hover': {
+                    backgroundColor: isSelected
+                      ? 'rgba(10, 206, 130, 0.28)'
+                      : 'rgba(10, 206, 130, 0.08)',
+                  },
+                  opacity: isCollectionNoSchema ? 0.5 : 1,
+                  color: isCollectionNoSchema ? 'text.disabled' : 'inherit',
+                  pointerEvents: isCollectionNoSchema ? 'none' : 'auto',
                 }}
+                onClick={() => handleNodeClick(node)}
+                onContextMenu={e => handleContextMenu(e, node)}
               >
-                <Typography noWrap className={classes.dbName}>
-                  {tree.name}
-                </Typography>
-              </Tooltip>
-            }
-            className={classes.treeItem}
-            slots={{
-              icon: DatabaseIcon,
-            }}
-            onClick={(event: any) => {
-              event.stopPropagation();
-              if (onNodeClick) {
-                onNodeClick(tree);
-              }
-            }}
-            onContextMenu={event =>
-              handleContextMenu(event, tree.id, tree.type, null)
-            }
-          >
-            {tree.children && tree.children.length > 0
-              ? renderTree(tree.children)
-              : [<div key="stub" />]}
-          </TreeItem>
-        }
-      </SimpleTreeView>
+                {/* Expand/Collapse Icon */}
+                {node.hasChildren ? (
+                  <IconButton
+                    size="small"
+                    onClick={e => {
+                      e.stopPropagation();
+                      handleToggleExpand(node.id);
+                    }}
+                    sx={{ mr: 0 }}
+                  >
+                    {node.isExpanded ? (
+                      <ExpandIcon sx={{ transform: `rotate(90deg)` }} />
+                    ) : (
+                      <ExpandIcon />
+                    )}
+                  </IconButton>
+                ) : (
+                  // Placeholder for alignment if needed
+                  <Box sx={{ width: 0, mr: 0.5 }} /> // Adjust width to match IconButton
+                )}
+
+                {/* Node Type Icon */}
+                <Box sx={{ mr: 1, display: 'flex', alignItems: 'center' }}>
+                  {node.type === 'db' ? <DatabaseIcon /> : <CollectionIcon />}
+                </Box>
+
+                {/* Label */}
+                {node.type === 'collection' && node.data ? (
+                  <CollectionNode data={node.data as CollectionObject} />
+                ) : (
+                  <Tooltip title={node.name} placement="top">
+                    <Typography noWrap className={classes.dbName}>
+                      {/* Reuse dbName style or create a generic one */}
+                      {node.name}
+                    </Typography>
+                  </Tooltip>
+                )}
+              </Box>
+            );
+          })}
+        </Box>
+      </Box>
+
+      {/* Context Menu Popover (keep existing) */}
       <Popover
         open={Boolean(contextMenu)}
-        onClose={handleClose}
+        onClose={handleCloseContextMenu}
         anchorReference="anchorPosition"
         anchorPosition={
           contextMenu !== null
@@ -290,7 +434,11 @@ const DatabaseTree: React.FC<DatabaseToolProps> = props => {
           sx: { pointerEvents: 'auto', boxShadow: 4, borderRadius: 2 },
         }}
       >
-        <TreeContextMenu onClick={handleClose} contextMenu={contextMenu!} />
+        {/* Pass the correct contextMenu object */}
+        <TreeContextMenu
+          onClick={handleCloseContextMenu}
+          contextMenu={contextMenu!}
+        />
       </Popover>
     </>
   );

client/src/pages/databases/tree/style.ts

@@ -6,21 +6,15 @@ export const useStyles = makeStyles((theme: Theme) => ({
     fontSize: '15px',
     color: theme.palette.text.primary,
     backgroundColor: theme.palette.background.default,
-  },
-  treeItem: {
-    '& .right-selected-on': {
-      '& .MuiTreeItem-content': {
-        backgroundColor: 'rgba(10, 206, 130, 0.08)',
-        '&.Mui-selected': {
-          backgroundColor: 'rgba(10, 206, 130, 0.28) !important',
-        },
-      },
+    '& .MuiSvgIcon-root': {
+      fontSize: '14px',
+      color: theme.palette.text.primary,
     },
   },
   collectionNode: {
     minHeight: '24px',
     lineHeight: '24px',
-    position: 'relative',
+    display: 'flex',
   },
   collectionName: {
     display: 'flex',
@@ -45,7 +39,7 @@ export const useStyles = makeStyles((theme: Theme) => ({
     height: '8px',
     borderRadius: '50%',
     position: 'absolute',
-    left: 160,
+    right: 6,
     top: 8,
     zIndex: 1,
   },

client/src/pages/databases/tree/types.ts

@@ -13,10 +13,11 @@ export interface DatabaseTreeItem {
   data?: CollectionObject;
 }
 
-export interface DatabaseToolProps {
+export interface DatabaseTreeProps {
   database: string;
   collections: CollectionObject[];
   params: Readonly<Params<string>>;
+  treeHeight?: number;
 }
 
 export type ContextMenu = {

client/src/pages/home/Home.tsx

@@ -90,16 +90,11 @@ const Home = () => {
     return `${duration.toFixed(2)} ${unit}`;
   }, [data.rootCoord]);
 
-  // make sure the database has collections data
-  const isLoading =
-    loadingDatabases ||
-    (collections.length > 0 && collections.every(c => !c.schema));
-
   return (
     <section className={`page-wrapper  ${classes.homeWrapper}`}>
       <section className={classes.section}>
         <Typography variant="h4">{databaseTrans('databases')}</Typography>
-        {isLoading ? (
+        {loadingDatabases ? (
           <StatusIcon type={LoadingType.CREATING} />
         ) : (
           <div className={classes.cardWrapper}>

client/yarn.lock

@@ -1177,6 +1177,18 @@
     "@svgr/hast-util-to-babel-ast" "8.0.0"
     svg-parser "^2.0.4"
 
+"@tanstack/react-virtual@^3.13.6":
+  version "3.13.6"
+  resolved "https://registry.yarnpkg.com/@tanstack/react-virtual/-/react-virtual-3.13.6.tgz#30243c8c3166673caf66bfbf5352e1b314a3a4cd"
+  integrity sha512-WT7nWs8ximoQ0CDx/ngoFP7HbQF9Q2wQe4nh2NB+u2486eX3nZRE40P9g6ccCVq7ZfTSH5gFOuCoVH5DLNS/aA==
+  dependencies:
+    "@tanstack/virtual-core" "3.13.6"
+
+"@tanstack/virtual-core@3.13.6":
+  version "3.13.6"
+  resolved "https://registry.yarnpkg.com/@tanstack/virtual-core/-/virtual-core-3.13.6.tgz#329f962f1596b3280736c266a982897ed2112157"
+  integrity sha512-cnQUeWnhNP8tJ4WsGcYiX24Gjkc9ALstLbHcBj1t3E7EimN6n6kHH+DPV4PpDnuw00NApQp+ViojMj1GRdwYQg==
+
 "@types/babel__core@^7.20.5":
   version "7.20.5"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.5.tgz#3df15f27ba85319caa07ba08d0721889bb39c017"

server/src/collections/collections.controller.ts

@@ -38,6 +38,8 @@ export class CollectionController {
     // get all collections
     this.router.get('/', this.showCollections.bind(this));
     this.router.get('/names', this.getCollectionNames.bind(this));
+    this.router.post('/details', this.getCollections.bind(this));
+
     // get all collections statistics
     this.router.get('/statistics', this.getStatistics.bind(this));
     // index
@@ -164,6 +166,21 @@ export class CollectionController {
     }
   }
 
+  async getCollections(req: Request, res: Response, next: NextFunction) {
+    const collections = req.body?.collections || [];
+
+    try {
+      const result = await this.collectionsService.getAllCollections(
+        req.clientId,
+        collections,
+        req.db_name
+      );
+      res.send(result);
+    } catch (error) {
+      next(error);
+    }
+  }
+
   async getCollectionNames(req: Request, res: Response, next: NextFunction) {
     try {
       const db_name = req.query?.db_name;

server/src/collections/collections.service.ts

@@ -41,7 +41,6 @@ import {
   convertFieldSchemaToFieldType,
   LOADING_STATE,
   DYNAMIC_FIELD,
-  SimpleQueue,
   MIN_INT64,
   VectorTypes,
   cloneObj,
@@ -195,7 +194,7 @@ export class CollectionsService {
 
   async renameCollection(clientId: string, data: RenameCollectionReq) {
     const { milvusClient } = clientCache.get(clientId);
-    const res = await milvusClient.renameCollection(data);
+    await milvusClient.renameCollection(data);
 
     const newCollection = (await this.getAllCollections(
       clientId,
@@ -233,21 +232,21 @@ export class CollectionsService {
 
   async loadCollection(clientId: string, data: LoadCollectionReq) {
     const { milvusClient } = clientCache.get(clientId);
-    const res = await milvusClient.loadCollection(data);
+    await milvusClient.loadCollection(data);
 
     return data.collection_name;
   }
 
   async loadCollectionAsync(clientId: string, data: LoadCollectionReq) {
     const { milvusClient } = clientCache.get(clientId);
-    const res = await milvusClient.loadCollectionAsync(data);
+    await milvusClient.loadCollectionAsync(data);
 
     return data.collection_name;
   }
 
   async releaseCollection(clientId: string, data: ReleaseLoadCollectionReq) {
     const { milvusClient } = clientCache.get(clientId);
-    const res = await milvusClient.releaseCollection(data);
+    await milvusClient.releaseCollection(data);
 
     // emit update to client
     this.updateCollectionsDetails(
@@ -370,7 +369,7 @@ export class CollectionsService {
 
   async dropAlias(clientId: string, collection_name: string, data: any) {
     const { milvusClient } = clientCache.get(clientId);
-    const res = await milvusClient.dropAlias(data);
+    await milvusClient.dropAlias(data);
 
     const newCollection = (await this.getAllCollections(
       clientId,
@@ -411,11 +410,7 @@ export class CollectionsService {
     lazy: boolean = false,
     database?: string
   ) {
-    const { collectionsQueue } = clientCache.get(clientId);
     if (lazy) {
-      // add to lazy queue
-      collectionsQueue.enqueue(collection.name);
-
       // return lazy object
       return {
         id: collection.id,
@@ -505,14 +500,6 @@ export class CollectionsService {
     collections: string[] = [],
     database?: string
   ): Promise<CollectionObject[]> {
-    const currentClient = clientCache.get(clientId);
-
-    // clear collectionsQueue if we fetch all collections
-    if (collections.length === 0) {
-      currentClient.collectionsQueue.stop();
-      currentClient.collectionsQueue = new SimpleQueue<string>();
-    }
-
     // get all collections(name, timestamp, id)
     const allCollections = await this.showCollections(clientId, {
       db_name: database,
@@ -544,37 +531,17 @@ export class CollectionsService {
         v => v.name === collection.name
       );
 
-      const notLazy = i <= 5; // lazy is true, only load full details for the first 10 collections
-
       data.push(
         await this.getCollection(
           clientId,
           collection,
           loadedCollection,
-          !notLazy,
+          collections.length === 0, // if no collection is specified, load all collections without detail
           database
         )
       );
     }
 
-    // start the queue
-    if (currentClient.collectionsQueue.size() > 0) {
-      currentClient.collectionsQueue.executeNext(
-        async (collectionsToGet, q) => {
-          // if the queue is obseleted, return
-          if (q.isObseleted) {
-            return;
-          }
-          await this.updateCollectionsDetails(
-            clientId,
-            collectionsToGet,
-            database
-          );
-        },
-        5
-      );
-    }
-
     // return data
     return data;
   }