Merge pull request #249 from czhen-zilliz/data-query

Add data query

client/src/components/advancedSearch/Condition.tsx

@@ -71,23 +71,24 @@ const Condition: FC<ConditionProps> = props => {
   useEffect(() => {
     const regInt = /^\d+$/;
     const regFloat = /^\d+\.\d+$/;
-    const regIntInterval = /^\[\d+,\d+\]$/;
-    const regFloatInterval = /^\[\d+\.\d+,\d+\.\d+]$/;
+    const regIntInterval = /^\[\d+(,\d+)*\]$/;
+    const regFloatInterval = /^\[\d+\.\d+(,\d+\.\d+)*\]$/;
 
     const type = conditionField?.type;
     const isIn = operator === 'in';
     let isLegal = false;
+    const conditionValueWithNoSpace = conditionValue.replaceAll(' ', '');
 
     switch (type) {
       case 'int':
         isLegal = isIn
-          ? regIntInterval.test(conditionValue)
-          : regInt.test(conditionValue);
+          ? regIntInterval.test(conditionValueWithNoSpace)
+          : regInt.test(conditionValueWithNoSpace);
         break;
       case 'float':
         isLegal = isIn
-          ? regFloatInterval.test(conditionValue)
-          : regFloat.test(conditionValue);
+          ? regFloatInterval.test(conditionValueWithNoSpace)
+          : regFloat.test(conditionValueWithNoSpace);
         break;
       default:
         isLegal = false;

client/src/components/advancedSearch/Filter.tsx

@@ -1,4 +1,4 @@
-import { useState, useEffect } from 'react';
+import { forwardRef, useState, useEffect, useImperativeHandle } from 'react';
 import {
   makeStyles,
   Theme,
@@ -12,10 +12,11 @@ import { FilterProps, ConditionData } from './Types';
 import { generateIdByHash } from '../../utils/Common';
 import CustomButton from '../customButton/CustomButton';
 
-const Filter = function Filter(props: FilterProps) {
+const Filter = forwardRef((props: FilterProps, ref) => {
   const {
     title = 'title',
     showTitle = true,
+    showTooltip = true,
     className = '',
     filterDisabled = false,
     tooltipPlacement = 'top',
@@ -245,6 +246,7 @@ const Filter = function Filter(props: FilterProps) {
     setInitConditions(flatConditions);
     setOpen(false);
   };
+  // Only reset current conditions. Former conditions are remained.
   const handleReset = () => {
     setFilteredFlatConditions([
       {
@@ -256,9 +258,23 @@ const Filter = function Filter(props: FilterProps) {
       },
     ]);
   };
+  // Reset all conditions(both current and former).
+  const handleHardReset = () => {
+    setInitConditions([]);
+    handleReset();
+  };
   const handleFallback = () => {
     setFilteredFlatConditions(initConditions);
   };
+  // Expose func
+  // useImperativeHandle customizes the instance value that is exposed to parent components when using ref.
+  // https://reactjs.org/docs/hooks-reference.html#useimperativehandle
+  useImperativeHandle(ref, () => ({
+    // Expose handleHardReset, parent components can call it by `ref.current.getReset()`
+    getReset() {
+      handleHardReset();
+    },
+  }));
 
   return (
     <>
@@ -271,7 +287,7 @@ const Filter = function Filter(props: FilterProps) {
           <FilterListIcon />
           {showTitle ? title : ''}
         </CustomButton>
-        {initConditions.length > 0 && (
+        {showTooltip && initConditions.length > 0 && (
           <Tooltip
             arrow
             interactive
@@ -304,7 +320,7 @@ const Filter = function Filter(props: FilterProps) {
       </div>
     </>
   );
-};
+});
 
 Filter.displayName = 'AdvancedFilter';
 

client/src/components/advancedSearch/Types.ts

@@ -69,8 +69,9 @@ export interface FilterProps {
   className?: string;
   title: string;
   showTitle?: boolean;
+  showTooltip?: boolean;
   filterDisabled?: boolean;
-  others?: object;
+  others?: { [key: string]: any };
   onSubmit: (data: any) => void;
   tooltipPlacement?: 'left' | 'right' | 'bottom' | 'top';
   fields: Field[];

client/src/components/grid/Types.ts

@@ -41,7 +41,7 @@ export type ToolBarConfig = Partial<TableSwitchType> &
     // when disabled "disabledTooltip" will replace "tooltip"
     disabledTooltip?: string;
     hidden?: boolean;
-    type?: 'iconBtn' | 'buttton' | 'switch' | 'select' | 'groupSelect';
+    type?: 'iconBtn' | 'button' | 'switch' | 'select' | 'groupSelect';
     position?: 'right' | 'left';
     component?: ReactElement;
     btnVariant?: 'contained' | 'outlined' | 'text';

client/src/http/BaseModel.ts

@@ -79,7 +79,7 @@ export default class BaseModel {
     return res.data;
   }
 
-  static async vectorSearch(options: updateParamsType) {
+  static async query(options: updateParamsType) {
     const { path, data } = options;
     const res = await http.post(path, data);
     return res.data.data;

client/src/http/Collection.ts

@@ -2,6 +2,7 @@ import { ChildrenStatusType } from '../components/status/Types';
 import { CollectionView, InsertDataParam } from '../pages/collections/Types';
 import { Field } from '../pages/schema/Types';
 import { VectorSearchParam } from '../pages/seach/Types';
+import { QueryParam } from '../pages/query/Types';
 import { IndexState, ShowCollectionsType } from '../types/Milvus';
 import { formatNumber } from '../utils/Common';
 import BaseModel from './BaseModel';
@@ -85,7 +86,7 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   }
 
   static vectorSearchData(collectionName: string, params: VectorSearchParam) {
-    return super.vectorSearch({
+    return super.query({
       path: `${this.COLLECTIONS_URL}/${collectionName}/search`,
       data: params,
     });
@@ -98,6 +99,13 @@ export class CollectionHttp extends BaseModel implements CollectionView {
     });
   }
 
+  static queryData(collectionName: string, params: QueryParam) {
+    return super.query({
+      path: `${this.COLLECTIONS_URL}/${collectionName}/query`,
+      data: params,
+    });
+  }
+
   get _autoId() {
     return this.autoID;
   }

client/src/i18n/cn/button.ts

@@ -16,6 +16,7 @@ const btnTrans = {
   previous: 'Previous',
   done: 'Done',
   vectorSearch: 'Vector search',
+  query: 'Query',
 };
 
 export default btnTrans;

client/src/i18n/cn/collection.ts

@@ -58,6 +58,9 @@ const collectionTrans = {
   // collection tabs
   partitionTab: 'Partitions',
   schemaTab: 'Schema',
+  queryTab: 'Data Query',
+  startTip: 'Start your data query',
+  exprPlaceHolder: 'Please enter your query by using advanced filter ->',
 };
 
 export default collectionTrans;

client/src/i18n/en/button.ts

@@ -16,6 +16,7 @@ const btnTrans = {
   previous: 'Previous',
   done: 'Done',
   vectorSearch: 'Vector search',
+  query: 'Query',
 };
 
 export default btnTrans;

client/src/i18n/en/collection.ts

@@ -58,6 +58,9 @@ const collectionTrans = {
   // collection tabs
   partitionTab: 'Partitions',
   schemaTab: 'Schema',
+  queryTab: 'Data Query',
+  startTip: 'Start your data query',
+  exprPlaceHolder: 'Please enter your query by using advanced filter ->',
 };
 
 export default collectionTrans;

client/src/pages/collections/Collection.tsx

@@ -8,6 +8,7 @@ import { useHistory, useLocation, useParams } from 'react-router-dom';
 import { useMemo } from 'react';
 import { parseLocationSearch } from '../../utils/Format';
 import Schema from '../schema/Schema';
+import Query from '../query/Query';
 
 enum TAB_EMUM {
   'schema',
@@ -39,6 +40,10 @@ const Collection = () => {
   };
 
   const tabs: ITab[] = [
+    {
+      label: collectionTrans('queryTab'),
+      component: <Query collectionName={collectionName} />,
+    },
     {
       label: collectionTrans('schemaTab'),
       component: <Schema collectionName={collectionName} />,

client/src/pages/query/Query.tsx

@@ -0,0 +1,192 @@
+import { FC, useEffect, useState, useRef, useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import EmptyCard from '../../components/cards/EmptyCard';
+import icons from '../../components/icons/Icons';
+import CustomButton from '../../components/customButton/CustomButton';
+import MilvusGrid from '../../components/grid/Grid';
+import { getQueryStyles } from './Styles';
+import Filter from '../../components/advancedSearch';
+import { CollectionHttp } from '../../http/Collection';
+import { FieldHttp } from '../../http/Field';
+import { usePaginationHook } from '../../hooks/Pagination';
+import CopyButton from '../../components/advancedSearch/CopyButton';
+
+const Query: FC<{
+  collectionName: string;
+}> = ({ collectionName }) => {
+  const [fields, setFields] = useState<any[]>([]);
+  const [expression, setExpression] = useState('');
+  const [tableLoading, setTableLoading] = useState<any>();
+  const [queryResult, setQueryResult] = useState<any>();
+
+  const VectorSearchIcon = icons.vectorSearch;
+  const ResetIcon = icons.refresh;
+
+  const { t: searchTrans } = useTranslation('search');
+  const { t: collectionTrans } = useTranslation('collection');
+  const { t: btnTrans } = useTranslation('btn');
+  const { t: commonTrans } = useTranslation();
+  const copyTrans = commonTrans('copy');
+
+  const classes = getQueryStyles();
+
+  // Format result list
+  const queryResultMemo = useMemo(
+    () =>
+      queryResult?.map((resultItem: { [key: string]: any }) => {
+        // Iterate resultItem keys, then format vector(array) items.
+        const tmp = Object.keys(resultItem).reduce(
+          (prev: { [key: string]: any }, item: string) => {
+            if (Array.isArray(resultItem[item])) {
+              const list2Str = `[${resultItem[item]}]`;
+              prev[item] = (
+                <div className={classes.vectorTableCell}>
+                  <div>{list2Str}</div>
+                  <CopyButton
+                    label={copyTrans.label}
+                    value={list2Str}
+                    className={classes.copyBtn}
+                  />
+                </div>
+              );
+            } else {
+              prev[item] = resultItem[item];
+            }
+            return prev;
+          },
+          {}
+        );
+        return tmp;
+      }),
+    [queryResult, classes.vectorTableCell, classes.copyBtn, copyTrans.label]
+  );
+
+  const {
+    pageSize,
+    handlePageSize,
+    currentPage,
+    handleCurrentPage,
+    total,
+    data: result,
+    order,
+    orderBy,
+    handleGridSort,
+  } = usePaginationHook(queryResultMemo || []);
+
+  const handlePageChange = (e: any, page: number) => {
+    handleCurrentPage(page);
+  };
+
+  const getFields = async (collectionName: string) => {
+    const schemaList = await FieldHttp.getFields(collectionName);
+    const nameList = schemaList.map(i => ({
+      name: i.name,
+      type: i.data_type.includes('Int') ? 'int' : 'float',
+    }));
+    setFields(nameList);
+  };
+
+  // Get fields at first or collection name changed.
+  useEffect(() => {
+    collectionName && getFields(collectionName);
+  }, [collectionName]);
+
+  const filterRef = useRef();
+
+  const handleFilterReset = () => {
+    const currentFilter: any = filterRef.current;
+    currentFilter?.getReset();
+    setExpression('');
+    setTableLoading(null);
+    setQueryResult(null);
+  };
+  const handleFilterSubmit = (expression: string) => {
+    setExpression(expression);
+    setQueryResult(null);
+  };
+  const handleQuery = async () => {
+    setTableLoading(true);
+    try {
+      const res = await CollectionHttp.queryData(collectionName, {
+        expr: expression,
+        output_fields: fields.map(i => i.name),
+      });
+      const result = res.data;
+      setQueryResult(result);
+    } catch (err) {
+      setQueryResult([]);
+    } finally {
+      setTableLoading(false);
+    }
+  };
+
+  return (
+    <div className={classes.root}>
+      <div className={classes.toolbar}>
+        <div className="left">
+          <div>{`${
+            expression || collectionTrans('exprPlaceHolder')
+          }`}</div>
+          <Filter
+            ref={filterRef}
+            title="Advanced Filter"
+            fields={fields}
+            filterDisabled={false}
+            onSubmit={handleFilterSubmit}
+            showTitle={false}
+            showTooltip={false}
+          />
+        </div>
+        <div className="right">
+          <CustomButton className="btn" onClick={handleFilterReset}>
+            <ResetIcon classes={{ root: 'icon' }} />
+            {btnTrans('reset')}
+          </CustomButton>
+          <CustomButton
+            variant="contained"
+            disabled={!expression}
+            onClick={() => handleQuery()}
+          >
+            {btnTrans('query')}
+          </CustomButton>
+        </div>
+      </div>
+      {tableLoading || queryResult?.length ? (
+        <MilvusGrid
+          toolbarConfigs={[]}
+          colDefinitions={fields.map(i => ({
+            id: i.name,
+            align: 'left',
+            disablePadding: false,
+            label: i.name,
+          }))}
+          primaryKey={fields.find(i => i.is_primary_key)?.name}
+          openCheckBox={false}
+          isLoading={!!tableLoading}
+          rows={result}
+          rowCount={total}
+          page={currentPage}
+          onChangePage={handlePageChange}
+          rowsPerPage={pageSize}
+          setRowsPerPage={handlePageSize}
+          orderBy={orderBy}
+          order={order}
+          handleSort={handleGridSort}
+        />
+      ) : (
+        <EmptyCard
+          wrapperClass={`page-empty-card ${classes.emptyCard}`}
+          icon={<VectorSearchIcon />}
+          text={
+            queryResult?.length === 0
+              ? searchTrans('empty')
+              : collectionTrans('startTip')
+          }
+        />
+      )}
+    </div>
+  );
+};
+
+export default Query;

client/src/pages/query/Styles.ts

@@ -0,0 +1,51 @@
+import { makeStyles, Theme } from '@material-ui/core';
+
+export const getQueryStyles = makeStyles((theme: Theme) => ({
+  root: {
+    display: 'flex',
+    flexDirection: 'column',
+    height: '100%',
+  },
+  emptyCard: {
+    height: '100%',
+    borderRadius: theme.spacing(0, 0, 0.5, 0.5),
+    boxShadow: 'none',
+  },
+  toolbar: {
+    display: 'flex',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+    backgroundColor: 'white',
+    padding: theme.spacing(1, 2),
+    gap: theme.spacing(2),
+    borderRadius: theme.spacing(0.5, 0.5, 0, 0),
+
+    '& .left': {
+      display: 'flex',
+      justifyContent: 'space-between',
+      alignItems: 'center',
+      width: 'calc(100% - 206px)',
+      padding: theme.spacing(0, 0, 0, 2),
+      fontSize: theme.spacing(2),
+      backgroundColor: '#F9F9F9',
+    },
+
+    '& .right': {
+      display: 'flex',
+      justifyContent: 'space-between',
+      alignItems: 'center',
+      gap: theme.spacing(2),
+    },
+  },
+  vectorTableCell: {
+    '& >div': {
+      maxWidth: theme.spacing(50),
+      overflow: 'hidden',
+      textOverflow: 'ellipsis',
+    },
+    display: 'flex',
+    flexDirection: 'row',
+    alignItems: 'center',
+  },
+  copyBtn: {},
+}));

client/src/pages/query/Types.ts

@@ -0,0 +1,5 @@
+export interface QueryParam {
+  expr: string;
+  partitions_names?: string[];
+  output_fields?: string[];
+}

express/src/app.ts

@@ -14,7 +14,7 @@ import {
   LoggingInterceptor,
   ErrorInterceptor,
 } from "./interceptors";
-import { getDirectories, generateCfgs } from "./utils";
+import { getDirectories, getDirectoriesSync, generateCfgs } from "./utils";
 import * as path from "path";
 
 const PLUGIN_DEV = process.env?.PLUGIN_DEV;
@@ -65,8 +65,8 @@ io.on("connection", (socket: Socket) => {
 
 // Read plugin files and start express server
 // Import all plguins under "src/plugins"
-getDirectories(SRC_PLUGIN_DIR, async (dirErr: Error, dirRes: [string]) => {
-  const cfgs: any = [];
+getDirectories(SRC_PLUGIN_DIR, async (dirErr: Error, dirRes: string[]) => {
+  const cfgs: any[] = [];
   if (dirErr) {
     console.log("Reading plugin directory Error", dirErr);
   } else {
@@ -74,18 +74,18 @@ getDirectories(SRC_PLUGIN_DIR, async (dirErr: Error, dirRes: [string]) => {
   }
   // If under plugin dev mode, import all plugins under "../../src/*/server"
   if (PLUGIN_DEV) {
-    await getDirectories(
+    getDirectoriesSync(
       DEV_PLUGIN_DIR,
-      (devDirErr: Error, devDirRes: [string]) => {
+      (devDirErr: Error, devDirRes: string[]) => {
         if (devDirErr) {
-          console.log("Reading plugin directory Error", dirErr);
+          console.log("Reading dev plugin directory Error", devDirErr);
         } else {
           generateCfgs(cfgs, devDirRes, false);
         }
       }
     );
   }
-  console.log(cfgs);
+  console.log("======/api/plugins configs======", cfgs);
   cfgs.forEach(async (cfg: any) => {
     const { api: pluginPath, componentPath } = cfg;
     if (!pluginPath) return;

express/src/collections/collections.service.ts

@@ -108,6 +108,17 @@ export class CollectionsService {
     return res;
   }
 
+  async query(data: {
+    collection_name: string;
+    expr: string;
+    partitions_names?: string[];
+    output_fields?: string[];
+  }) {
+    const res = await this.dataManager.query(data);
+    throwErrorFromSDK(res.status);
+    return res;
+  }
+
   /**
    * We do not throw error for this.
    * Because if collection dont have index, it will throw error.

express/src/collections/index.ts

@@ -126,6 +126,31 @@ router.post("/:name/search", async (req, res, next) => {
     next(error);
   }
 });
+router.post("/:name/query", async (req, res, next) => {
+  const name = req.params?.name;
+  const data = req.body;
+  const resultiLmit: any = req.query?.limit;
+  const resultPage: any = req.query?.page;
+  try {
+    const limit = isNaN(resultiLmit) ? 100 : parseInt(resultiLmit, 10);
+    const page = isNaN(resultPage) ? 0 : parseInt(resultPage, 10);
+  // TODO: add page and limit to node SDK
+  // Here may raise "Error: 8 RESOURCE_EXHAUSTED: Received message larger than max"
+    const result = await collectionsService.query({
+      collection_name: name,
+      ...data,
+    });
+    const queryResultList = result.data;
+    const queryResultLength = result.data.length;
+    const startNum = page * limit;
+    const endNum = (page + 1) * limit;
+    const slicedResult = queryResultList.slice(startNum, endNum);
+    result.data = slicedResult;
+    res.send({ ...result, limit, page, total: queryResultLength });
+  } catch (error) {
+    next(error);
+  }
+});
 
 router.post("/:name/alias", async (req, res, next) => {
   const name = req.params?.name;

express/src/utils/index.ts

@@ -4,14 +4,27 @@ import fs from "fs";
 // Utils: read files under specified directories
 export const getDirectories = (
   src: string,
-  callback: (err: Error, res: [string]) => void
+  callback: (err: Error, res: string[]) => void
 ) => {
   glob(src + "/**/*", callback);
 };
 
+// sync: read files under specified directories
+export const getDirectoriesSync = (
+  src: string,
+  callback: (err: Error, res: string[]) => void
+) => {
+  try {
+    const results = glob.sync(src + "/**/*");
+    callback(undefined, results);
+  } catch (error) {
+    callback(error, []);
+  }
+};
+
 export const generateCfgs = (
-  cfgs: [any],
-  dirRes: [string],
+  cfgs: any[],
+  dirRes: string[],
   isSrcPlugin: boolean = true
 ) => {
   dirRes.forEach((item: string) => {