Support setup output fields on query page (#572)

* support setup output fields for vector search

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* remove console

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* use same filter components for search and query page

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* support select output fields for query page

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* support dynamic fields selection

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix code block

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

---------

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

client/src/hooks/Query.ts

@@ -27,6 +27,9 @@ export const useQuery = (params: {
   const [total, setTotal] = useState<number>(collection.rowCount);
   const [expr, setExpr] = useState<string>('');
   const [queryResult, setQueryResult] = useState<any>({ data: [], latency: 0 });
+  const [outputFields, setOutputFields] = useState<string[]>(
+    fields.map(i => i.name) || []
+  );
 
   // build local cache for pk ids
   const pageCache = useRef(new Map());
@@ -76,7 +79,7 @@ export const useQuery = (params: {
     try {
       const queryParams = {
         expr: _expr,
-        output_fields: fields.map(i => i.name),
+        output_fields: outputFields,
         limit: pageSize || 10,
         consistency_level,
         // travel_timestamp: timeTravelInfo.timestamp,
@@ -181,5 +184,9 @@ export const useQuery = (params: {
     count,
     // get expression
     getPageExpr,
+    // output fields
+    outputFields,
+    // set output fields
+    setOutputFields,
   };
 };

client/src/i18n/cn/collection.ts

@@ -90,7 +90,8 @@ const collectionTrans = {
   segmentsTab: '数据段(Segments)',
   propertiesTab: '属性',
   startTip: '开始你的数据查询',
-  exprPlaceHolder: '请输入你的数据查询，例如 id > 0',
+  exprPlaceHolder: '请输入你的查询表达式，例如 id > 0',
+  queryExpression: '查询表达式',
 
   // alias dialog
   aliasCreatePlaceholder: '别名',

client/src/i18n/en/collection.ts

@@ -92,7 +92,8 @@ const collectionTrans = {
   segmentsTab: 'Segments',
   propertiesTab: 'Properties',
   startTip: 'Start Your Data Query',
-  exprPlaceHolder: 'Please enter your data query, for example id > 0',
+  exprPlaceHolder: 'Please enter your query expression, eg: id > 0',
+  queryExpression: 'Query Expression',
 
   // alias dialog
   aliasCreatePlaceholder: 'Alias Name',

client/src/pages/databases/Databases.tsx

@@ -16,7 +16,7 @@ import Search from './collections/search/Search';
 import { dataContext, authContext } from '@/context';
 import Collections from './collections/Collections';
 import StatusIcon, { LoadingType } from '@/components/status/StatusIcon';
-import { ConsistencyLevelEnum } from '@/consts';
+import { ConsistencyLevelEnum, DYNAMIC_FIELD } from '@/consts';
 import RefreshButton from './RefreshButton';
 import CopyButton from '@/components/advancedSearch/CopyButton';
 import { SearchParams } from './types';
@@ -74,6 +74,8 @@ const Databases = () => {
       // if search params not found, and the schema is ready, create new search params
       if (!searchParam && c.schema) {
         setSearchParams(prevParams => {
+          const scalarFields = c.schema.scalarFields.map(s => s.name);
+
           return [
             ...prevParams,
             {
@@ -97,7 +99,9 @@ const Databases = () => {
                 weightedParams: {
                   weights: Array(c.schema.vectorFields.length).fill(0.5),
                 },
-                output_fields: c.schema.scalarFields.map(s => s.name),
+                output_fields: c.schema.enable_dynamic_field
+                  ? [...scalarFields, DYNAMIC_FIELD]
+                  : scalarFields,
               },
               searchResult: null,
               searchLatency: 0,

client/src/pages/databases/collections/data/CollectionData.tsx

@@ -1,5 +1,5 @@
 import { useState, useEffect, useRef, useContext } from 'react';
-import { TextField, Typography } from '@material-ui/core';
+import { Typography } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
 import { rootContext, dataContext } from '@/context';
 import { DataService } from '@/http';
@@ -27,6 +27,8 @@ import ImportSampleDialog from '@/pages/dialogs/ImportSampleDialog';
 import { detectItemType } from '@/utils';
 import { CollectionObject, CollectionFullObject } from '@server/types';
 import StatusIcon, { LoadingType } from '@/components/status/StatusIcon';
+import CustomInput from '@/components/customInput/CustomInput';
+import CustomMultiSelector from '@/components/customSelector/CustomMultiSelector';
 
 export interface CollectionDataProps {
   collectionName: string;
@@ -144,6 +146,8 @@ const CollectionData = (props: CollectionDataProps) => {
     query,
     reset,
     count,
+    outputFields,
+    setOutputFields,
   } = useQuery({
     collection,
     consistencyLevel,
@@ -318,44 +322,51 @@ const CollectionData = (props: CollectionDataProps) => {
           <CustomToolBar toolbarConfigs={toolbarConfigs} hideOnDisable={true} />
           <div className={classes.toolbar}>
             <div className="left">
-              <TextField
-                className="textarea"
-                value={expression}
-                onChange={(e: React.ChangeEvent<{ value: unknown }>) => {
-                  setExpression(e.target.value as string);
-                }}
-                disabled={!collection.loaded}
-                InputLabelProps={{ shrink: true }}
-                label={expression ? ' ' : collectionTrans('exprPlaceHolder')}
-                onKeyDown={e => {
-                  if (e.key === 'Enter') {
-                    // reset page
-                    setCurrentPage(0);
-                    if (expr !== expression) {
-                      setExpr(expression);
-                    } else {
-                      // ensure query
-                      query();
+              <CustomInput
+                type="text"
+                textConfig={{
+                  label: expression
+                    ? collectionTrans('queryExpression')
+                    : collectionTrans('exprPlaceHolder'),
+                  key: 'advFilter',
+                  className: 'textarea',
+                  onChange: (value: string) => {
+                    setExpression(value);
+                  },
+                  value: expression,
+                  disabled: !collection.loaded,
+                  variant: 'filled',
+                  required: false,
+                  InputLabelProps: { shrink: true },
+                  InputProps: {
+                    endAdornment: (
+                      <Filter
+                        title={''}
+                        showTitle={false}
+                        fields={collection.schema.scalarFields}
+                        filterDisabled={!collection.loaded}
+                        onSubmit={handleFilterSubmit}
+                        showTooltip={false}
+                      />
+                    ),
+                  },
+                  onKeyDown: (e: any) => {
+                    if (e.key === 'Enter') {
+                      // reset page
+                      setCurrentPage(0);
+                      if (expr !== expression) {
+                        setExpr(expression);
+                      } else {
+                        // ensure query
+                        query();
+                      }
+                      e.preventDefault();
                     }
-                    e.preventDefault();
-                  }
+                  },
                 }}
-                inputRef={inputRef}
-              />
-              <Filter
-                ref={filterRef}
-                title={btnTrans('advFilter')}
-                fields={collection.schema.fields.filter(
-                  i =>
-                    i.data_type !== DataTypeStringEnum.FloatVector &&
-                    i.data_type !== DataTypeStringEnum.BinaryVector
-                )}
-                filterDisabled={!collection.loaded}
-                onSubmit={handleFilterSubmit}
-                showTitle={false}
-                showTooltip={false}
+                checkValid={() => true}
               />
-              {/* </div> */}
+
               <CustomSelector
                 options={CONSISTENCY_LEVEL_OPTIONS}
                 value={consistencyLevel}
@@ -369,7 +380,55 @@ const CollectionData = (props: CollectionDataProps) => {
                 }}
               />
             </div>
+
             <div className="right">
+              <CustomMultiSelector
+                options={fields.map(f => {
+                  return {
+                    label:
+                      f.name === DYNAMIC_FIELD
+                        ? searchTrans('dynamicFields')
+                        : f.name,
+                    value: f.name,
+                  };
+                })}
+                values={outputFields}
+                renderValue={selected => (
+                  <span>{`${(selected as string[]).length} ${
+                    gridTrans[
+                      (selected as string[]).length > 1 ? 'fields' : 'field'
+                    ]
+                  }`}</span>
+                )}
+                label={searchTrans('outputFields')}
+                wrapperClass="selector"
+                variant="filled"
+                onChange={(e: { target: { value: unknown } }) => {
+                  // add value to output fields if not exist, remove if exist
+                  const newOutputFields = [...outputFields];
+                  const values = e.target.value as string[];
+                  const newFields = values.filter(
+                    v => !newOutputFields.includes(v as string)
+                  );
+                  const removeFields = newOutputFields.filter(
+                    v => !values.includes(v as string)
+                  );
+                  newOutputFields.push(...newFields);
+                  removeFields.forEach(f => {
+                    const index = newOutputFields.indexOf(f);
+                    newOutputFields.splice(index, 1);
+                  });
+
+                  // sort output fields by schema order
+                  newOutputFields.sort(
+                    (a, b) =>
+                      fields.findIndex(f => f.name === a) -
+                      fields.findIndex(f => f.name === b)
+                  );
+
+                  setOutputFields(newOutputFields);
+                }}
+              />
               <CustomButton
                 className="btn"
                 onClick={handleFilterReset}
@@ -397,9 +456,9 @@ const CollectionData = (props: CollectionDataProps) => {
           </div>
           <AttuGrid
             toolbarConfigs={[]}
-            colDefinitions={fields.map(i => {
+            colDefinitions={outputFields.map(i => {
               return {
-                id: i.name,
+                id: i,
                 align: 'left',
                 disablePadding: false,
                 needCopy: true,
@@ -424,10 +483,7 @@ const CollectionData = (props: CollectionDataProps) => {
                     minWidth: getColumnWidth(d.field),
                   };
                 },
-                label:
-                  i.name === DYNAMIC_FIELD
-                    ? searchTrans('dynamicFields')
-                    : i.name,
+                label: i === DYNAMIC_FIELD ? searchTrans('dynamicFields') : i,
               };
             })}
             primaryKey={collection.schema.primaryField.name}

client/src/pages/databases/collections/data/Styles.ts

@@ -23,7 +23,7 @@ export const getQueryStyles = makeStyles((theme: Theme) => ({
       justifyContent: 'space-between',
       alignItems: 'center',
       flex: 1,
-      padding: theme.spacing(0, 0, 0, 2),
+      padding: theme.spacing(0, 0, 0, 0),
       fontSize: theme.spacing(2),
       backgroundColor: '#f4f4f4',
 
@@ -37,6 +37,9 @@ export const getQueryStyles = makeStyles((theme: Theme) => ({
       },
       '& .textarea': {
         width: '100%',
+        '& .MuiFormHelperText-root': {
+          display: 'none',
+        },
       },
     },
 

client/src/pages/databases/collections/search/Search.tsx

@@ -168,7 +168,7 @@ const Search = (props: CollectionDataProps) => {
 
   // execute search
   const onSearchClicked = useCallback(async () => {
-    const params = buildSearchParams(searchParams, outputFields);
+    const params = buildSearchParams(searchParams);
 
     setTableLoading(true);
     try {
@@ -179,7 +179,6 @@ const Search = (props: CollectionDataProps) => {
 
       setTableLoading(false);
       setSearchResult(res);
-      // setLatency(res.latency);
     } catch (err) {
       setTableLoading(false);
     }
@@ -191,21 +190,6 @@ const Search = (props: CollectionDataProps) => {
 
   let primaryKeyField = 'id';
 
-  const outputFields: string[] = useMemo(() => {
-    if (!searchParams || !searchParams.collection) {
-      return [];
-    }
-
-    const s = searchParams.collection.schema!;
-    const _outputFields = [...searchParams.globalParams.output_fields];
-
-    if (s.enable_dynamic_field) {
-      _outputFields.push(DYNAMIC_FIELD);
-    }
-
-    return _outputFields;
-  }, [JSON.stringify(searchParams)]);
-
   const {
     pageSize,
     handlePageSize,
@@ -229,7 +213,16 @@ const Search = (props: CollectionDataProps) => {
   }, [JSON.stringify(searchParams), setCurrentPage]);
 
   const colDefinitions: ColDefinitionsType[] = useMemo(() => {
-    const orderArray = [primaryKeyField, 'id', 'score', ...outputFields];
+    if (!searchParams || !searchParams.collection) {
+      return [];
+    }
+    // collection fields, combine static and dynamic fields
+    const orderArray = [
+      primaryKeyField,
+      'id',
+      'score',
+      ...searchParams.globalParams.output_fields,
+    ];
 
     return searchParams &&
       searchParams.searchResult &&
@@ -278,7 +271,11 @@ const Search = (props: CollectionDataProps) => {
             };
           })
       : [];
-  }, [JSON.stringify({ searchParams, outputFields })]);
+  }, [
+    JSON.stringify({
+      searchParams,
+    }),
+  ]);
 
   // methods
   const handlePageChange = (e: any, page: number) => {
@@ -410,10 +407,7 @@ const Search = (props: CollectionDataProps) => {
               onSlideChangeCommitted={() => {
                 setHighlightField('');
               }}
-              fields={searchParams.collection.schema.scalarFields}
-              outputFields={searchParams.collection.schema.scalarFields}
               searchParams={searchParams}
-              searchGlobalParams={searchParams.globalParams}
               handleFormChange={(params: any) => {
                 searchParams.globalParams = params;
                 setSearchParams({ ...searchParams });
@@ -496,11 +490,7 @@ const Search = (props: CollectionDataProps) => {
                       params: {
                         component: (
                           <CodeDialog
-                            data={buildSearchCode(
-                              searchParams,
-                              outputFields,
-                              collection
-                            )}
+                            data={buildSearchCode(searchParams, collection)}
                           />
                         ),
                       },

client/src/pages/databases/collections/search/SearchGlobalParams.tsx

@@ -4,24 +4,20 @@ import { Slider } from '@material-ui/core';
 import CustomInput from '@/components/customInput/CustomInput';
 import CustomSelector from '@/components/customSelector/CustomSelector';
 import CustomMultiSelector from '@/components/customSelector/CustomMultiSelector';
-
 import {
+  DYNAMIC_FIELD,
   CONSISTENCY_LEVEL_OPTIONS,
   TOP_K_OPTIONS,
   RERANKER_OPTIONS,
   DataTypeStringEnum,
 } from '@/consts';
 import { SearchParams, GlobalParams } from '../../types';
-import { FieldObject } from '@server/types';
 
 export interface SearchGlobalProps {
-  searchGlobalParams: GlobalParams;
   searchParams: SearchParams;
   handleFormChange: (form: GlobalParams) => void;
   onSlideChange: (field: string) => void;
   onSlideChangeCommitted: () => void;
-  fields: FieldObject[];
-  outputFields: FieldObject[];
 }
 
 const UNSPORTED_GROUPBY_TYPES = [
@@ -34,13 +30,18 @@ const SearchGlobalParams = (props: SearchGlobalProps) => {
   // props
   const {
     searchParams,
-    searchGlobalParams,
     handleFormChange,
     onSlideChange,
     onSlideChangeCommitted,
-    fields,
-    outputFields,
   } = props;
+  // values
+  const searchGlobalParams = searchParams.globalParams;
+  const fields = searchParams.collection.schema?.scalarFields || [];
+  const outputFields = [
+    ...(searchParams.collection.schema?.scalarFields || []),
+    ...(searchParams.collection.schema?.dynamicFields || []),
+  ];
+
   const selectedCount = searchParams.searchParams.filter(
     sp => sp.selected
   ).length;
@@ -103,7 +104,11 @@ const SearchGlobalParams = (props: SearchGlobalProps) => {
 
       <CustomMultiSelector
         options={outputFields.map(f => {
-          return { label: f.name, value: f.name };
+          return {
+            label:
+              f.name === DYNAMIC_FIELD ? searchTrans('dynamicFields') : f.name,
+            value: f.name,
+          };
         })}
         values={searchGlobalParams.output_fields}
         renderValue={selected => (
@@ -116,21 +121,28 @@ const SearchGlobalParams = (props: SearchGlobalProps) => {
         variant="filled"
         onChange={(e: { target: { value: unknown } }) => {
           // add value to output fields if not exist, remove if exist
-          const outputFields = [...searchGlobalParams.output_fields];
+          const newOutputFields = [...searchGlobalParams.output_fields];
           const values = e.target.value as string[];
           const newFields = values.filter(
-            v => !outputFields.includes(v as string)
+            v => !newOutputFields.includes(v as string)
           );
-          const removeFields = outputFields.filter(
+          const removeFields = newOutputFields.filter(
             v => !values.includes(v as string)
           );
-          outputFields.push(...newFields);
+          newOutputFields.push(...newFields);
           removeFields.forEach(f => {
-            const index = outputFields.indexOf(f);
-            outputFields.splice(index, 1);
+            const index = newOutputFields.indexOf(f);
+            newOutputFields.splice(index, 1);
+          });
+
+          // sort output fields by schema order
+          newOutputFields.sort((a, b) => {
+            const aIndex = outputFields.findIndex(f => f.name === a);
+            const bIndex = outputFields.findIndex(f => f.name === b);
+            return aIndex - bIndex;
           });
 
-          handleInputChange('output_fields', outputFields);
+          handleInputChange('output_fields', newOutputFields);
         }}
       />
 

client/src/utils/search.ts

@@ -16,10 +16,7 @@ export const getVectorFieldOptions = (fields: FieldObject[]): FieldOption[] => {
 };
 
 // new search: build search params
-export const buildSearchParams = (
-  searchParams: SearchParams,
-  output_fields: string[]
-) => {
+export const buildSearchParams = (searchParams: SearchParams) => {
   const data: any = [];
   const weightedParams: number[] = [];
 
@@ -39,7 +36,7 @@ export const buildSearchParams = (
   });
 
   const params: any = {
-    output_fields: output_fields,
+    output_fields: searchParams.globalParams.output_fields,
     limit: searchParams.globalParams.topK,
     data: data,
     filter: searchParams.globalParams.filter,
@@ -72,10 +69,9 @@ export const buildSearchParams = (
 
 export const buildSearchCode = (
   searchParams: SearchParams,
-  output_fields: string[],
   collection: CollectionFullObject
 ) => {
-  const params = buildSearchParams(searchParams, output_fields);
+  const params = buildSearchParams(searchParams);
   const isMultiple = params.data.length > 1;
 
   return {
@@ -100,7 +96,7 @@ res = client.search(
   }, # Search parameters
   limit=${params.limit}, # Max. number of search results to return
   output_fields=${JSON.stringify(
-    params.output_fields
+    params.output_fields.filter((f: string) => f !== '$meta')
   )}, # Fields to return in the search results
   consistency_level="${params.consistency_level}"
 )