enhance(nginx_log): indexing status management

CLAUDE.md

@@ -78,4 +78,5 @@ This project is a web-based NGINX management interface built with Go backend and
 ## Language Requirements
 - **All code comments, documentation, and communication must be in English**
 - Maintain consistency and accessibility across the codebase
-- 优先使用 context7 mcp 搜索文档
+- 优先使用 context7 mcp 搜索文档
+- 生成 find 命令的时候应该排除掉 ./.go/ 这个文件夹，因为那里面是 devcontainer 的依赖

api/nginx_log/analytics.go

@@ -6,7 +6,6 @@ import (
 	"sort"
 	"time"
 
-	"github.com/0xJacky/Nginx-UI/internal/event"
 	"github.com/0xJacky/Nginx-UI/internal/nginx"
 	"github.com/0xJacky/Nginx-UI/internal/nginx_log"
 	"github.com/0xJacky/Nginx-UI/internal/nginx_log/analytics"
@@ -75,12 +74,6 @@ type AdvancedSearchResponseAPI struct {
 }
 
 // PreflightResponse represents the response for preflight query
-type PreflightResponse struct {
-	StartTime   *int64 `json:"start_time,omitempty"`
-	EndTime     *int64 `json:"end_time,omitempty"`
-	Available   bool   `json:"available"`
-	IndexStatus string `json:"index_status"`
-}
 
 // GetLogAnalytics provides comprehensive log analytics
 func GetLogAnalytics(c *gin.Context) {
@@ -136,78 +129,37 @@ func GetLogPreflight(c *gin.Context) {
 	// Get optional log path parameter
 	logPath := c.Query("log_path")
 
-	// Use default access log path if logPath is empty
-	if logPath == "" {
-		defaultLogPath := nginx.GetAccessLogPath()
-		if defaultLogPath != "" {
-			logPath = defaultLogPath
-			logger.Debugf("Using default access log path for preflight: %s", logPath)
-		}
-	}
-
-	// Get searcher to check index status
-	searcherService := nginx_log.GetModernSearcher()
-	if searcherService == nil {
-		cosy.ErrHandler(c, nginx_log.ErrModernSearcherNotAvailable)
+	// Create preflight service and perform check
+	preflightService := nginx_log.NewPreflightService()
+	internalResponse, err := preflightService.CheckLogPreflight(logPath)
+	if err != nil {
+		cosy.ErrHandler(c, err)
 		return
 	}
 
-	// Check if the specific file is currently being indexed
-	processingManager := event.GetProcessingStatusManager()
-	currentStatus := processingManager.GetCurrentStatus()
-	
-	// Check if searcher is healthy (indicates index is available)
-	available := searcherService.IsHealthy()
-	indexStatus := "not_ready"
-	
-	if available {
-		indexStatus = analytics.IndexStatusReady
-	}
-	
-	// If global indexing is in progress, check if this specific file has existing index data
-	// Only mark as unavailable if the file specifically doesn't have indexed data yet
-	if currentStatus.NginxLogIndexing {
-		// Check if this specific file has been indexed before
-		logFileManager := nginx_log.GetLogFileManager()
-		if logFileManager != nil {
-			logGroup, err := logFileManager.GetLogByPath(logPath)
-			if err != nil || logGroup == nil || !logGroup.HasTimeRange {
-				// This specific file hasn't been indexed yet
-				indexStatus = "indexing"
-				available = false
-			}
-			// If logGroup exists with time range, file was previously indexed and remains available
-		}
+	// Convert internal response to API response
+	response := PreflightResponse{
+		Available:   internalResponse.Available,
+		IndexStatus: internalResponse.IndexStatus,
+		Message:     internalResponse.Message,
 	}
 
-	// Try to get the actual time range from the persisted log metadata.
-	var startTime, endTime *int64
-	logFileManager := nginx_log.GetLogFileManager()
-	if logFileManager != nil {
-		logGroup, err := logFileManager.GetLogByPath(logPath)
-		if err == nil && logGroup != nil && logGroup.HasTimeRange {
-			startTime = &logGroup.TimeRangeStart
-			endTime = &logGroup.TimeRangeEnd
-		} else {
-			// Fallback for when there is no DB record or no time range yet.
-			now := time.Now().Unix()
-			monthAgo := now - (30 * 24 * 60 * 60) // 30 days ago
-			startTime = &monthAgo
-			endTime = &now
+	if internalResponse.TimeRange != nil {
+		response.TimeRange = &TimeRange{
+			Start: internalResponse.TimeRange.Start,
+			End:   internalResponse.TimeRange.End,
 		}
 	}
 
-	// Convert internal result to API response
-	response := PreflightResponse{
-		StartTime:   startTime,
-		EndTime:     endTime,
-		Available:   available,
-		IndexStatus: indexStatus,
+	if internalResponse.FileInfo != nil {
+		response.FileInfo = &FileInfo{
+			Exists:       internalResponse.FileInfo.Exists,
+			Readable:     internalResponse.FileInfo.Readable,
+			Size:         internalResponse.FileInfo.Size,
+			LastModified: internalResponse.FileInfo.LastModified,
+		}
 	}
 
-	logger.Debugf("Preflight response: log_path=%s, available=%v, index_status=%s",
-		logPath, available, indexStatus)
-
 	c.JSON(http.StatusOK, response)
 }
 
@@ -255,11 +207,11 @@ func AdvancedSearchLogs(c *gin.Context) {
 		SortBy:              req.SortBy,
 		SortOrder:           req.SortOrder,
 		UseCache:            true,
-		Timeout:             30 * time.Second, // Add timeout for large facet operations
+		Timeout:             60 * time.Second, // Add timeout for large facet operations
 		IncludeHighlighting: true,
-		IncludeFacets:       true,                        // Re-enable facets for accurate summary stats
+		IncludeFacets:       true,                         // Re-enable facets for accurate summary stats
 		FacetFields:         []string{"ip", "path_exact"}, // For UV and Unique Pages
-		FacetSize:           10000,                       // Balanced: large enough for most cases, but not excessive
+		FacetSize:           10000,                        // Balanced: large enough for most cases, but not excessive
 	}
 
 	// If no sorting is specified, default to sorting by timestamp descending.
@@ -273,11 +225,17 @@ func AdvancedSearchLogs(c *gin.Context) {
 		logPaths, err := nginx_log.ExpandLogGroupPath(req.LogPath)
 		if err != nil {
 			logger.Warnf("Could not expand log group path %s: %v", req.LogPath, err)
-			// Fallback to using the raw path
+			// Fallback to using the raw path when expansion fails
+			searchReq.LogPaths = []string{req.LogPath}
+		} else if len(logPaths) == 0 {
+			// ExpandLogGroupPath succeeded but returned empty slice (file doesn't exist on filesystem)
+			// Still search for historical indexed data using the requested path
+			logger.Debugf("Log file %s does not exist on filesystem, but searching for historical indexed data", req.LogPath)
 			searchReq.LogPaths = []string{req.LogPath}
 		} else {
 			searchReq.LogPaths = logPaths
 		}
+		logger.Debugf("Search request LogPaths: %v", searchReq.LogPaths)
 	}
 
 	// Add time filters
@@ -350,7 +308,7 @@ func AdvancedSearchLogs(c *gin.Context) {
 	pv := int(result.TotalHits)
 	var uv, uniquePages int
 	var facetUV, facetUniquePages int
-	
+
 	// First get facet values as fallback
 	if result.Facets != nil {
 		if ipFacet, ok := result.Facets["ip"]; ok {
@@ -362,17 +320,17 @@ func AdvancedSearchLogs(c *gin.Context) {
 			uniquePages = facetUniquePages
 		}
 	}
-	
+
 	// Override with CardinalityCounter results for better accuracy
 	if analyticsService != nil {
 		// Get cardinality counts for UV (unique IPs)
-		if uvResult := getCardinalityCount(ctx, analyticsService, "ip", searchReq); uvResult > 0 {
+		if uvResult := getCardinalityCount(ctx, "ip", searchReq); uvResult > 0 {
 			uv = uvResult
 			logger.Debugf("🔢 Search endpoint - UV from CardinalityCounter: %d (vs facet: %d)", uvResult, facetUV)
 		}
-		
-		// Get cardinality counts for Unique Pages (unique paths)  
-		if upResult := getCardinalityCount(ctx, analyticsService, "path_exact", searchReq); upResult > 0 {
+
+		// Get cardinality counts for Unique Pages (unique paths)
+		if upResult := getCardinalityCount(ctx, "path_exact", searchReq); upResult > 0 {
 			uniquePages = upResult
 			logger.Debugf("🔢 Search endpoint - Unique Pages from CardinalityCounter: %d (vs facet: %d)", upResult, facetUniquePages)
 		}
@@ -476,9 +434,9 @@ func GetLogEntries(c *gin.Context) {
 		entries = append(entries, hit.Fields)
 	}
 
-	c.JSON(http.StatusOK, gin.H{
-		"entries": entries,
-		"count":   len(entries),
+	c.JSON(http.StatusOK, AnalyticsResponse{
+		Entries: entries,
+		Count:   len(entries),
 	})
 }
 
@@ -593,7 +551,7 @@ func GetDashboardAnalytics(c *gin.Context) {
 	if req.StartDate != "" {
 		startTime, err = time.Parse("2006-01-02", req.StartDate)
 		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid start_date format, expected YYYY-MM-DD: " + err.Error()})
+			c.JSON(http.StatusBadRequest, ErrorResponse{Error: "Invalid start_date format, expected YYYY-MM-DD: " + err.Error()})
 			return
 		}
 	}
@@ -601,7 +559,7 @@ func GetDashboardAnalytics(c *gin.Context) {
 	if req.EndDate != "" {
 		endTime, err = time.Parse("2006-01-02", req.EndDate)
 		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid end_date format, expected YYYY-MM-DD: " + err.Error()})
+			c.JSON(http.StatusBadRequest, ErrorResponse{Error: "Invalid end_date format, expected YYYY-MM-DD: " + err.Error()})
 			return
 		}
 		// Set end time to end of day
@@ -620,13 +578,12 @@ func GetDashboardAnalytics(c *gin.Context) {
 
 	logger.Debugf("Dashboard request for log_path: %s, parsed start_time: %v, end_time: %v", req.LogPath, startTime, endTime)
 
-	// Expand the log path to its full list of physical files
-	logPaths, err := nginx_log.ExpandLogGroupPath(req.LogPath)
-	if err != nil {
-		// Log the error but proceed with the base path as a fallback
-		logger.Warnf("Could not expand log group path for dashboard %s: %v", req.LogPath, err)
-		logPaths = []string{req.LogPath}
-	}
+	// For Dashboard queries, only query the specific file requested, not the entire log group
+	// This ensures that when user clicks on a specific file, they see data only from that file
+	logPaths := []string{req.LogPath}
+	
+	// Debug: Log the paths being queried
+	logger.Debugf("Dashboard querying specific log path: %s", req.LogPath)
 
 	// Build dashboard query request
 	dashboardReq := &analytics.DashboardQueryRequest{
@@ -760,8 +717,8 @@ func GetWorldMapData(c *gin.Context) {
 	}
 	logger.Debugf("=== DEBUG GetWorldMapData END ===")
 
-	c.JSON(http.StatusOK, gin.H{
-		"data": chartData,
+	c.JSON(http.StatusOK, GeoRegionResponse{
+		Data: chartData,
 	})
 }
 
@@ -862,8 +819,8 @@ func GetChinaMapData(c *gin.Context) {
 	}
 	logger.Debugf("=== DEBUG GetChinaMapData END ===")
 
-	c.JSON(http.StatusOK, gin.H{
-		"data": chartData,
+	c.JSON(http.StatusOK, GeoDataResponse{
+		Data: chartData,
 	})
 }
 
@@ -871,7 +828,7 @@ func GetChinaMapData(c *gin.Context) {
 func GetGeoStats(c *gin.Context) {
 	var req AnalyticsRequest
 	if err := c.ShouldBindJSON(&req); err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid JSON request body: " + err.Error()})
+		c.JSON(http.StatusBadRequest, ErrorResponse{Error: "Invalid JSON request body: " + err.Error()})
 		return
 	}
 
@@ -928,13 +885,21 @@ func GetGeoStats(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
-		"stats": stats,
+	// Convert to []interface{} for JSON serialization
+	statsInterface := make([]interface{}, len(stats))
+	for i, stat := range stats {
+		statsInterface[i] = stat
+	}
+	
+	c.JSON(http.StatusOK, GeoStatsResponse{
+		Stats: statsInterface,
 	})
 }
 
 // getCardinalityCount is a helper function to get accurate cardinality counts using the analytics service
-func getCardinalityCount(ctx context.Context, analyticsService analytics.Service, field string, searchReq *searcher.SearchRequest) int {
+func getCardinalityCount(ctx context.Context, field string, searchReq *searcher.SearchRequest) int {
+	logger.Debugf("🔍 getCardinalityCount: Starting cardinality count for field '%s'", field)
+
 	// Create a CardinalityRequest from the SearchRequest
 	cardReq := &searcher.CardinalityRequest{
 		Field:     field,
@@ -942,30 +907,41 @@ func getCardinalityCount(ctx context.Context, analyticsService analytics.Service
 		EndTime:   searchReq.EndTime,
 		LogPaths:  searchReq.LogPaths,
 	}
-	
+	logger.Debugf("🔍 CardinalityRequest: Field=%s, StartTime=%v, EndTime=%v, LogPaths=%v",
+		cardReq.Field, cardReq.StartTime, cardReq.EndTime, cardReq.LogPaths)
+
 	// Try to get the searcher to access cardinality counter
 	searcherService := nginx_log.GetModernSearcher()
 	if searcherService == nil {
 		logger.Debugf("🚨 getCardinalityCount: ModernSearcher not available for field %s", field)
 		return 0
 	}
-	
+	logger.Debugf("🔍 getCardinalityCount: ModernSearcher available, type: %T", searcherService)
+
 	// Try to cast to DistributedSearcher to access CardinalityCounter
 	if ds, ok := searcherService.(*searcher.DistributedSearcher); ok {
+		logger.Debugf("🔍 getCardinalityCount: Successfully cast to DistributedSearcher")
 		shards := ds.GetShards()
+		logger.Debugf("🔍 getCardinalityCount: Retrieved %d shards", len(shards))
 		if len(shards) > 0 {
+			// Check shard health
+			for i, shard := range shards {
+				logger.Debugf("🔍 getCardinalityCount: Shard %d: %v", i, shard != nil)
+			}
+
 			cardinalityCounter := searcher.NewCardinalityCounter(shards)
+			logger.Debugf("🔍 getCardinalityCount: Created CardinalityCounter")
 			result, err := cardinalityCounter.CountCardinality(ctx, cardReq)
 			if err != nil {
 				logger.Debugf("🚨 getCardinalityCount: CardinalityCounter failed for field %s: %v", field, err)
 				return 0
 			}
-			
+
 			if result.Error != "" {
 				logger.Debugf("🚨 getCardinalityCount: CardinalityCounter returned error for field %s: %s", field, result.Error)
 				return 0
 			}
-			
+
 			logger.Debugf("✅ getCardinalityCount: Successfully got cardinality for field %s: %d", field, result.Cardinality)
 			return int(result.Cardinality)
 		} else {
@@ -974,6 +950,6 @@ func getCardinalityCount(ctx context.Context, analyticsService analytics.Service
 	} else {
 		logger.Debugf("🚨 getCardinalityCount: Searcher is not DistributedSearcher (type: %T) for field %s", searcherService, field)
 	}
-	
+
 	return 0
 }

api/nginx_log/index_management.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/0xJacky/Nginx-UI/internal/event"
@@ -14,6 +15,50 @@ import (
 	"github.com/uozi-tech/cosy/logger"
 )
 
+// rebuildLocks tracks ongoing rebuild operations for specific log groups
+var (
+	rebuildLocks     = make(map[string]*sync.Mutex)
+	rebuildLocksLock sync.RWMutex
+)
+
+// acquireRebuildLock gets or creates a mutex for a specific log group
+func acquireRebuildLock(logGroupPath string) *sync.Mutex {
+	rebuildLocksLock.Lock()
+	defer rebuildLocksLock.Unlock()
+	
+	if lock, exists := rebuildLocks[logGroupPath]; exists {
+		return lock
+	}
+	
+	lock := &sync.Mutex{}
+	rebuildLocks[logGroupPath] = lock
+	return lock
+}
+
+// releaseRebuildLock removes the mutex for a specific log group after completion
+func releaseRebuildLock(logGroupPath string) {
+	rebuildLocksLock.Lock()
+	defer rebuildLocksLock.Unlock()
+	delete(rebuildLocks, logGroupPath)
+}
+
+// isRebuildInProgress checks if a rebuild is currently running for a specific log group
+func isRebuildInProgress(logGroupPath string) bool {
+	rebuildLocksLock.RLock()
+	defer rebuildLocksLock.RUnlock()
+	
+	if lock, exists := rebuildLocks[logGroupPath]; exists {
+		// Try to acquire the lock with a short timeout
+		// If we can't acquire it, it means rebuild is in progress
+		if lock.TryLock() {
+			lock.Unlock()
+			return false
+		}
+		return true
+	}
+	return false
+}
+
 // RebuildIndex rebuilds the log index asynchronously (all files or specific file)
 // The API call returns immediately and the rebuild happens in background
 func RebuildIndex(c *gin.Context) {
@@ -40,14 +85,20 @@ func RebuildIndex(c *gin.Context) {
 	processingManager := event.GetProcessingStatusManager()
 	currentStatus := processingManager.GetCurrentStatus()
 	if currentStatus.NginxLogIndexing {
-		cosy.ErrHandler(c, fmt.Errorf("index rebuild is already in progress"))
+		cosy.ErrHandler(c, nginx_log.ErrFailedToRebuildIndex)
+		return
+	}
+
+	// Check if specific log group rebuild is already in progress
+	if request.Path != "" && isRebuildInProgress(request.Path) {
+		cosy.ErrHandler(c, nginx_log.ErrFailedToRebuildFileIndex)
 		return
 	}
 
 	// Return immediate response to client
-	c.JSON(http.StatusOK, gin.H{
-		"message": "Index rebuild started in background",
-		"status":  "started",
+	c.JSON(http.StatusOK, IndexRebuildResponse{
+		Message: "Index rebuild started in background",
+		Status:  "started",
 	})
 
 	// Start async rebuild in goroutine
@@ -93,9 +144,7 @@ func performAsyncRebuild(modernIndexer interface{}, path string) {
 		}
 
 		// Re-initialize the indexer to create new, empty shards
-		if err := modernIndexer.(interface {
-			Start(context.Context) error
-		}).Start(ctx); err != nil {
+		if err := modernIndexer.(indexer.RestartableIndexer).Start(ctx); err != nil {
 			logger.Errorf("Failed to re-initialize indexer after destruction: %v", err)
 			return
 		}
@@ -200,6 +249,13 @@ func performAsyncRebuild(modernIndexer interface{}, path string) {
 
 // rebuildSingleFile rebuilds index for a single file
 func rebuildSingleFile(modernIndexer interface{}, path string, logFileManager interface{}, progressConfig *indexer.ProgressConfig) (*time.Time, *time.Time) {
+	// Acquire lock for this specific log group
+	lock := acquireRebuildLock(path)
+	lock.Lock()
+	defer func() {
+		lock.Unlock()
+		releaseRebuildLock(path)
+	}()
 	// For a single file, we need to check its type first
 	allLogsForTypeCheck := nginx_log.GetAllLogsWithIndexGrouped()
 	var targetLog *nginx_log.NginxLogWithIndex
@@ -215,9 +271,7 @@ func rebuildSingleFile(modernIndexer interface{}, path string, logFileManager in
 	if targetLog != nil && targetLog.Type == "error" {
 		logger.Infof("Skipping index rebuild for error log as requested: %s", path)
 		if logFileManager != nil {
-			if err := logFileManager.(interface {
-				SaveIndexMetadata(string, uint64, time.Time, time.Duration, *time.Time, *time.Time) error
-			}).SaveIndexMetadata(path, 0, time.Now(), 0, nil, nil); err != nil {
+			if err := logFileManager.(indexer.MetadataManager).SaveIndexMetadata(path, 0, time.Now(), 0, nil, nil); err != nil {
 				logger.Warnf("Could not reset metadata for skipped error log %s: %v", path, err)
 			}
 		}
@@ -226,9 +280,7 @@ func rebuildSingleFile(modernIndexer interface{}, path string, logFileManager in
 
 		// Clear existing database records for this log group before rebuilding
 		if logFileManager != nil {
-			if err := logFileManager.(interface {
-				DeleteIndexMetadataByGroup(string) error
-			}).DeleteIndexMetadataByGroup(path); err != nil {
+			if err := logFileManager.(indexer.MetadataManager).DeleteIndexMetadataByGroup(path); err != nil {
 				logger.Warnf("Could not clean up existing DB records for log group %s: %v", path, err)
 			}
 		}
@@ -247,21 +299,29 @@ func rebuildSingleFile(modernIndexer interface{}, path string, logFileManager in
 		duration := time.Since(startTime)
 		var totalDocsIndexed uint64
 		if logFileManager != nil {
-			for filePath, docCount := range docsCountMap {
+			// Calculate total document count
+			for _, docCount := range docsCountMap {
 				totalDocsIndexed += docCount
-				if err := logFileManager.(interface {
-					SaveIndexMetadata(string, uint64, time.Time, time.Duration, *time.Time, *time.Time) error
-				}).SaveIndexMetadata(filePath, docCount, startTime, duration, minTime, maxTime); err != nil {
-					logger.Errorf("Failed to save index metadata for %s: %v", filePath, err)
+			}
+			
+			// Save metadata for the base log path with total count
+			if err := logFileManager.(indexer.MetadataManager).SaveIndexMetadata(path, totalDocsIndexed, startTime, duration, minTime, maxTime); err != nil {
+				logger.Errorf("Failed to save index metadata for %s: %v", path, err)
+			}
+			
+			// Also save individual file metadata if needed
+			for filePath, docCount := range docsCountMap {
+				if filePath != path { // Don't duplicate the base path
+					if err := logFileManager.(indexer.MetadataManager).SaveIndexMetadata(filePath, docCount, startTime, duration, minTime, maxTime); err != nil {
+						logger.Errorf("Failed to save index metadata for %s: %v", filePath, err)
+					}
 				}
 			}
 		}
 		logger.Infof("Successfully completed modern rebuild for file group: %s, Documents: %d", path, totalDocsIndexed)
 	}
 
-	if err := modernIndexer.(interface {
-		FlushAll() error
-	}).FlushAll(); err != nil {
+	if err := modernIndexer.(indexer.FlushableIndexer).FlushAll(); err != nil {
 		logger.Errorf("Failed to flush all indexer data for single file: %v", err)
 	}
 	nginx_log.UpdateSearcherShards()
@@ -269,40 +329,84 @@ func rebuildSingleFile(modernIndexer interface{}, path string, logFileManager in
 	return minTime, maxTime
 }
 
-// rebuildAllFiles rebuilds indexes for all files
+// rebuildAllFiles rebuilds indexes for all files with proper queue management
 func rebuildAllFiles(modernIndexer interface{}, logFileManager interface{}, progressConfig *indexer.ProgressConfig) (*time.Time, *time.Time) {
+	// For full rebuild, we use a special global lock key
+	globalLockKey := "__GLOBAL_REBUILD__"
+	lock := acquireRebuildLock(globalLockKey)
+	lock.Lock()
+	defer func() {
+		lock.Unlock()
+		releaseRebuildLock(globalLockKey)
+	}()
+	
 	if logFileManager != nil {
-		if err := logFileManager.(interface {
-			DeleteAllIndexMetadata() error
-		}).DeleteAllIndexMetadata(); err != nil {
+		if err := logFileManager.(indexer.MetadataManager).DeleteAllIndexMetadata(); err != nil {
 			logger.Errorf("Could not clean up all old DB records before full rebuild: %v", err)
 		}
 	}
 
-	logger.Info("Starting full modern index rebuild")
+	logger.Info("Starting full modern index rebuild with queue management")
 	allLogs := nginx_log.GetAllLogsWithIndexGrouped()
+	
+	// Get persistence manager for queue management
+	var persistence *indexer.PersistenceManager
+	if lfm, ok := logFileManager.(*indexer.LogFileManager); ok {
+		persistence = lfm.GetPersistence()
+	}
 
-	startTime := time.Now()
-	var overallMinTime, overallMaxTime *time.Time
-
+	// First pass: Set all access logs to queued status
+	queuePosition := 1
+	accessLogs := make([]*nginx_log.NginxLogWithIndex, 0)
+	
 	for _, log := range allLogs {
 		if log.Type == "error" {
 			logger.Infof("Skipping indexing for error log: %s", log.Path)
 			if logFileManager != nil {
-				if err := logFileManager.(interface {
-					SaveIndexMetadata(string, uint64, time.Time, time.Duration, *time.Time, *time.Time) error
-				}).SaveIndexMetadata(log.Path, 0, time.Now(), 0, nil, nil); err != nil {
+				if err := logFileManager.(indexer.MetadataManager).SaveIndexMetadata(log.Path, 0, time.Now(), 0, nil, nil); err != nil {
 					logger.Warnf("Could not reset metadata for skipped error log %s: %v", log.Path, err)
 				}
 			}
 			continue
 		}
+		
+		// Set to queued status with position
+		if persistence != nil {
+			if err := persistence.SetIndexStatus(log.Path, string(indexer.IndexStatusQueued), queuePosition, ""); err != nil {
+				logger.Errorf("Failed to set queued status for %s: %v", log.Path, err)
+			}
+		}
+		
+		accessLogs = append(accessLogs, log)
+		queuePosition++
+	}
+
+	// Give the frontend a moment to refresh and show queued status
+	time.Sleep(2 * time.Second)
+
+	startTime := time.Now()
+	var overallMinTime, overallMaxTime *time.Time
+
+	// Second pass: Process each queued log and set to indexing, then indexed
+	for _, log := range accessLogs {
+		// Set to indexing status
+		if persistence != nil {
+			if err := persistence.SetIndexStatus(log.Path, string(indexer.IndexStatusIndexing), 0, ""); err != nil {
+				logger.Errorf("Failed to set indexing status for %s: %v", log.Path, err)
+			}
+		}
 
 		loopStartTime := time.Now()
 		docsCountMap, minTime, maxTime, err := modernIndexer.(*indexer.ParallelIndexer).IndexLogGroupWithProgress(log.Path, progressConfig)
 
 		if err != nil {
-			logger.Warnf("Failed to index file group, skipping: %s, error: %v", log.Path, err)
+			logger.Warnf("Failed to index file group: %s, error: %v", log.Path, err)
+			// Set error status
+			if persistence != nil {
+				if err := persistence.SetIndexStatus(log.Path, string(indexer.IndexStatusError), 0, err.Error()); err != nil {
+					logger.Errorf("Failed to set error status for %s: %v", log.Path, err)
+				}
+			}
 		} else {
 			// Track overall time range across all log files
 			if minTime != nil {
@@ -318,23 +422,40 @@ func rebuildAllFiles(modernIndexer interface{}, logFileManager interface{}, prog
 
 			if logFileManager != nil {
 				duration := time.Since(loopStartTime)
+				// Calculate total document count for the log group
+				var totalDocCount uint64
+				for _, docCount := range docsCountMap {
+					totalDocCount += docCount
+				}
+				
+				// Save metadata for the base log path with total count
+				if err := logFileManager.(indexer.MetadataManager).SaveIndexMetadata(log.Path, totalDocCount, loopStartTime, duration, minTime, maxTime); err != nil {
+					logger.Errorf("Failed to save index metadata for %s: %v", log.Path, err)
+				}
+				
+				// Also save individual file metadata if needed
 				for path, docCount := range docsCountMap {
-					if err := logFileManager.(interface {
-						SaveIndexMetadata(string, uint64, time.Time, time.Duration, *time.Time, *time.Time) error
-					}).SaveIndexMetadata(path, docCount, loopStartTime, duration, minTime, maxTime); err != nil {
-						logger.Errorf("Failed to save index metadata for %s: %v", path, err)
+					if path != log.Path { // Don't duplicate the base path
+						if err := logFileManager.(indexer.MetadataManager).SaveIndexMetadata(path, docCount, loopStartTime, duration, minTime, maxTime); err != nil {
+							logger.Errorf("Failed to save index metadata for %s: %v", path, err)
+						}
 					}
 				}
 			}
+			
+			// Set to indexed status
+			if persistence != nil {
+				if err := persistence.SetIndexStatus(log.Path, string(indexer.IndexStatusIndexed), 0, ""); err != nil {
+					logger.Errorf("Failed to set indexed status for %s: %v", log.Path, err)
+				}
+			}
 		}
 	}
 
 	totalDuration := time.Since(startTime)
 	logger.Infof("Successfully completed full modern index rebuild in %s", totalDuration)
 
-	if err := modernIndexer.(interface {
-		FlushAll() error
-	}).FlushAll(); err != nil {
+	if err := modernIndexer.(indexer.FlushableIndexer).FlushAll(); err != nil {
 		logger.Errorf("Failed to flush all indexer data: %v", err)
 	}
 

api/nginx_log/log_list.go

@@ -89,13 +89,13 @@ func GetLogList(c *gin.Context) {
 		}
 	}
 
-	c.JSON(http.StatusOK, gin.H{
-		"data": data,
-		"summary": gin.H{
-			"total_files":    totalCount,
-			"indexed_files":  indexedCount,
-			"indexing_files": indexingCount,
-			"document_count": totalDocuments,
+	c.JSON(http.StatusOK, LogListResponse{
+		Data: data,
+		Summary: LogListSummary{
+			TotalFiles:    totalCount,
+			IndexedFiles:  indexedCount,
+			IndexingFiles: indexingCount,
+			DocumentCount: int(totalDocuments),
 		},
 	})
 }

api/nginx_log/types.go

@@ -18,4 +18,73 @@ type nginxLogPageResp struct {
 	Content string                 `json:"content"`         // Log content
 	Page    int64                  `json:"page"`            // Current page number
 	Error   *translation.Container `json:"error,omitempty"` // Error message if any
+}
+
+// FileInfo represents basic file information
+type FileInfo struct {
+	Exists        bool  `json:"exists"`
+	Readable      bool  `json:"readable"`
+	Size          int64 `json:"size,omitempty"`
+	LastModified  int64 `json:"last_modified,omitempty"`
+}
+
+// TimeRange represents a time range for log data
+type TimeRange struct {
+	Start int64 `json:"start"`
+	End   int64 `json:"end"`
+}
+
+// PreflightResponse represents the response from preflight checks
+type PreflightResponse struct {
+	Available   bool       `json:"available"`
+	IndexStatus string     `json:"index_status"`
+	Message     string     `json:"message,omitempty"`
+	TimeRange   *TimeRange `json:"time_range,omitempty"`
+	FileInfo    *FileInfo  `json:"file_info,omitempty"`
+}
+
+// AnalyticsResponse represents the response for analytics endpoints
+type AnalyticsResponse struct {
+	Entries []map[string]interface{} `json:"entries"`
+	Count   int                      `json:"count"`
+}
+
+// GeoDataResponse represents the response for geographic data
+type GeoDataResponse struct {
+	Data []GeoDataItem `json:"data"`
+}
+
+// GeoRegionResponse represents the response for geographic region data
+type GeoRegionResponse struct {
+	Data []GeoRegionItem `json:"data"`
+}
+
+// GeoStatsResponse represents the response for geographic statistics
+type GeoStatsResponse struct {
+	Stats []interface{} `json:"stats"`
+}
+
+// ErrorResponse represents a standard error response
+type ErrorResponse struct {
+	Error string `json:"error"`
+}
+
+// IndexRebuildResponse represents the response for index rebuild operations
+type IndexRebuildResponse struct {
+	Message string `json:"message"`
+	Status  string `json:"status"`
+}
+
+// LogListSummary represents summary statistics for log list
+type LogListSummary struct {
+	TotalFiles    int `json:"total_files"`
+	IndexedFiles  int `json:"indexed_files"`
+	IndexingFiles int `json:"indexing_files"`
+	DocumentCount int `json:"document_count"`
+}
+
+// LogListResponse represents the response for log list endpoint
+type LogListResponse struct {
+	Data    interface{}    `json:"data"`
+	Summary LogListSummary `json:"summary"`
 }

app/src/api/nginx_log.ts

@@ -15,6 +15,12 @@ export interface NginxLogData {
   timerange_start?: number
   timerange_end?: number
   document_count?: number
+  // Enhanced status tracking fields
+  error_message?: string
+  error_time?: number
+  retry_count?: number
+  queue_position?: number
+  partial_offset?: number
 }
 
 export interface AnalyticsRequest {
@@ -159,10 +165,19 @@ export interface AdvancedSearchResponse {
 }
 
 export interface PreflightResponse {
-  start_time: number
-  end_time: number
   available: boolean
   index_status: string
+  message?: string
+  time_range?: {
+    start: number
+    end: number
+  }
+  file_info?: {
+    exists: boolean
+    readable: boolean
+    size?: number
+    last_modified?: number
+  }
 }
 
 // Index status related interfaces

app/src/views/nginx_log/NginxLogList.vue

@@ -125,10 +125,32 @@ const indexColumns: StdTableColumn[] = [
           return (
             <Badge status="success" text={$gettext('Indexed')} />
           )
+        case 'ready':
+          return (
+            <Badge status="success" text={$gettext('Ready')} />
+          )
         case 'indexing':
           return (
             <Badge status="processing" text={$gettext('Indexing')} />
           )
+        case 'error':
+          return (
+            <Tooltip title={record.error_message || $gettext('Index failed')}>
+              <Badge status="error" text={$gettext('Error')} />
+            </Tooltip>
+          )
+        case 'partial':
+          return (
+            <Badge status="processing" text={$gettext('Partial')} />
+          )
+        case 'queued': {
+          const queueText = record.queue_position
+            ? `${$gettext('Queued')} (#${record.queue_position})`
+            : $gettext('Queued')
+          return (
+            <Badge status="processing" text={queueText} />
+          )
+        }
         case 'not_indexed':
         default:
           return (
@@ -142,16 +164,32 @@ const indexColumns: StdTableColumn[] = [
       select: {
         options: [
           {
-            label: () => $gettext('Indexed'),
-            value: 'true',
+            label: () => $gettext('Not Indexed'),
+            value: 'not_indexed',
           },
           {
             label: () => $gettext('Indexing'),
             value: 'indexing',
           },
           {
-            label: () => $gettext('Not Indexed'),
-            value: 'false',
+            label: () => $gettext('Indexed'),
+            value: 'indexed',
+          },
+          {
+            label: () => $gettext('Ready'),
+            value: 'ready',
+          },
+          {
+            label: () => $gettext('Error'),
+            value: 'error',
+          },
+          {
+            label: () => $gettext('Partial'),
+            value: 'partial',
+          },
+          {
+            label: () => $gettext('Queued'),
+            value: 'queued',
           },
         ],
       },
@@ -187,28 +225,12 @@ const indexColumns: StdTableColumn[] = [
         }
       }
 
-      const tooltipContent = (
-        <div>
-          <div>{lastIndexed.format('YYYY-MM-DD HH:mm:ss')}</div>
-          {durationText && (
-            <div class="text-xs text-gray-100 dark:text-gray-300 mt-1">
-              { $gettext('Duration') }
-              :
-              {' '}
-              {durationText.slice(1, -1)}
-            </div>
-          )}
-        </div>
-      )
-
       return (
-        <Tooltip title={tooltipContent}>
-          <span>
-            {displayText}
-            {durationText && <span class="text-xs text-gray-500 dark:text-gray-400 ml-1">{durationText}</span>}
-            {statusIcon}
-          </span>
-        </Tooltip>
+        <span>
+          {displayText}
+          {durationText && <span class="text-xs text-gray-500 dark:text-gray-400 ml-1">{durationText}</span>}
+          {statusIcon}
+        </span>
       )
     },
     sorter: true,
@@ -238,38 +260,15 @@ const indexColumns: StdTableColumn[] = [
 
       const start = dayjs.unix(record.timerange_start)
       const end = dayjs.unix(record.timerange_end)
-      const duration = end.diff(start, 'day')
-
-      // Format duration display
-      let durationText = ''
-      if (duration === 0) {
-        durationText = $gettext('Today')
-      }
-      else if (duration === 1) {
-        durationText = '1 day'
-      }
-      else if (duration < 30) {
-        durationText = `${duration} days`
-      }
-      else if (duration < 365) {
-        const months = Math.floor(duration / 30)
-        durationText = `${months} month${months > 1 ? 's' : ''}`
-      }
-      else {
-        const years = Math.floor(duration / 365)
-        durationText = `${years} year${years > 1 ? 's' : ''}`
-      }
 
       return (
-        <Tooltip title={durationText}>
-          <span>
-            {start.format('YYYY-MM-DD HH:mm:ss')}
-            {' '}
-            ~
-            {' '}
-            {end.format('YYYY-MM-DD HH:mm:ss')}
-          </span>
-        </Tooltip>
+        <span>
+          {start.format('YYYY-MM-DD HH:mm:ss')}
+          {' '}
+          ~
+          {' '}
+          {end.format('YYYY-MM-DD HH:mm:ss')}
+        </span>
       )
     },
     width: 380,

app/src/views/nginx_log/components/LoadingState.vue

@@ -0,0 +1,156 @@
+<script setup lang="ts">
+import type { CSSProperties } from 'vue'
+import { LoadingOutlined } from '@ant-design/icons-vue'
+import { computed } from 'vue'
+import { useIndexProgress } from '../composables/useIndexProgress'
+import IndexProgressBar from '../indexing/components/IndexProgressBar.vue'
+
+interface IndexStatusDetails {
+  available: boolean
+  index_status: string
+  message?: string
+}
+
+const props = defineProps<{
+  logPath?: string
+  size?: 'small' | 'default' | 'large'
+  indexStatus?: IndexStatusDetails
+}>()
+
+// Index progress tracking
+const { getProgressForFile, isFileIndexing } = useIndexProgress()
+const indexProgress = computed(() => props.logPath ? getProgressForFile(props.logPath) : null)
+const isCurrentFileIndexing = computed(() => props.logPath ? isFileIndexing(props.logPath) : false)
+
+// Status-based loading message and icon
+const statusInfo = computed(() => {
+  const status = props.indexStatus?.index_status
+
+  switch (status) {
+    case 'indexing':
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Indexing logs...'),
+        color: 'text-blue-500',
+        showProgress: true,
+      }
+    case 'indexed':
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Loading...'),
+        color: 'text-blue-500',
+        showProgress: false,
+      }
+    case 'ready':
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Loading...'),
+        color: 'text-green-500',
+        showProgress: false,
+      }
+    case 'queued':
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Queued for indexing...'),
+        color: 'text-orange-500',
+        showProgress: false,
+      }
+    case 'partial':
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Partially indexed, resuming...'),
+        color: 'text-blue-500',
+        showProgress: true,
+      }
+    case 'error':
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Index failed, please try rebuilding'),
+        color: 'text-red-500',
+        showProgress: false,
+      }
+    case 'not_indexed':
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Log file not indexed yet'),
+        color: 'text-gray-500',
+        showProgress: false,
+      }
+    default:
+      // Fallback for active indexing check
+      if (isCurrentFileIndexing.value) {
+        return {
+          icon: LoadingOutlined,
+          message: $gettext('Indexing...'),
+          color: 'text-blue-500',
+          showProgress: true,
+        }
+      }
+      return {
+        icon: LoadingOutlined,
+        message: $gettext('Loading...'),
+        color: 'text-blue-500',
+        showProgress: false,
+      }
+  }
+})
+
+const iconClass = computed(() => {
+  const baseColor = statusInfo.value.color || 'text-blue-500'
+  switch (props.size) {
+    case 'small':
+      return `text-lg ${baseColor}`
+    case 'large':
+      return `text-4xl ${baseColor}`
+    default:
+      return `text-2xl ${baseColor}`
+  }
+})
+
+const containerStyle = computed((): CSSProperties => {
+  let height = '50vh' // Default responsive height
+  let padding = '40px'
+
+  switch (props.size) {
+    case 'small':
+      height = '30vh'
+      padding = '20px'
+      break
+    case 'large':
+      height = '70vh'
+      padding = '60px'
+      break
+  }
+
+  return {
+    minHeight: height,
+    padding,
+    display: 'flex',
+    flexDirection: 'column' as const,
+    justifyContent: 'center',
+    alignItems: 'center',
+  }
+})
+</script>
+
+<template>
+  <div :style="containerStyle">
+    <!-- Status Icon -->
+    <component :is="statusInfo.icon" :class="iconClass" />
+
+    <!-- Progress Bar (only show when actively indexing or if progress data exists) -->
+    <div v-if="(statusInfo.showProgress && indexProgress) || indexProgress" class="mt-4 flex flex-col items-center">
+      <div class="max-w-75 w-full">
+        <IndexProgressBar
+          :progress="indexProgress"
+          size="small"
+        />
+      </div>
+    </div>
+
+    <!-- Status Message -->
+    <p class="mt-4">
+      {{ statusInfo.message }}
+    </p>
+  </div>
+</template>

app/src/views/nginx_log/dashboard/DashboardViewer.vue

@@ -1,9 +1,9 @@
 <script setup lang="ts">
 import type { AnalyticsRequest, ChinaMapData, DashboardAnalytics, DashboardRequest, WorldMapData } from '@/api/nginx_log'
-import { LoadingOutlined } from '@ant-design/icons-vue'
 import { Col, Row } from 'ant-design-vue'
 import dayjs from 'dayjs'
 import nginx_log from '@/api/nginx_log'
+import LoadingState from '../components/LoadingState.vue'
 import BrowserStatsTable from './components/BrowserStatsTable.vue'
 import DailyTrendsChart from './components/DailyTrendsChart.vue'
 import DateRangeSelector from './components/DateRangeSelector.vue'
@@ -27,6 +27,7 @@ const dateRange = ref<[dayjs.Dayjs, dayjs.Dayjs]>([
   dayjs(),
 ])
 const timeRangeLoaded = ref(false)
+const hasValidTimeRange = ref(false)
 
 // Geographic data
 const worldMapData = ref<WorldMapData[] | null>(null)
@@ -44,24 +45,40 @@ async function loadTimeRange() {
   try {
     const preflight = await nginx_log.getPreflight(props.logPath)
 
-    if (preflight.available && preflight.start_time && preflight.end_time) {
+    if (preflight.time_range && preflight.time_range.start && preflight.time_range.end) {
       // Set start_date to 00:00:00 and end_date to 23:59:59
-      const endTime = dayjs.unix(preflight.end_time).endOf('day')
+      const endTime = dayjs.unix(preflight.time_range.end).endOf('day')
 
       // Use last week's data as default range (from last day back to 7 days ago)
       const weekStart = endTime.subtract(7, 'day').startOf('day')
       const lastDayEnd = endTime
       dateRange.value = [weekStart, lastDayEnd]
       timeRangeLoaded.value = true
-
-      // Time range loaded successfully
+      hasValidTimeRange.value = true
     }
     else {
-      console.warn(`No valid time range available for ${props.logPath}, using default range`)
+      // Still set timeRangeLoaded to true so we can proceed with loading
+      timeRangeLoaded.value = true
+      hasValidTimeRange.value = false
+
+      // Use default range (last 7 days from now)
+      dateRange.value = [
+        dayjs().subtract(7, 'day').startOf('day'),
+        dayjs().endOf('day'),
+      ]
     }
   }
   catch (error) {
     console.error('Failed to load time range from preflight:', error)
+    // Still set timeRangeLoaded to true so we don't get stuck in loading state
+    timeRangeLoaded.value = true
+    hasValidTimeRange.value = false
+
+    // Use default range
+    dateRange.value = [
+      dayjs().subtract(7, 'day').startOf('day'),
+      dayjs().endOf('day'),
+    ]
   }
 }
 
@@ -124,9 +141,9 @@ function refreshAllData() {
 watch(() => props.logPath, async () => {
   timeRangeLoaded.value = false
   const oldDateRange = dateRange.value
-  loadTimeRange()
+  await loadTimeRange()
 
-  // Only load dashboard data if dateRange didn't change (no automatic trigger)
+  // Load dashboard data if dateRange didn't change (no automatic trigger from watch below)
   if (timeRangeLoaded.value
     && oldDateRange[0].isSame(dateRange.value[0])
     && oldDateRange[1].isSame(dateRange.value[1])) {
@@ -144,18 +161,17 @@ watch(dateRange, () => {
 
 <template>
   <div class="dashboard-viewer">
-    <!-- Loading Spinner -->
-    <div v-if="loading" class="text-center" style="padding: 40px;">
-      <LoadingOutlined class="text-2xl text-blue-500" />
-      <p style="margin-top: 16px;">
-        {{ $gettext('Loading dashboard data...') }}
-      </p>
-    </div>
+    <!-- Loading State with Index Progress -->
+    <LoadingState
+      v-if="loading"
+      :log-path="logPath"
+    />
 
     <!-- Dashboard Content -->
     <div v-else>
-      <!-- Date Range Selector -->
+      <!-- Date Range Selector - only show when we have valid time range -->
       <DateRangeSelector
+        v-if="hasValidTimeRange"
         v-model:date-range="dateRange"
         :log-path="logPath"
         :refresh-loading="refreshLoading"

app/src/views/nginx_log/dashboard/components/BrowserStatsTable.vue

@@ -37,7 +37,7 @@ const browserColumns = [
     <Table
       v-if="dashboardData"
       :columns="browserColumns"
-      :data-source="dashboardData.browsers.slice(0, 10)"
+      :data-source="dashboardData?.browsers?.slice(0, 10) || []"
       :pagination="false"
       row-key="browser"
       size="small"

app/src/views/nginx_log/dashboard/components/DeviceStatsTable.vue

@@ -36,7 +36,7 @@ const deviceColumns = [
     <ATable
       v-if="dashboardData"
       :columns="deviceColumns"
-      :data-source="dashboardData.devices.slice(0, 10)"
+      :data-source="dashboardData?.devices?.slice(0, 10) || []"
       :pagination="false"
       row-key="device"
       size="small"

app/src/views/nginx_log/dashboard/components/OSStatsTable.vue

@@ -37,7 +37,7 @@ const osColumns = [
     <Table
       v-if="dashboardData"
       :columns="osColumns"
-      :data-source="dashboardData.operating_systems.slice(0, 10)"
+      :data-source="dashboardData?.operating_systems?.slice(0, 10) || []"
       :pagination="false"
       row-key="os"
       size="small"

app/src/views/nginx_log/dashboard/components/TopUrlsTable.vue

@@ -38,7 +38,7 @@ const urlColumns = [
     <Table
       v-if="dashboardData"
       :columns="urlColumns"
-      :data-source="dashboardData.top_urls"
+      :data-source="dashboardData?.top_urls || []"
       :pagination="false"
       row-key="url"
       size="small"

app/src/views/nginx_log/indexing/components/IndexProgressBar.vue

@@ -1,6 +1,4 @@
 <script setup lang="ts">
-import { Progress, Tag } from 'ant-design-vue'
-
 export interface IndexProgress {
   logPath: string
   progress: number
@@ -64,9 +62,9 @@ const stageText = computed(() => {
   <div v-if="progress" class="index-progress">
     <div class="progress-info">
       <div class="info-left">
-        <Tag :color="progressColor" size="small" class="stage-tag">
+        <ATag :color="progressColor" size="small" class="stage-tag">
           {{ stageText }}
-        </Tag>
+        </ATag>
       </div>
       <div class="info-right text-gray-600 dark:text-gray-400">
         <span class="time-elapsed text-gray-500 dark:text-gray-400">{{ formatTime(progress.elapsedTime) }}</span>
@@ -77,7 +75,7 @@ const stageText = computed(() => {
     </div>
 
     <div class="progress-container">
-      <Progress
+      <AProgress
         :percent="Math.round(progress.progress)"
         size="small"
         :stroke-color="progressColor"
@@ -95,6 +93,7 @@ const stageText = computed(() => {
 .index-progress {
   width: 100%;
   min-width: 200px;
+  max-width: 300px;
   padding: 2px 0;
 }
 

app/src/views/nginx_log/structured/StructuredLogViewer.vue

@@ -1,12 +1,13 @@
 <script setup lang="ts">
 import type { SorterResult, TablePaginationConfig } from 'ant-design-vue/es/table/interface'
 import type { AccessLogEntry, AdvancedSearchRequest, PreflightResponse } from '@/api/nginx_log'
-import { DownOutlined, ExclamationCircleOutlined, LoadingOutlined, ReloadOutlined } from '@ant-design/icons-vue'
+import { DownOutlined, ExclamationCircleOutlined, ReloadOutlined } from '@ant-design/icons-vue'
 import { message, Tag } from 'ant-design-vue'
 import dayjs from 'dayjs'
 import nginx_log from '@/api/nginx_log'
 import { useWebSocketEventBus } from '@/composables/useWebSocketEventBus'
 import { bytesToSize } from '@/lib/helper'
+import LoadingState from '../components/LoadingState.vue'
 import { useIndexProgress } from '../composables/useIndexProgress'
 import SearchFilters from './components/SearchFilters.vue'
 
@@ -384,12 +385,15 @@ async function loadPreflight(): Promise<boolean> {
   try {
     preflightResponse.value = await nginx_log.getPreflight(logPath.value)
 
-    if (preflightResponse.value.available) {
+    if (preflightResponse.value.available && preflightResponse.value.time_range) {
       // Cache this path as valid and set time range
       pathValidationCache.value.set(currentPath, true)
       // Set time range to full days: start_date 00:00:00 to end_date 23:59:59
-      timeRange.value.start = dayjs.unix(preflightResponse.value.start_time).startOf('day')
-      timeRange.value.end = dayjs.unix(preflightResponse.value.end_time).endOf('day')
+      const startTime = dayjs.unix(preflightResponse.value.time_range.start).startOf('day')
+      const endTime = dayjs.unix(preflightResponse.value.time_range.end).endOf('day')
+
+      timeRange.value.start = startTime
+      timeRange.value.end = endTime
       return true // Index is ready
     }
     else {
@@ -575,7 +579,7 @@ async function handleIndexReadyNotification(data: {
 }) {
   const currentPath = logPath.value || ''
   // Check if the notification is for the current log path
-  if (data.log_path === currentPath || (!currentPath && data.log_path)) {
+  if (data.log_path === currentPath) {
     message.success($gettext('Log indexing completed! Loading updated data...'))
 
     try {
@@ -734,29 +738,11 @@ watch(timeRange, () => {
         </div>
       </div>
 
-      <!-- Loading State -->
-      <div v-if="isLoading" class="text-center" style="padding: 40px;">
-        <LoadingOutlined class="text-2xl text-blue-500" />
-        <p style="margin-top: 16px;">
-          {{ $gettext('Searching logs...') }}
-        </p>
-      </div>
-
-      <!-- Indexing Status -->
-      <div v-else-if="shouldShowIndexingSpinner" class="text-center" style="padding: 40px;">
-        <LoadingOutlined class="text-2xl text-blue-500" />
-        <p style="margin-top: 16px;">
-          <template v-if="isCurrentFileIndexing">
-            {{ $gettext('This log file is currently being indexed, please wait...') }}
-          </template>
-          <template v-else-if="!isFileAvailable">
-            {{ $gettext('Waiting for this log file to be indexed...') }}
-          </template>
-          <template v-else>
-            {{ $gettext('Loading log data...') }}
-          </template>
-        </p>
-      </div>
+      <!-- Loading/Indexing State -->
+      <LoadingState
+        v-if="isLoading || shouldShowIndexingSpinner"
+        :log-path="logPath || ''"
+      />
 
       <!-- Search Results (show when indexing is ready and we have search results) -->
       <div v-else-if="shouldShowResults">

internal/cache/search.go

@@ -295,9 +295,6 @@ func (si *SearchIndexer) IndexDocument(doc SearchDocument) (err error) {
 	// Update memory usage tracking only for new documents
 	if isNewDocument {
 		si.updateMemoryUsage(doc.ID, contentSize, true)
-		logger.Debugf("Indexed new document: ID=%s, Type=%s, Name=%s", doc.ID, doc.Type, doc.Name)
-	} else {
-		logger.Debugf("Updated existing document: ID=%s, Type=%s, Name=%s", doc.ID, doc.Type, doc.Name)
 	}
 
 	return nil

internal/cron/cron.go

@@ -53,6 +53,12 @@ func InitCronJobs(ctx context.Context) {
 		logger.Fatalf("UpstreamAvailability Err: %v\n", err)
 	}
 
+	// Initialize incremental log indexing job
+	_, err = setupIncrementalIndexingJob(s)
+	if err != nil {
+		logger.Fatalf("IncrementalIndexing Err: %v\n", err)
+	}
+
 	// Start the scheduler
 	s.Start()
 

internal/cron/incremental_indexing.go

@@ -0,0 +1,208 @@
+package cron
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log"
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
+	"github.com/go-co-op/gocron/v2"
+	"github.com/uozi-tech/cosy/logger"
+)
+
+// setupIncrementalIndexingJob sets up the periodic incremental log indexing job
+func setupIncrementalIndexingJob(s gocron.Scheduler) (gocron.Job, error) {
+	logger.Info("Setting up incremental log indexing job")
+
+	// Run every 5 minutes to check for log file changes
+	job, err := s.NewJob(
+		gocron.DurationJob(5*time.Minute),
+		gocron.NewTask(performIncrementalIndexing),
+		gocron.WithName("incremental_log_indexing"),
+		gocron.WithStartAt(gocron.WithStartImmediately()),
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	logger.Info("Incremental log indexing job scheduled to run every 5 minutes")
+	return job, nil
+}
+
+// performIncrementalIndexing performs the actual incremental indexing check
+func performIncrementalIndexing() {
+	logger.Debug("Starting incremental log indexing scan")
+
+	// Get log file manager
+	logFileManager := nginx_log.GetLogFileManager()
+	if logFileManager == nil {
+		logger.Warn("Log file manager not available for incremental indexing")
+		return
+	}
+
+	// Get modern indexer
+	modernIndexer := nginx_log.GetModernIndexer()
+	if modernIndexer == nil {
+		logger.Warn("Modern indexer not available for incremental indexing")
+		return
+	}
+
+	// Check if indexer is healthy
+	if !modernIndexer.IsHealthy() {
+		logger.Warn("Modern indexer is not healthy, skipping incremental indexing")
+		return
+	}
+
+	// Get all log groups to check for changes
+	allLogs := nginx_log.GetAllLogsWithIndexGrouped(func(log *nginx_log.NginxLogWithIndex) bool {
+		// Only process access logs (skip error logs as they are not indexed)
+		return log.Type == "access"
+	})
+
+	changedCount := 0
+	for _, log := range allLogs {
+		// Check if file needs incremental indexing
+		if needsIncrementalIndexing(log) {
+			if err := queueIncrementalIndexing(log.Path, modernIndexer, logFileManager); err != nil {
+				logger.Errorf("Failed to queue incremental indexing for %s: %v", log.Path, err)
+			} else {
+				changedCount++
+			}
+		}
+	}
+
+	if changedCount > 0 {
+		logger.Infof("Queued %d log files for incremental indexing", changedCount)
+	} else {
+		logger.Debug("No log files need incremental indexing")
+	}
+}
+
+// needsIncrementalIndexing checks if a log file needs incremental indexing
+func needsIncrementalIndexing(log *nginx_log.NginxLogWithIndex) bool {
+	// Skip if already indexing or queued
+	if log.IndexStatus == string(indexer.IndexStatusIndexing) || 
+	   log.IndexStatus == string(indexer.IndexStatusQueued) {
+		return false
+	}
+
+	// Check file system status
+	fileInfo, err := os.Stat(log.Path)
+	if os.IsNotExist(err) {
+		// File doesn't exist, but we have index data - this is fine for historical queries
+		return false
+	}
+	if err != nil {
+		logger.Warnf("Cannot stat file %s: %v", log.Path, err)
+		return false
+	}
+
+	// Check if file has been modified since last index
+	fileModTime := fileInfo.ModTime()
+	fileSize := fileInfo.Size()
+	lastModified := time.Unix(log.LastModified, 0)
+
+	// File was modified after last index and size increased
+	if fileModTime.After(lastModified) && fileSize > log.LastSize {
+		logger.Debugf("File %s needs incremental indexing: mod_time=%s, size=%d", 
+			log.Path, fileModTime.Format("2006-01-02 15:04:05"), fileSize)
+		return true
+	}
+
+	// File size decreased - might be file rotation
+	if fileSize < log.LastSize {
+		logger.Debugf("File %s needs full re-indexing due to size decrease: old_size=%d, new_size=%d", 
+			log.Path, log.LastSize, fileSize)
+		return true
+	}
+
+	return false
+}
+
+// queueIncrementalIndexing queues a file for incremental indexing
+func queueIncrementalIndexing(logPath string, modernIndexer interface{}, logFileManager interface{}) error {
+	// Set the file status to queued
+	if err := setFileIndexStatus(logPath, string(indexer.IndexStatusQueued), logFileManager); err != nil {
+		return err
+	}
+
+	// Queue the indexing job asynchronously
+	go func() {
+		logger.Infof("Starting incremental indexing for file: %s", logPath)
+		
+		// Set status to indexing
+		if err := setFileIndexStatus(logPath, string(indexer.IndexStatusIndexing), logFileManager); err != nil {
+			logger.Errorf("Failed to set indexing status for %s: %v", logPath, err)
+			return
+		}
+
+		// Perform incremental indexing
+		startTime := time.Now()
+		docsCountMap, minTime, maxTime, err := modernIndexer.(*indexer.ParallelIndexer).IndexLogGroupWithProgress(logPath, nil)
+		
+		if err != nil {
+			logger.Errorf("Failed incremental indexing for %s: %v", logPath, err)
+			// Set error status
+			if statusErr := setFileIndexStatus(logPath, string(indexer.IndexStatusError), logFileManager); statusErr != nil {
+				logger.Errorf("Failed to set error status for %s: %v", logPath, statusErr)
+			}
+			return
+		}
+
+		// Calculate total documents indexed
+		var totalDocsIndexed uint64
+		for _, docCount := range docsCountMap {
+			totalDocsIndexed += docCount
+		}
+
+		// Save indexing metadata
+		duration := time.Since(startTime)
+		if metadataManager, ok := logFileManager.(indexer.MetadataManager); ok {
+			if err := metadataManager.SaveIndexMetadata(logPath, totalDocsIndexed, startTime, duration, minTime, maxTime); err != nil {
+				logger.Errorf("Failed to save incremental index metadata for %s: %v", logPath, err)
+			}
+		}
+
+		// Set status to indexed
+		if err := setFileIndexStatus(logPath, string(indexer.IndexStatusIndexed), logFileManager); err != nil {
+			logger.Errorf("Failed to set indexed status for %s: %v", logPath, err)
+		}
+
+		// Update searcher shards
+		nginx_log.UpdateSearcherShards()
+
+		logger.Infof("Successfully completed incremental indexing for %s, Documents: %d", logPath, totalDocsIndexed)
+	}()
+
+	return nil
+}
+
+// setFileIndexStatus updates the index status for a file in the database using enhanced status management
+func setFileIndexStatus(logPath, status string, logFileManager interface{}) error {
+	if logFileManager == nil {
+		return fmt.Errorf("log file manager not available")
+	}
+	
+	// Get persistence manager
+	lfm, ok := logFileManager.(*indexer.LogFileManager)
+	if !ok {
+		return fmt.Errorf("invalid log file manager type")
+	}
+	
+	persistence := lfm.GetPersistence()
+	if persistence == nil {
+		return fmt.Errorf("persistence manager not available")
+	}
+	
+	// Use enhanced SetIndexStatus method with queue position for queued status
+	queuePosition := 0
+	if status == string(indexer.IndexStatusQueued) {
+		// For incremental indexing, we don't need specific queue positions
+		// They will be processed as they come
+		queuePosition = int(time.Now().Unix() % 1000) // Simple ordering by time
+	}
+	
+	return persistence.SetIndexStatus(logPath, status, queuePosition, "")
+}

internal/kernel/boot.go

@@ -90,6 +90,7 @@ func InitAfterDatabase(ctx context.Context) {
 		mcp.Init,
 		sitecheck.Init,
 		nginx_log.InitializeModernServices,
+		nginx_log.InitTaskRecovery,
 		user.InitTokenCache,
 	}
 

internal/nginx_log/PERFORMANCE_REPORT.md

@@ -104,15 +104,45 @@ This report presents the latest benchmark results for the nginx-ui log processin
 
 ## 📈 Real-World Impact
 
-### High-Volume Log Processing (estimated)
-- **Indexing throughput**: ~20% improvement in document processing
-- **Search performance**: ~15% faster query execution  
-- **Memory usage**: ~30% reduction in allocation rate
+### High-Volume Log Processing (Measured 1.2M Records)
+- **Indexing throughput**: **3,860 records/second** sustained performance
+- **Total processing time**: **5 minutes 11 seconds** for 1.2M records
+- **Index creation**: 4 distributed shards with perfect distribution (300K records each)
+- **Search performance**: Successfully executing analytics queries on complete dataset
+- **Memory usage**: ~30% reduction in allocation rate from optimizations
 - **Concurrent safety**: 100% thread-safe operations
 
+### Detailed Performance Breakdown
+| File | Records | Processing Time | Rate (records/sec) |
+|------|---------|----------------|-------------------|
+| access_2.log | 400,000 | 1m 44s | 3,800 |
+| access_3.log | 400,000 | 1m 40s | 4,000 |
+| access_1.log | 400,000 | 1m 46s | 3,750 |
+| **Total** | **1,200,000** | **5m 11s** | **3,860** |
+
+### Real-World Test Environment
+- **Hardware**: Apple M2 Pro (ARM64)
+- **Test Date**: August 30, 2025  
+- **Dataset**: 1.2M synthetic nginx access log records
+- **Processing**: Full-text indexing with GeoIP, User-Agent parsing
+- **Result**: 4 Bleve search shards with 1.2M searchable documents
+
+### Production Scaling Estimates
+
+Based on the measured **3,860 records/second** performance:
+
+| Daily Log Volume | Processing Time | Recommended Hardware |
+|------------------|----------------|---------------------|
+| 1M records/day | ~4.3 minutes | Single M2 Pro sufficient |
+| 10M records/day | ~43 minutes | Single M2 Pro sufficient |
+| 100M records/day | ~7.2 hours | Multi-core server recommended |
+| 1B records/day | ~3 days | Distributed processing needed |
+
+**Memory Requirements**: ~800MB RAM per 1M indexed records (including search indices)
+
 ### Critical Path Optimizations
 1. **Document ID Generation**: Used in every indexed log entry
-2. **Cache Key Generation**: Used for every search query
+2. **Cache Key Generation**: Used for every search query  
 3. **String Interning**: Reduces memory for repeated values
 4. **Progress Tracking**: Zero-allocation status updates
 
@@ -164,13 +194,18 @@ This report presents the latest benchmark results for the nginx-ui log processin
 
 The performance optimizations have delivered significant improvements across all nginx-log processing components:
 
-- **Ultra-fast string operations** with zero allocations
+- **Ultra-fast string operations** with zero allocations  
 - **Highly efficient caching** with proper concurrency control
 - **Reduced memory pressure** through intelligent pooling
-- **Maintained functionality** while achieving 20-1900x performance gains
+- **Real-world performance**: **3,860 records/second** sustained throughput
+- **Production ready**: Successfully processes 1.2M records in 5 minutes
+- **Maintained functionality** while achieving 20-1900x performance gains in micro-benchmarks
+
+### Key Achievement
+🚀 **Proven at scale**: The optimized nginx-ui log processing system successfully indexed and made searchable **1.2 million log records** in just **5 minutes and 11 seconds**, demonstrating production-ready performance for high-volume enterprise workloads.
 
-These optimizations ensure the nginx-ui log processing system can handle high-volume production workloads with minimal resource consumption and maximum throughput.
+These optimizations ensure the nginx-ui log processing system can handle substantial production workloads with minimal resource consumption and maximum throughput, making it suitable for environments processing millions of log records daily.
 
 ---
 
-*Report generated after successful integration of unified performance utils package*
+*Report updated with real-world performance measurements from 1.2M record integration test*

internal/nginx_log/indexer/log_file_manager.go

@@ -2,6 +2,7 @@ package indexer
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"regexp"
 	"sort"
@@ -13,12 +14,7 @@ import (
 	"github.com/uozi-tech/cosy/logger"
 )
 
-// IndexStatus constants
-const (
-	IndexStatusIndexed    = "indexed"
-	IndexStatusIndexing   = "indexing"
-	IndexStatusNotIndexed = "not_indexed"
-)
+// Legacy constants for backward compatibility - use IndexStatus enum in types.go instead
 
 // NginxLogCache represents a cached log entry from nginx configuration
 type NginxLogCache struct {
@@ -190,7 +186,7 @@ func (lm *LogFileManager) GetAllLogsWithIndexGrouped(filters ...func(*NginxLogWi
 			Type:         cache.Type,
 			Name:         cache.Name,
 			ConfigFile:   cache.ConfigFile,
-			IndexStatus:  IndexStatusNotIndexed,
+			IndexStatus:  string(IndexStatusNotIndexed),
 			IsCompressed: false,
 			HasTimeRange: false,
 		}
@@ -204,14 +200,6 @@ func (lm *LogFileManager) GetAllLogsWithIndexGrouped(filters ...func(*NginxLogWi
 		persistenceIndexes = []*model.NginxLogIndex{}
 	}
 
-	// --- START DIAGNOSTIC LOGGING ---
-	logger.Debugf("===== DB STATE BEFORE GROUPING =====")
-	for _, pIdx := range persistenceIndexes {
-		logger.Debugf("DB Record: Path=%s, MainLogPath=%s, DocCount=%d, LastIndexed=%s", pIdx.Path, pIdx.MainLogPath, pIdx.DocumentCount, pIdx.LastIndexed)
-	}
-	logger.Debugf("===================================")
-	// --- END DIAGNOSTIC LOGGING ---
-
 	// Add all indexed files from persistence (including rotated files)
 	for _, idx := range persistenceIndexes {
 		if _, exists := allLogsMap[idx.Path]; !exists {
@@ -226,7 +214,7 @@ func (lm *LogFileManager) GetAllLogsWithIndexGrouped(filters ...func(*NginxLogWi
 				Type:        logType,
 				Name:        filepath.Base(idx.Path),
 				ConfigFile:  "",
-				IndexStatus: IndexStatusNotIndexed,
+				IndexStatus: string(IndexStatusNotIndexed),
 			}
 			allLogsMap[idx.Path] = logWithIndex
 		}
@@ -236,7 +224,9 @@ func (lm *LogFileManager) GetAllLogsWithIndexGrouped(filters ...func(*NginxLogWi
 		logWithIndex.LastModified = idx.LastModified.Unix()
 		logWithIndex.LastSize = idx.LastSize
 		logWithIndex.LastIndexed = idx.LastIndexed.Unix()
-		logWithIndex.IndexStartTime = idx.IndexStartTime.Unix()
+		if idx.IndexStartTime != nil {
+			logWithIndex.IndexStartTime = idx.IndexStartTime.Unix()
+		}
 		if idx.IndexDuration != nil {
 			logWithIndex.IndexDuration = *idx.IndexDuration
 		}
@@ -248,9 +238,10 @@ func (lm *LogFileManager) GetAllLogsWithIndexGrouped(filters ...func(*NginxLogWi
 		lm.indexingMutex.RUnlock()
 
 		if isIndexing {
-			logWithIndex.IndexStatus = IndexStatusIndexing
-		} else if idx.DocumentCount > 0 {
-			logWithIndex.IndexStatus = IndexStatusIndexed
+			logWithIndex.IndexStatus = string(IndexStatusIndexing)
+		} else if !idx.LastIndexed.IsZero() {
+			// If file has been indexed (regardless of document count), it's indexed
+			logWithIndex.IndexStatus = string(IndexStatusIndexed)
 		}
 
 		// Set time range if available
@@ -292,41 +283,63 @@ func (lm *LogFileManager) GetAllLogsWithIndexGrouped(filters ...func(*NginxLogWi
 		baseLogName := getBaseLogName(log.Path)
 
 		if existing, exists := groupedMap[baseLogName]; exists {
-			// Merge with existing entry using consistent rules
-			// Always use the most recent data for single-value fields
-			if log.LastIndexed > existing.LastIndexed {
-				existing.LastModified = log.LastModified
-				existing.LastIndexed = log.LastIndexed
-				existing.IndexStartTime = log.IndexStartTime
-				existing.IndexDuration = log.IndexDuration
-			}
+			// Check if current log is a main log path record (already aggregated)
+			// or if existing record is a main log path record
+			logIsMainPath := (log.Path == baseLogName)
+			existingIsMainPath := (existing.Path == baseLogName)
+			
+			if logIsMainPath && !existingIsMainPath {
+				// Current log is the main aggregated record, replace existing
+				groupedLog := *log
+				groupedLog.Path = baseLogName
+				groupedLog.Name = filepath.Base(baseLogName)
+				groupedMap[baseLogName] = &groupedLog
+			} else if !logIsMainPath && existingIsMainPath {
+				// Existing is main record, keep it, don't accumulate
+				// Only update status if needed
+				if log.IndexStatus == string(IndexStatusIndexing) {
+					existing.IndexStatus = string(IndexStatusIndexing)
+				}
+			} else if !logIsMainPath && !existingIsMainPath {
+				// Both are individual files, accumulate normally
+				if log.LastIndexed > existing.LastIndexed {
+					existing.LastModified = log.LastModified
+					existing.LastIndexed = log.LastIndexed
+					existing.IndexStartTime = log.IndexStartTime
+					existing.IndexDuration = log.IndexDuration
+				}
 
-			// Accumulate countable metrics
-			existing.DocumentCount += log.DocumentCount
-			existing.LastSize += log.LastSize
+				existing.DocumentCount += log.DocumentCount
+				existing.LastSize += log.LastSize
 
-			// Update status to most significant (indexing > indexed > not_indexed)
-			if log.IndexStatus == IndexStatusIndexing {
-				existing.IndexStatus = IndexStatusIndexing
-			} else if log.IndexStatus == IndexStatusIndexed && existing.IndexStatus != IndexStatusIndexing {
-				existing.IndexStatus = IndexStatusIndexed
-			}
+				if log.IndexStatus == string(IndexStatusIndexing) {
+					existing.IndexStatus = string(IndexStatusIndexing)
+				} else if log.IndexStatus == string(IndexStatusIndexed) && existing.IndexStatus != string(IndexStatusIndexing) {
+					existing.IndexStatus = string(IndexStatusIndexed)
+				}
 
-			// Expand time range to encompass both (deterministic expansion)
-			if log.HasTimeRange {
-				if !existing.HasTimeRange {
-					existing.HasTimeRange = true
-					existing.TimeRangeStart = log.TimeRangeStart
-					existing.TimeRangeEnd = log.TimeRangeEnd
-				} else {
-					// Always expand range consistently
-					if log.TimeRangeStart > 0 && (existing.TimeRangeStart == 0 || log.TimeRangeStart < existing.TimeRangeStart) {
+				if log.HasTimeRange {
+					if !existing.HasTimeRange {
+						existing.HasTimeRange = true
 						existing.TimeRangeStart = log.TimeRangeStart
-					}
-					if log.TimeRangeEnd > existing.TimeRangeEnd {
 						existing.TimeRangeEnd = log.TimeRangeEnd
+					} else {
+						if log.TimeRangeStart > 0 && (existing.TimeRangeStart == 0 || log.TimeRangeStart < existing.TimeRangeStart) {
+							existing.TimeRangeStart = log.TimeRangeStart
+						}
+						if log.TimeRangeEnd > existing.TimeRangeEnd {
+							existing.TimeRangeEnd = log.TimeRangeEnd
+						}
 					}
 				}
+			} else if logIsMainPath && existingIsMainPath {
+				// If both are main paths, use the one with more recent LastIndexed
+				if log.LastIndexed > existing.LastIndexed {
+					groupedLog := *log
+					groupedLog.Path = baseLogName
+					groupedLog.Name = filepath.Base(baseLogName)
+					groupedMap[baseLogName] = &groupedLog
+				}
 			}
 		} else {
 			// Create new entry with base log name as path for grouping
@@ -375,6 +388,12 @@ func (lm *LogFileManager) SaveIndexMetadata(basePath string, documentCount uint6
 		return fmt.Errorf("could not get or create log index for '%s': %w", basePath, err)
 	}
 
+	// Get file stats to update LastModified and LastSize
+	if fileInfo, err := os.Stat(basePath); err == nil {
+		logIndex.LastModified = fileInfo.ModTime()
+		logIndex.LastSize = fileInfo.Size()
+	}
+
 	// Update the record with the new metadata
 	logIndex.DocumentCount = documentCount
 	logIndex.LastIndexed = time.Now()
@@ -431,6 +450,11 @@ func (lm *LogFileManager) GetFilePathsForGroup(basePath string) ([]string, error
 	return filePaths, nil
 }
 
+// GetPersistence returns the persistence manager for advanced operations
+func (lm *LogFileManager) GetPersistence() *PersistenceManager {
+	return lm.persistence
+}
+
 // maxInt64 returns the maximum of two int64 values
 func maxInt64(a, b int64) int64 {
 	if a > b {
@@ -466,7 +490,7 @@ func (lm *LogFileManager) GetAllLogsWithIndex(filters ...func(*NginxLogWithIndex
 			Type:         cache.Type,
 			Name:         cache.Name,
 			ConfigFile:   cache.ConfigFile,
-			IndexStatus:  IndexStatusNotIndexed,
+			IndexStatus:  string(IndexStatusNotIndexed),
 			IsCompressed: strings.HasSuffix(cache.Path, ".gz") || strings.HasSuffix(cache.Path, ".bz2"),
 		}
 
@@ -475,7 +499,9 @@ func (lm *LogFileManager) GetAllLogsWithIndex(filters ...func(*NginxLogWithIndex
 			logWithIndex.LastModified = idx.LastModified.Unix()
 			logWithIndex.LastSize = idx.LastSize
 			logWithIndex.LastIndexed = idx.LastIndexed.Unix()
-			logWithIndex.IndexStartTime = idx.IndexStartTime.Unix()
+			if idx.IndexStartTime != nil {
+				logWithIndex.IndexStartTime = idx.IndexStartTime.Unix()
+			}
 			if idx.IndexDuration != nil {
 				logWithIndex.IndexDuration = *idx.IndexDuration
 			}
@@ -487,9 +513,10 @@ func (lm *LogFileManager) GetAllLogsWithIndex(filters ...func(*NginxLogWithIndex
 			lm.indexingMutex.RUnlock()
 
 			if isIndexing {
-				logWithIndex.IndexStatus = IndexStatusIndexing
-			} else if idx.DocumentCount > 0 {
-				logWithIndex.IndexStatus = IndexStatusIndexed
+				logWithIndex.IndexStatus = string(IndexStatusIndexing)
+			} else if !idx.LastIndexed.IsZero() {
+				// If file has been indexed (regardless of document count), it's indexed
+				logWithIndex.IndexStatus = string(IndexStatusIndexed)
 			}
 
 			// Set time range if available

internal/nginx_log/indexer/parallel_indexer.go

@@ -486,9 +486,7 @@ func (pi *ParallelIndexer) DeleteIndexByLogGroup(basePath string, logFileManager
 		return fmt.Errorf("log file manager is required")
 	}
 
-	lfm, ok := logFileManager.(interface {
-		GetFilePathsForGroup(string) ([]string, error)
-	})
+	lfm, ok := logFileManager.(GroupFileProvider)
 	if !ok {
 		return fmt.Errorf("log file manager does not support GetFilePathsForGroup")
 	}

internal/nginx_log/indexer/persistence.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/0xJacky/Nginx-UI/internal/event"
 	"github.com/0xJacky/Nginx-UI/model"
 	"github.com/0xJacky/Nginx-UI/query"
 	"github.com/uozi-tech/cosy"
@@ -365,6 +366,145 @@ func (pm *PersistenceManager) SaveLogFileInfo(path string, info *LogFileInfo) er
 	return pm.UpdateIncrementalInfo(path, info)
 }
 
+// SetIndexStatus updates the index status for a specific file path with enhanced status support
+func (pm *PersistenceManager) SetIndexStatus(path, status string, queuePosition int, errorMessage string) error {
+	logIndex, err := pm.GetLogIndex(path)
+	if err != nil {
+		return fmt.Errorf("failed to get log index for status update: %w", err)
+	}
+
+	// Update status based on the new status
+	switch status {
+	case string(IndexStatusQueued):
+		logIndex.SetQueuedStatus(queuePosition)
+	case string(IndexStatusIndexing):
+		logIndex.SetIndexingStatus(status)
+	case string(IndexStatusIndexed):
+		logIndex.SetCompletedStatus()
+	case string(IndexStatusReady):
+		logIndex.IndexStatus = string(IndexStatusReady)
+		logIndex.ErrorMessage = ""
+		logIndex.ErrorTime = nil
+	case string(IndexStatusError):
+		logIndex.SetErrorStatus(errorMessage)
+	case string(IndexStatusPartial):
+		logIndex.IndexStatus = string(IndexStatusPartial)
+	default:
+		logIndex.IndexStatus = status
+	}
+
+	err = pm.SaveLogIndex(logIndex)
+	if err != nil {
+		return err
+	}
+
+	// Broadcast status change event to frontend
+	event.Publish(event.Event{
+		Type: event.TypeNginxLogIndexProgress,
+		Data: event.NginxLogIndexProgressData{
+			LogPath: path,
+			Status:  status,
+			Stage:   "status_update",
+		},
+	})
+
+	return nil
+}
+
+// GetIncompleteIndexingTasks returns all files that have incomplete indexing tasks
+func (pm *PersistenceManager) GetIncompleteIndexingTasks() ([]*model.NginxLogIndex, error) {
+	// Use direct database query since query fields are not generated yet
+	db := cosy.UseDB(context.Background())
+	var indexes []*model.NginxLogIndex
+	
+	err := db.Where("enabled = ? AND index_status IN ?", true, []string{
+		string(IndexStatusIndexing),
+		string(IndexStatusQueued),
+		string(IndexStatusPartial),
+	}).Order("queue_position").Find(&indexes).Error
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to get incomplete indexing tasks: %w", err)
+	}
+
+	return indexes, nil
+}
+
+// GetQueuedTasks returns all queued indexing tasks ordered by queue position
+func (pm *PersistenceManager) GetQueuedTasks() ([]*model.NginxLogIndex, error) {
+	// Use direct database query since query fields are not generated yet
+	db := cosy.UseDB(context.Background())
+	var indexes []*model.NginxLogIndex
+	
+	err := db.Where("enabled = ? AND index_status = ?", true, string(IndexStatusQueued)).Order("queue_position").Find(&indexes).Error
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to get queued tasks: %w", err)
+	}
+
+	return indexes, nil
+}
+
+// ResetIndexingTasks resets all indexing and queued tasks to not_indexed state
+// This is useful during startup to clear stale states
+func (pm *PersistenceManager) ResetIndexingTasks() error {
+	// Use direct database query
+	db := cosy.UseDB(context.Background())
+	
+	err := db.Model(&model.NginxLogIndex{}).Where("index_status IN ?", []string{
+		string(IndexStatusIndexing),
+		string(IndexStatusQueued),
+	}).Updates(map[string]interface{}{
+		"index_status":    string(IndexStatusNotIndexed),
+		"queue_position":  0,
+		"partial_offset":  0,
+		"error_message":   "",
+		"error_time":      nil,
+		"index_start_time": nil,
+	}).Error
+
+	if err != nil {
+		return fmt.Errorf("failed to reset indexing tasks: %w", err)
+	}
+
+	// Clear cache
+	pm.enabledPaths = make(map[string]bool)
+	
+	logger.Info("Reset all incomplete indexing tasks")
+	return nil
+}
+
+// GetIndexingTaskStats returns statistics about indexing tasks
+func (pm *PersistenceManager) GetIndexingTaskStats() (map[string]int64, error) {
+	// Use direct database query
+	db := cosy.UseDB(context.Background())
+	stats := make(map[string]int64)
+	
+	// Count by status
+	statuses := []string{
+		string(IndexStatusNotIndexed),
+		string(IndexStatusIndexing),
+		string(IndexStatusIndexed),
+		string(IndexStatusQueued),
+		string(IndexStatusError),
+		string(IndexStatusPartial),
+		string(IndexStatusReady),
+	}
+	
+	for _, status := range statuses {
+		var count int64
+		err := db.Model(&model.NginxLogIndex{}).Where("enabled = ? AND index_status = ?", true, status).Count(&count).Error
+		
+		if err != nil {
+			return nil, fmt.Errorf("failed to count status %s: %w", status, err)
+		}
+		
+		stats[status] = count
+	}
+	
+	return stats, nil
+}
+
 // Close flushes any pending operations and cleans up resources
 func (pm *PersistenceManager) Close() error {
 	// Flush any pending operations

internal/nginx_log/indexer/types.go

@@ -8,6 +8,43 @@ import (
 	"github.com/blevesearch/bleve/v2/mapping"
 )
 
+// IndexStatus represents different states of log indexing
+type IndexStatus string
+
+// Index status constants
+const (
+	IndexStatusNotIndexed IndexStatus = "not_indexed" // File not indexed
+	IndexStatusIndexing   IndexStatus = "indexing"    // Currently being indexed
+	IndexStatusIndexed    IndexStatus = "indexed"     // Successfully indexed
+	IndexStatusError      IndexStatus = "error"       // Index failed with error
+	IndexStatusPartial    IndexStatus = "partial"     // Partially indexed (large file)
+	IndexStatusQueued     IndexStatus = "queued"      // Waiting in queue
+	IndexStatusReady      IndexStatus = "ready"       // Ready for search (alias for indexed)
+)
+
+// IndexStatusDetails contains detailed status information
+type IndexStatusDetails struct {
+	Status        IndexStatus `json:"status"`
+	Message       string      `json:"message,omitempty"`
+	ErrorMessage  string      `json:"error_message,omitempty"`
+	ErrorTime     *time.Time  `json:"error_time,omitempty"`
+	RetryCount    int         `json:"retry_count,omitempty"`
+	QueuePosition int         `json:"queue_position,omitempty"`
+	PartialOffset int64       `json:"partial_offset,omitempty"`
+	Progress      *IndexProgress `json:"progress,omitempty"`
+}
+
+// IndexProgress contains indexing progress information
+type IndexProgress struct {
+	Percent        float64 `json:"percent"`
+	ProcessedLines int64   `json:"processed_lines"`
+	TotalLines     int64   `json:"total_lines"`
+	ProcessedBytes int64   `json:"processed_bytes"`
+	TotalBytes     int64   `json:"total_bytes"`
+	Speed          int64   `json:"speed"` // lines per second
+	ETA            int64   `json:"eta"`   // estimated time to completion in seconds
+}
+
 // IndexerConfig holds configuration for the indexer
 type Config struct {
 	IndexPath         string        `json:"index_path"`
@@ -338,3 +375,33 @@ var (
 	ErrInvalidDocument    = "invalid document"
 	ErrOptimizationFailed = "optimization failed"
 )
+
+// MetadataManager defines the interface for managing log index metadata
+type MetadataManager interface {
+	// SaveIndexMetadata saves metadata for a log group after indexing
+	SaveIndexMetadata(basePath string, documentCount uint64, startTime time.Time, duration time.Duration, minTime *time.Time, maxTime *time.Time) error
+	// DeleteIndexMetadataByGroup deletes all database records for a log group
+	DeleteIndexMetadataByGroup(basePath string) error
+	// DeleteAllIndexMetadata deletes all index metadata from the database
+	DeleteAllIndexMetadata() error
+	// GetFilePathsForGroup returns all physical file paths for a given log group
+	GetFilePathsForGroup(basePath string) ([]string, error)
+}
+
+// GroupFileProvider defines the interface for getting file paths for a log group
+type GroupFileProvider interface {
+	// GetFilePathsForGroup returns all physical file paths for a given log group
+	GetFilePathsForGroup(basePath string) ([]string, error)
+}
+
+// FlushableIndexer defines the interface for indexers that can be flushed
+type FlushableIndexer interface {
+	// FlushAll flushes all pending operations
+	FlushAll() error
+}
+
+// RestartableIndexer defines the interface for indexers that can be restarted
+type RestartableIndexer interface {
+	// Start begins the indexer operation
+	Start(context.Context) error
+}

internal/nginx_log/integration_small_test.go

@@ -0,0 +1,470 @@
+package nginx_log
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/analytics"
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/searcher"
+	"github.com/blevesearch/bleve/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	// Small test configuration for faster execution
+	SmallTestRecordsPerFile = 1000  // 1000条记录每个文件
+	SmallTestFileCount      = 3     // 3个测试文件
+)
+
+// SmallIntegrationTestSuite contains all integration test data and services (small version)
+type SmallIntegrationTestSuite struct {
+	ctx              context.Context
+	cancel           context.CancelFunc
+	tempDir          string
+	indexDir         string
+	logFiles         []string
+	logFilePaths     []string
+	indexer          *indexer.ParallelIndexer
+	searcher         searcher.Searcher
+	analytics        analytics.Service
+	logFileManager   *TestLogFileManager
+	expectedMetrics  map[string]*SmallExpectedFileMetrics
+	mu               sync.RWMutex
+	cleanup          func()
+}
+
+// SmallExpectedFileMetrics stores expected statistics for each log file (small version)
+type SmallExpectedFileMetrics struct {
+	TotalRecords  uint64
+	UniqueIPs     uint64
+	UniquePaths   uint64
+	UniqueAgents  uint64
+	StatusCodes   map[int]uint64
+	Methods       map[string]uint64
+	TimeRange     SmallTestTimeRange
+}
+
+// SmallTestTimeRange represents the time range of log entries for small testing
+type SmallTestTimeRange struct {
+	StartTime time.Time
+	EndTime   time.Time
+}
+
+// NewSmallIntegrationTestSuite creates a new small integration test suite
+func NewSmallIntegrationTestSuite(t *testing.T) *SmallIntegrationTestSuite {
+	ctx, cancel := context.WithCancel(context.Background())
+	
+	// Create temporary directories
+	tempDir, err := os.MkdirTemp("", "nginx_ui_small_integration_test_*")
+	require.NoError(t, err)
+	
+	indexDir := filepath.Join(tempDir, "index")
+	logsDir := filepath.Join(tempDir, "logs")
+	
+	err = os.MkdirAll(indexDir, 0755)
+	require.NoError(t, err)
+	
+	err = os.MkdirAll(logsDir, 0755)
+	require.NoError(t, err)
+
+	suite := &SmallIntegrationTestSuite{
+		ctx:             ctx,
+		cancel:          cancel,
+		tempDir:         tempDir,
+		indexDir:        indexDir,
+		expectedMetrics: make(map[string]*SmallExpectedFileMetrics),
+	}
+
+	// Set cleanup function
+	suite.cleanup = func() {
+		// Stop services
+		if suite.indexer != nil {
+			suite.indexer.Stop()
+		}
+		if suite.searcher != nil {
+			suite.searcher.Stop()
+		}
+		
+		// Cancel context
+		cancel()
+		
+		// Remove temporary directories
+		os.RemoveAll(tempDir)
+	}
+
+	return suite
+}
+
+// GenerateSmallTestData generates the small test log files with expected statistics
+func (suite *SmallIntegrationTestSuite) GenerateSmallTestData(t *testing.T) {
+	t.Logf("Generating %d test files with %d records each", SmallTestFileCount, SmallTestRecordsPerFile)
+	
+	baseTime := time.Now().Add(-24 * time.Hour)
+	
+	for i := 0; i < SmallTestFileCount; i++ {
+		filename := fmt.Sprintf("small_access_%d.log", i+1)
+		filepath := filepath.Join(suite.tempDir, "logs", filename)
+		
+		metrics := suite.generateSmallSingleLogFile(t, filepath, baseTime.Add(time.Duration(i)*time.Hour))
+		
+		suite.logFiles = append(suite.logFiles, filename)
+		suite.logFilePaths = append(suite.logFilePaths, filepath)
+		suite.expectedMetrics[filepath] = metrics
+		
+		t.Logf("Generated %s with %d records", filename, metrics.TotalRecords)
+	}
+	
+	t.Logf("Small test data generation completed. Total files: %d", len(suite.logFiles))
+}
+
+// generateSmallSingleLogFile generates a single small log file with known statistics
+func (suite *SmallIntegrationTestSuite) generateSmallSingleLogFile(t *testing.T, filepath string, baseTime time.Time) *SmallExpectedFileMetrics {
+	file, err := os.Create(filepath)
+	require.NoError(t, err)
+	defer file.Close()
+
+	metrics := &SmallExpectedFileMetrics{
+		StatusCodes: make(map[int]uint64),
+		Methods:     make(map[string]uint64),
+		TimeRange: SmallTestTimeRange{
+			StartTime: baseTime,
+			EndTime:   baseTime.Add(time.Duration(SmallTestRecordsPerFile) * time.Second),
+		},
+	}
+
+	// Predefined test data for consistent testing
+	ips := []string{
+		"192.168.1.1", "192.168.1.2", "192.168.1.3", "10.0.0.1", "10.0.0.2",
+	}
+	
+	paths := []string{
+		"/", "/api/v1/status", "/api/v1/logs", "/admin", "/login",
+	}
+	
+	userAgents := []string{
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+		"PostmanRuntime/7.28.4",
+	}
+	
+	statusCodes := []int{200, 301, 404, 500}
+	methods := []string{"GET", "POST", "PUT"}
+
+	// Track unique values
+	uniqueIPs := make(map[string]bool)
+	uniquePaths := make(map[string]bool)
+	uniqueAgents := make(map[string]bool)
+
+	rand.Seed(time.Now().UnixNano() + int64(len(filepath))) // Different seed per file
+
+	for i := 0; i < SmallTestRecordsPerFile; i++ {
+		// Generate log entry timestamp
+		timestamp := baseTime.Add(time.Duration(i) * time.Second)
+		
+		// Select random values
+		ip := ips[rand.Intn(len(ips))]
+		path := paths[rand.Intn(len(paths))]
+		agent := userAgents[rand.Intn(len(userAgents))]
+		status := statusCodes[rand.Intn(len(statusCodes))]
+		method := methods[rand.Intn(len(methods))]
+		size := rand.Intn(1000) + 100 // 100-1100 bytes
+		
+		// Track unique values
+		uniqueIPs[ip] = true
+		uniquePaths[path] = true
+		uniqueAgents[agent] = true
+		
+		// Update metrics
+		metrics.StatusCodes[status]++
+		metrics.Methods[method]++
+		
+		// Generate nginx log line (Common Log Format)
+		logLine := fmt.Sprintf(`%s - - [%s] "%s %s HTTP/1.1" %d %d "-" "%s"`+"\n",
+			ip,
+			timestamp.Format("02/Jan/2006:15:04:05 -0700"),
+			method,
+			path,
+			status,
+			size,
+			agent,
+		)
+		
+		_, err := file.WriteString(logLine)
+		require.NoError(t, err)
+	}
+
+	// Finalize metrics
+	metrics.TotalRecords = SmallTestRecordsPerFile
+	metrics.UniqueIPs = uint64(len(uniqueIPs))
+	metrics.UniquePaths = uint64(len(uniquePaths))
+	metrics.UniqueAgents = uint64(len(uniqueAgents))
+
+	return metrics
+}
+
+// InitializeSmallServices initializes all nginx_log services for small testing
+func (suite *SmallIntegrationTestSuite) InitializeSmallServices(t *testing.T) {
+	t.Log("Initializing small test services...")
+	
+	// Initialize indexer
+	indexerConfig := indexer.DefaultIndexerConfig()
+	indexerConfig.IndexPath = suite.indexDir
+	shardManager := indexer.NewDefaultShardManager(indexerConfig)
+	suite.indexer = indexer.NewParallelIndexer(indexerConfig, shardManager)
+	
+	err := suite.indexer.Start(suite.ctx)
+	require.NoError(t, err)
+	
+	// Initialize searcher (empty initially)
+	searcherConfig := searcher.DefaultSearcherConfig()
+	suite.searcher = searcher.NewDistributedSearcher(searcherConfig, []bleve.Index{})
+	
+	// Initialize analytics
+	suite.analytics = analytics.NewService(suite.searcher)
+	
+	// Initialize log file manager with test-specific behavior
+	suite.logFileManager = &TestLogFileManager{
+		logCache:       make(map[string]*indexer.NginxLogCache),
+		indexingStatus: make(map[string]bool),
+		indexMetadata:  make(map[string]*TestIndexMetadata),
+	}
+	
+	// Register test log files
+	for _, logPath := range suite.logFilePaths {
+		suite.logFileManager.AddLogPath(logPath, "access", filepath.Base(logPath), "test_config")
+	}
+	
+	t.Log("Small services initialized successfully")
+}
+
+// PerformSmallGlobalIndexRebuild performs a complete index rebuild of all small files
+func (suite *SmallIntegrationTestSuite) PerformSmallGlobalIndexRebuild(t *testing.T) {
+	t.Log("Starting small global index rebuild...")
+	
+	startTime := time.Now()
+	
+	// Create progress tracking
+	var completedFiles []string
+	var mu sync.Mutex
+	
+	progressConfig := &indexer.ProgressConfig{
+		NotifyInterval: 500 * time.Millisecond,
+		OnProgress: func(progress indexer.ProgressNotification) {
+			t.Logf("Index progress: %s - %.1f%% (Files: %d/%d, Lines: %d/%d)",
+				progress.LogGroupPath, progress.Percentage, progress.CompletedFiles,
+				progress.TotalFiles, progress.ProcessedLines, progress.EstimatedLines)
+		},
+		OnCompletion: func(completion indexer.CompletionNotification) {
+			mu.Lock()
+			completedFiles = append(completedFiles, completion.LogGroupPath)
+			mu.Unlock()
+			
+			t.Logf("Index completion: %s - Success: %t, Duration: %s, Lines: %d",
+				completion.LogGroupPath, completion.Success, completion.Duration, completion.TotalLines)
+		},
+	}
+	
+	// Destroy existing indexes
+	err := suite.indexer.DestroyAllIndexes(suite.ctx)
+	require.NoError(t, err)
+	
+	// Re-initialize indexer
+	err = suite.indexer.Start(suite.ctx)
+	require.NoError(t, err)
+	
+	// Index all log files
+	allLogs := suite.logFileManager.GetAllLogsWithIndexGrouped()
+	for _, log := range allLogs {
+		docsCountMap, minTime, maxTime, err := suite.indexer.IndexLogGroupWithProgress(log.Path, progressConfig)
+		require.NoError(t, err, "Failed to index log group: %s", log.Path)
+		
+		// Save metadata
+		duration := time.Since(startTime)
+		var totalDocs uint64
+		for _, docCount := range docsCountMap {
+			totalDocs += docCount
+		}
+		
+		err = suite.logFileManager.SaveIndexMetadata(log.Path, totalDocs, startTime, duration, minTime, maxTime)
+		require.NoError(t, err)
+	}
+	
+	// Flush and update searcher
+	err = suite.indexer.FlushAll()
+	require.NoError(t, err)
+	
+	suite.updateSmallSearcher(t)
+	
+	totalDuration := time.Since(startTime)
+	t.Logf("Small global index rebuild completed in %s. Completed files: %v", totalDuration, completedFiles)
+}
+
+// updateSmallSearcher updates the searcher with current shards
+func (suite *SmallIntegrationTestSuite) updateSmallSearcher(t *testing.T) {
+	if !suite.indexer.IsHealthy() {
+		t.Fatal("Indexer is not healthy, cannot update searcher")
+	}
+	
+	newShards := suite.indexer.GetAllShards()
+	t.Logf("Updating searcher with %d shards", len(newShards))
+	
+	if ds, ok := suite.searcher.(*searcher.DistributedSearcher); ok {
+		err := ds.SwapShards(newShards)
+		require.NoError(t, err)
+		t.Log("Searcher shards updated successfully")
+	} else {
+		t.Fatal("Searcher is not a DistributedSearcher")
+	}
+}
+
+// ValidateSmallCardinalityCounter validates the accuracy of cardinality counting
+func (suite *SmallIntegrationTestSuite) ValidateSmallCardinalityCounter(t *testing.T, filePath string) {
+	t.Logf("Validating CardinalityCounter accuracy for: %s", filePath)
+	
+	expected := suite.expectedMetrics[filePath]
+	require.NotNil(t, expected, "Expected metrics not found for file: %s", filePath)
+	
+	if ds, ok := suite.searcher.(*searcher.DistributedSearcher); ok {
+		cardinalityCounter := searcher.NewCardinalityCounter(ds.GetShards())
+		
+		// Test IP cardinality (for all files combined since we can't filter by file path yet)
+		req := &searcher.CardinalityRequest{
+			Field: "remote_addr",
+		}
+		
+		result, err := cardinalityCounter.CountCardinality(suite.ctx, req)
+		require.NoError(t, err, "Failed to count IP cardinality")
+		
+		// For combined files, we expect at least the unique IPs from this file
+		// but possibly more since we're counting across all files
+		assert.GreaterOrEqual(t, result.Cardinality, expected.UniqueIPs,
+			"IP cardinality should be at least %d, got %d", expected.UniqueIPs, result.Cardinality)
+		
+		t.Logf("✓ IP cardinality (all files): actual=%d (expected at least %d), total_docs=%d",
+			result.Cardinality, expected.UniqueIPs, result.TotalDocs)
+	} else {
+		t.Fatal("Searcher is not a DistributedSearcher")
+	}
+}
+
+// ValidateSmallAnalyticsData validates the accuracy of analytics statistics
+func (suite *SmallIntegrationTestSuite) ValidateSmallAnalyticsData(t *testing.T, filePath string) {
+	t.Logf("Validating Analytics data accuracy for: %s", filePath)
+	
+	expected := suite.expectedMetrics[filePath]
+	require.NotNil(t, expected, "Expected metrics not found for file: %s", filePath)
+	
+	// Test dashboard analytics
+	dashboardReq := &analytics.DashboardQueryRequest{
+		LogPaths:  []string{filePath},
+		StartTime: expected.TimeRange.StartTime.Unix(),
+		EndTime:   expected.TimeRange.EndTime.Unix(),
+	}
+	
+	dashboard, err := suite.analytics.GetDashboardAnalytics(suite.ctx, dashboardReq)
+	require.NoError(t, err, "Failed to get dashboard data for: %s", filePath)
+	
+	// Validate basic metrics
+	tolerance := float64(10) // Small tolerance for small datasets
+	assert.InDelta(t, expected.TotalRecords, dashboard.Summary.TotalPV, tolerance,
+		"Total requests mismatch for %s", filePath)
+	
+	t.Logf("✓ Dashboard validation completed for: %s", filePath)
+	t.Logf("  Total requests: expected=%d, actual=%d", expected.TotalRecords, dashboard.Summary.TotalPV)
+	t.Logf("  Unique visitors: %d", dashboard.Summary.TotalUV)
+	t.Logf("  Average daily PV: %f", dashboard.Summary.AvgDailyPV)
+}
+
+// ValidateSmallPaginationFunctionality validates pagination works correctly using searcher
+func (suite *SmallIntegrationTestSuite) ValidateSmallPaginationFunctionality(t *testing.T, filePath string) {
+	t.Logf("Validating pagination functionality for: %s", filePath)
+	
+	expected := suite.expectedMetrics[filePath]
+	require.NotNil(t, expected, "Expected metrics not found for file: %s", filePath)
+	
+	// Test first page - search all records without any filters
+	searchReq1 := &searcher.SearchRequest{
+		Query:     "", // Empty query should use match_all
+		Limit:     50,
+		Offset:    0,
+		SortBy:    "timestamp",
+		SortOrder: "desc",
+	}
+	
+	result1, err := suite.searcher.Search(suite.ctx, searchReq1)
+	require.NoError(t, err, "Failed to get page 1 for: %s", filePath)
+	
+	// For small integration test, we expect at least some results from all files combined
+	totalExpectedRecords := uint64(SmallTestFileCount * SmallTestRecordsPerFile)
+	assert.Greater(t, len(result1.Hits), 0, "First page should have some entries")
+	assert.Equal(t, totalExpectedRecords, result1.TotalHits, "Total count should match all files")
+	
+	// Test second page  
+	searchReq2 := &searcher.SearchRequest{
+		Query:     "", // Empty query should use match_all
+		Limit:     50,
+		Offset:    50,
+		SortBy:    "timestamp",
+		SortOrder: "desc",
+	}
+	
+	result2, err := suite.searcher.Search(suite.ctx, searchReq2)
+	require.NoError(t, err, "Failed to get page 2 for: %s", filePath)
+	
+	// Check that pagination works by ensuring we get different results
+	assert.Greater(t, len(result2.Hits), 0, "Second page should have some entries")
+	assert.Equal(t, totalExpectedRecords, result2.TotalHits, "Total count should be consistent")
+	
+	// Ensure different pages return different entries
+	if len(result1.Hits) > 0 && len(result2.Hits) > 0 {
+		firstPageFirstEntry := result1.Hits[0].ID
+		secondPageFirstEntry := result2.Hits[0].ID
+		assert.NotEqual(t, firstPageFirstEntry, secondPageFirstEntry,
+			"Different pages should return different entries")
+	}
+	
+	t.Logf("✓ Pagination validation completed for: %s", filePath)
+	t.Logf("  Page 1 entries: %d", len(result1.Hits))
+	t.Logf("  Page 2 entries: %d", len(result2.Hits))
+	t.Logf("  Total entries: %d", result1.TotalHits)
+}
+
+// TestSmallNginxLogIntegration is the main small integration test function
+func TestSmallNginxLogIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	
+	suite := NewSmallIntegrationTestSuite(t)
+	defer suite.cleanup()
+	
+	t.Log("=== Starting Small Nginx Log Integration Test ===")
+	
+	// Step 1: Generate test data
+	suite.GenerateSmallTestData(t)
+	
+	// Step 2: Initialize services
+	suite.InitializeSmallServices(t)
+	
+	// Step 3: Perform global index rebuild and validate
+	t.Log("\n=== Testing Small Global Index Rebuild ===")
+	suite.PerformSmallGlobalIndexRebuild(t)
+	
+	// Step 4: Validate all files after global rebuild
+	for _, filePath := range suite.logFilePaths {
+		t.Logf("\n--- Validating file after global rebuild: %s ---", filepath.Base(filePath))
+		suite.ValidateSmallCardinalityCounter(t, filePath)
+		suite.ValidateSmallAnalyticsData(t, filePath)
+		suite.ValidateSmallPaginationFunctionality(t, filePath)
+	}
+	
+	t.Log("\n=== Small Integration Test Completed Successfully ===")
+}

internal/nginx_log/integration_test.go

@@ -0,0 +1,742 @@
+package nginx_log
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/analytics"
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/searcher"
+	"github.com/blevesearch/bleve/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	// Test configuration
+	TestRecordsPerFile = 400000 // 40万条记录每个文件
+	TestFileCount      = 3      // 3个测试文件
+	TestBaseDir        = "./test_integration_logs"
+	TestIndexDir       = "./test_integration_index"
+)
+
+// IntegrationTestSuite contains all integration test data and services
+type IntegrationTestSuite struct {
+	ctx              context.Context
+	cancel           context.CancelFunc
+	tempDir          string
+	indexDir         string
+	logFiles         []string
+	logFilePaths     []string
+	indexer          *indexer.ParallelIndexer
+	searcher         searcher.Searcher
+	analytics        analytics.Service
+	logFileManager   *TestLogFileManager
+	expectedMetrics  map[string]*ExpectedFileMetrics
+	mu               sync.RWMutex
+	cleanup          func()
+}
+
+// TestLogFileManager is a simplified log file manager for testing that doesn't require database
+type TestLogFileManager struct {
+	logCache       map[string]*indexer.NginxLogCache
+	cacheMutex     sync.RWMutex
+	indexingStatus map[string]bool
+	indexingMutex  sync.RWMutex
+	indexMetadata  map[string]*TestIndexMetadata
+	metadataMutex  sync.RWMutex
+}
+
+// TestIndexMetadata holds index metadata for testing
+type TestIndexMetadata struct {
+	Path         string
+	DocumentCount uint64
+	LastIndexed  time.Time
+	Duration     time.Duration
+	MinTime      *time.Time
+	MaxTime      *time.Time
+}
+
+// ExpectedFileMetrics stores expected statistics for each log file
+type ExpectedFileMetrics struct {
+	TotalRecords  uint64
+	UniqueIPs     uint64
+	UniquePaths   uint64
+	UniqueAgents  uint64
+	StatusCodes   map[int]uint64
+	Methods       map[string]uint64
+	TimeRange     TestTimeRange
+}
+
+// TestTimeRange represents the time range of log entries for testing
+type TestTimeRange struct {
+	StartTime time.Time
+	EndTime   time.Time
+}
+
+// NewIntegrationTestSuite creates a new integration test suite
+func NewIntegrationTestSuite(t *testing.T) *IntegrationTestSuite {
+	ctx, cancel := context.WithCancel(context.Background())
+	
+	// Create temporary directories
+	tempDir, err := os.MkdirTemp("", "nginx_ui_integration_test_*")
+	require.NoError(t, err)
+	
+	indexDir := filepath.Join(tempDir, "index")
+	logsDir := filepath.Join(tempDir, "logs")
+	
+	err = os.MkdirAll(indexDir, 0755)
+	require.NoError(t, err)
+	
+	err = os.MkdirAll(logsDir, 0755)
+	require.NoError(t, err)
+
+	suite := &IntegrationTestSuite{
+		ctx:             ctx,
+		cancel:          cancel,
+		tempDir:         tempDir,
+		indexDir:        indexDir,
+		expectedMetrics: make(map[string]*ExpectedFileMetrics),
+	}
+
+	// Set cleanup function
+	suite.cleanup = func() {
+		// Stop services
+		if suite.indexer != nil {
+			suite.indexer.Stop()
+		}
+		if suite.searcher != nil {
+			suite.searcher.Stop()
+		}
+		
+		// Cancel context
+		cancel()
+		
+		// Remove temporary directories
+		os.RemoveAll(tempDir)
+	}
+
+	return suite
+}
+
+// GenerateTestData generates the test log files with expected statistics
+func (suite *IntegrationTestSuite) GenerateTestData(t *testing.T) {
+	t.Logf("Generating %d test files with %d records each", TestFileCount, TestRecordsPerFile)
+	
+	baseTime := time.Now().Add(-24 * time.Hour)
+	
+	for i := 0; i < TestFileCount; i++ {
+		filename := fmt.Sprintf("access_%d.log", i+1)
+		filepath := filepath.Join(suite.tempDir, "logs", filename)
+		
+		metrics := suite.generateSingleLogFile(t, filepath, baseTime.Add(time.Duration(i)*time.Hour))
+		
+		suite.logFiles = append(suite.logFiles, filename)
+		suite.logFilePaths = append(suite.logFilePaths, filepath)
+		suite.expectedMetrics[filepath] = metrics
+		
+		t.Logf("Generated %s with %d records", filename, metrics.TotalRecords)
+	}
+	
+	t.Logf("Test data generation completed. Total files: %d", len(suite.logFiles))
+}
+
+// generateSingleLogFile generates a single log file with known statistics
+func (suite *IntegrationTestSuite) generateSingleLogFile(t *testing.T, filepath string, baseTime time.Time) *ExpectedFileMetrics {
+	file, err := os.Create(filepath)
+	require.NoError(t, err)
+	defer file.Close()
+
+	metrics := &ExpectedFileMetrics{
+		StatusCodes: make(map[int]uint64),
+		Methods:     make(map[string]uint64),
+		TimeRange: TestTimeRange{
+			StartTime: baseTime,
+			EndTime:   baseTime.Add(time.Duration(TestRecordsPerFile) * time.Second),
+		},
+	}
+
+	// Predefined test data for consistent testing
+	ips := []string{
+		"192.168.1.1", "192.168.1.2", "192.168.1.3", "10.0.0.1", "10.0.0.2",
+		"172.16.0.1", "172.16.0.2", "203.0.113.1", "203.0.113.2", "198.51.100.1",
+	}
+	
+	paths := []string{
+		"/", "/api/v1/status", "/api/v1/logs", "/admin", "/login",
+		"/dashboard", "/api/v1/config", "/static/css/main.css", "/static/js/app.js", "/favicon.ico",
+	}
+	
+	userAgents := []string{
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+		"PostmanRuntime/7.28.4",
+		"Go-http-client/1.1",
+	}
+	
+	statusCodes := []int{200, 301, 404, 500, 502}
+	methods := []string{"GET", "POST", "PUT", "DELETE"}
+
+	// Track unique values
+	uniqueIPs := make(map[string]bool)
+	uniquePaths := make(map[string]bool)
+	uniqueAgents := make(map[string]bool)
+
+	rand.Seed(time.Now().UnixNano() + int64(len(filepath))) // Different seed per file
+
+	for i := 0; i < TestRecordsPerFile; i++ {
+		// Generate log entry timestamp
+		timestamp := baseTime.Add(time.Duration(i) * time.Second)
+		
+		// Select random values
+		ip := ips[rand.Intn(len(ips))]
+		path := paths[rand.Intn(len(paths))]
+		agent := userAgents[rand.Intn(len(userAgents))]
+		status := statusCodes[rand.Intn(len(statusCodes))]
+		method := methods[rand.Intn(len(methods))]
+		size := rand.Intn(10000) + 100 // 100-10100 bytes
+		
+		// Track unique values
+		uniqueIPs[ip] = true
+		uniquePaths[path] = true
+		uniqueAgents[agent] = true
+		
+		// Update metrics
+		metrics.StatusCodes[status]++
+		metrics.Methods[method]++
+		
+		// Generate nginx log line (Common Log Format)
+		logLine := fmt.Sprintf(`%s - - [%s] "%s %s HTTP/1.1" %d %d "-" "%s"`+"\n",
+			ip,
+			timestamp.Format("02/Jan/2006:15:04:05 -0700"),
+			method,
+			path,
+			status,
+			size,
+			agent,
+		)
+		
+		_, err := file.WriteString(logLine)
+		require.NoError(t, err)
+	}
+
+	// Finalize metrics
+	metrics.TotalRecords = TestRecordsPerFile
+	metrics.UniqueIPs = uint64(len(uniqueIPs))
+	metrics.UniquePaths = uint64(len(uniquePaths))
+	metrics.UniqueAgents = uint64(len(uniqueAgents))
+
+	return metrics
+}
+
+// InitializeServices initializes all nginx_log services for testing
+func (suite *IntegrationTestSuite) InitializeServices(t *testing.T) {
+	t.Log("Initializing test services...")
+	
+	// Initialize indexer
+	indexerConfig := indexer.DefaultIndexerConfig()
+	indexerConfig.IndexPath = suite.indexDir
+	shardManager := indexer.NewDefaultShardManager(indexerConfig)
+	suite.indexer = indexer.NewParallelIndexer(indexerConfig, shardManager)
+	
+	err := suite.indexer.Start(suite.ctx)
+	require.NoError(t, err)
+	
+	// Initialize searcher (empty initially)
+	searcherConfig := searcher.DefaultSearcherConfig()
+	suite.searcher = searcher.NewDistributedSearcher(searcherConfig, []bleve.Index{})
+	
+	// Initialize analytics
+	suite.analytics = analytics.NewService(suite.searcher)
+	
+	// Initialize log file manager with test-specific behavior
+	suite.logFileManager = suite.createTestLogFileManager(t)
+	
+	// Register test log files
+	for _, logPath := range suite.logFilePaths {
+		suite.logFileManager.AddLogPath(logPath, "access", filepath.Base(logPath), "test_config")
+	}
+	
+	t.Log("Services initialized successfully")
+}
+
+// createTestLogFileManager creates a log file manager suitable for testing
+func (suite *IntegrationTestSuite) createTestLogFileManager(t *testing.T) *TestLogFileManager {
+	return &TestLogFileManager{
+		logCache:       make(map[string]*indexer.NginxLogCache),
+		indexingStatus: make(map[string]bool),
+		indexMetadata:  make(map[string]*TestIndexMetadata),
+	}
+}
+
+// AddLogPath adds a log path to the test log cache
+func (tlm *TestLogFileManager) AddLogPath(path, logType, name, configFile string) {
+	tlm.cacheMutex.Lock()
+	defer tlm.cacheMutex.Unlock()
+
+	tlm.logCache[path] = &indexer.NginxLogCache{
+		Path:       path,
+		Type:       logType,
+		Name:       name,
+		ConfigFile: configFile,
+	}
+}
+
+// GetAllLogsWithIndexGrouped returns all cached log paths with their index status for testing
+func (tlm *TestLogFileManager) GetAllLogsWithIndexGrouped(filters ...func(*indexer.NginxLogWithIndex) bool) []*indexer.NginxLogWithIndex {
+	tlm.cacheMutex.RLock()
+	defer tlm.cacheMutex.RUnlock()
+	
+	tlm.metadataMutex.RLock()
+	defer tlm.metadataMutex.RUnlock()
+
+	var logs []*indexer.NginxLogWithIndex
+
+	for _, logEntry := range tlm.logCache {
+		logWithIndex := &indexer.NginxLogWithIndex{
+			Path:        logEntry.Path,
+			Type:        logEntry.Type,
+			Name:        logEntry.Name,
+			ConfigFile:  logEntry.ConfigFile,
+			IndexStatus: "not_indexed",
+		}
+
+		// Check if we have index metadata for this path
+		if metadata, exists := tlm.indexMetadata[logEntry.Path]; exists {
+			logWithIndex.IndexStatus = "indexed"
+			logWithIndex.DocumentCount = metadata.DocumentCount
+			logWithIndex.LastIndexed = metadata.LastIndexed.Unix()
+			logWithIndex.IndexDuration = int64(metadata.Duration.Milliseconds())
+			
+			if metadata.MinTime != nil {
+				logWithIndex.HasTimeRange = true
+				logWithIndex.TimeRangeStart = metadata.MinTime.Unix()
+			}
+			
+			if metadata.MaxTime != nil {
+				logWithIndex.HasTimeRange = true
+				logWithIndex.TimeRangeEnd = metadata.MaxTime.Unix()
+			}
+		}
+
+		// Apply filters
+		include := true
+		for _, filter := range filters {
+			if !filter(logWithIndex) {
+				include = false
+				break
+			}
+		}
+
+		if include {
+			logs = append(logs, logWithIndex)
+		}
+	}
+
+	return logs
+}
+
+// SaveIndexMetadata saves index metadata for testing
+func (tlm *TestLogFileManager) SaveIndexMetadata(path string, docCount uint64, indexTime time.Time, duration time.Duration, minTime, maxTime *time.Time) error {
+	tlm.metadataMutex.Lock()
+	defer tlm.metadataMutex.Unlock()
+
+	tlm.indexMetadata[path] = &TestIndexMetadata{
+		Path:         path,
+		DocumentCount: docCount,
+		LastIndexed:  indexTime,
+		Duration:     duration,
+		MinTime:      minTime,
+		MaxTime:      maxTime,
+	}
+
+	return nil
+}
+
+// DeleteIndexMetadataByGroup deletes index metadata for a log group (for testing)
+func (tlm *TestLogFileManager) DeleteIndexMetadataByGroup(logGroup string) error {
+	tlm.metadataMutex.Lock()
+	defer tlm.metadataMutex.Unlock()
+
+	delete(tlm.indexMetadata, logGroup)
+	return nil
+}
+
+// DeleteAllIndexMetadata deletes all index metadata (for testing)
+func (tlm *TestLogFileManager) DeleteAllIndexMetadata() error {
+	tlm.metadataMutex.Lock()
+	defer tlm.metadataMutex.Unlock()
+
+	tlm.indexMetadata = make(map[string]*TestIndexMetadata)
+	return nil
+}
+
+// PerformGlobalIndexRebuild performs a complete index rebuild of all files
+func (suite *IntegrationTestSuite) PerformGlobalIndexRebuild(t *testing.T) {
+	t.Log("Starting global index rebuild...")
+	
+	startTime := time.Now()
+	
+	// Create progress tracking
+	var completedFiles []string
+	var mu sync.Mutex
+	
+	progressConfig := &indexer.ProgressConfig{
+		NotifyInterval: 1 * time.Second,
+		OnProgress: func(progress indexer.ProgressNotification) {
+			t.Logf("Index progress: %s - %.1f%% (Files: %d/%d, Lines: %d/%d)",
+				progress.LogGroupPath, progress.Percentage, progress.CompletedFiles,
+				progress.TotalFiles, progress.ProcessedLines, progress.EstimatedLines)
+		},
+		OnCompletion: func(completion indexer.CompletionNotification) {
+			mu.Lock()
+			completedFiles = append(completedFiles, completion.LogGroupPath)
+			mu.Unlock()
+			
+			t.Logf("Index completion: %s - Success: %t, Duration: %s, Lines: %d",
+				completion.LogGroupPath, completion.Success, completion.Duration, completion.TotalLines)
+		},
+	}
+	
+	// Destroy existing indexes
+	err := suite.indexer.DestroyAllIndexes(suite.ctx)
+	require.NoError(t, err)
+	
+	// Re-initialize indexer
+	err = suite.indexer.Start(suite.ctx)
+	require.NoError(t, err)
+	
+	// Index all log files
+	allLogs := suite.logFileManager.GetAllLogsWithIndexGrouped()
+	for _, log := range allLogs {
+		docsCountMap, minTime, maxTime, err := suite.indexer.IndexLogGroupWithProgress(log.Path, progressConfig)
+		require.NoError(t, err, "Failed to index log group: %s", log.Path)
+		
+		// Save metadata
+		duration := time.Since(startTime)
+		var totalDocs uint64
+		for _, docCount := range docsCountMap {
+			totalDocs += docCount
+		}
+		
+		err = suite.logFileManager.SaveIndexMetadata(log.Path, totalDocs, startTime, duration, minTime, maxTime)
+		require.NoError(t, err)
+	}
+	
+	// Flush and update searcher
+	err = suite.indexer.FlushAll()
+	require.NoError(t, err)
+	
+	suite.updateSearcher(t)
+	
+	totalDuration := time.Since(startTime)
+	t.Logf("Global index rebuild completed in %s. Completed files: %v", totalDuration, completedFiles)
+}
+
+// PerformSingleFileIndexRebuild rebuilds index for a single file
+func (suite *IntegrationTestSuite) PerformSingleFileIndexRebuild(t *testing.T, targetFile string) {
+	t.Logf("Starting single file index rebuild for: %s", targetFile)
+	
+	startTime := time.Now()
+	
+	progressConfig := &indexer.ProgressConfig{
+		NotifyInterval: 1 * time.Second,
+		OnProgress: func(progress indexer.ProgressNotification) {
+			t.Logf("Single file index progress: %s - %.1f%%", progress.LogGroupPath, progress.Percentage)
+		},
+		OnCompletion: func(completion indexer.CompletionNotification) {
+			t.Logf("Single file index completion: %s - Success: %t, Lines: %d", 
+				completion.LogGroupPath, completion.Success, completion.TotalLines)
+		},
+	}
+	
+	// Delete existing index for this log group
+	err := suite.indexer.DeleteIndexByLogGroup(targetFile, suite.logFileManager)
+	require.NoError(t, err)
+	
+	// Clean up database records for this log group
+	err = suite.logFileManager.DeleteIndexMetadataByGroup(targetFile)
+	require.NoError(t, err)
+	
+	// Index the specific file
+	docsCountMap, minTime, maxTime, err := suite.indexer.IndexLogGroupWithProgress(targetFile, progressConfig)
+	require.NoError(t, err, "Failed to index single file: %s", targetFile)
+	
+	// Save metadata
+	duration := time.Since(startTime)
+	var totalDocs uint64
+	for _, docCount := range docsCountMap {
+		totalDocs += docCount
+	}
+	
+	err = suite.logFileManager.SaveIndexMetadata(targetFile, totalDocs, startTime, duration, minTime, maxTime)
+	require.NoError(t, err)
+	
+	// Flush and update searcher
+	err = suite.indexer.FlushAll()
+	require.NoError(t, err)
+	
+	suite.updateSearcher(t)
+	
+	totalDuration := time.Since(startTime)
+	t.Logf("Single file index rebuild completed in %s for: %s", totalDuration, targetFile)
+}
+
+// updateSearcher updates the searcher with current shards
+func (suite *IntegrationTestSuite) updateSearcher(t *testing.T) {
+	if !suite.indexer.IsHealthy() {
+		t.Fatal("Indexer is not healthy, cannot update searcher")
+	}
+	
+	newShards := suite.indexer.GetAllShards()
+	t.Logf("Updating searcher with %d shards", len(newShards))
+	
+	if ds, ok := suite.searcher.(*searcher.DistributedSearcher); ok {
+		err := ds.SwapShards(newShards)
+		require.NoError(t, err)
+		t.Log("Searcher shards updated successfully")
+	} else {
+		t.Fatal("Searcher is not a DistributedSearcher")
+	}
+}
+
+// ValidateCardinalityCounter validates the accuracy of cardinality counting
+func (suite *IntegrationTestSuite) ValidateCardinalityCounter(t *testing.T, filePath string) {
+	t.Logf("Validating CardinalityCounter accuracy for: %s", filePath)
+	
+	expected := suite.expectedMetrics[filePath]
+	require.NotNil(t, expected, "Expected metrics not found for file: %s", filePath)
+	
+	// Test IP cardinality
+	suite.testFieldCardinality(t, filePath, "remote_addr", expected.UniqueIPs, "IP addresses")
+	
+	// Test path cardinality
+	suite.testFieldCardinality(t, filePath, "uri_path", expected.UniquePaths, "URI paths")
+	
+	// Test user agent cardinality
+	suite.testFieldCardinality(t, filePath, "http_user_agent", expected.UniqueAgents, "User agents")
+	
+	t.Logf("CardinalityCounter validation completed for: %s", filePath)
+}
+
+// testFieldCardinality tests cardinality counting for a specific field
+func (suite *IntegrationTestSuite) testFieldCardinality(t *testing.T, filePath string, field string, expectedCount uint64, fieldName string) {
+	if ds, ok := suite.searcher.(*searcher.DistributedSearcher); ok {
+		cardinalityCounter := searcher.NewCardinalityCounter(ds.GetShards())
+		
+		req := &searcher.CardinalityRequest{
+			Field:    field,
+			LogPaths: []string{filePath},
+		}
+		
+		result, err := cardinalityCounter.CountCardinality(suite.ctx, req)
+		require.NoError(t, err, "Failed to count cardinality for field: %s", field)
+		
+		// Allow for small discrepancies due to indexing behavior
+		tolerance := uint64(expectedCount) / 100 // 1% tolerance
+		if tolerance < 1 {
+			tolerance = 1
+		}
+		
+		assert.InDelta(t, expectedCount, result.Cardinality, float64(tolerance),
+			"Cardinality mismatch for %s in %s: expected %d, got %d",
+			fieldName, filePath, expectedCount, result.Cardinality)
+		
+		t.Logf("✓ %s cardinality: expected=%d, actual=%d, total_docs=%d",
+			fieldName, expectedCount, result.Cardinality, result.TotalDocs)
+	} else {
+		t.Fatal("Searcher is not a DistributedSearcher")
+	}
+}
+
+// ValidateAnalyticsData validates the accuracy of analytics statistics
+func (suite *IntegrationTestSuite) ValidateAnalyticsData(t *testing.T, filePath string) {
+	t.Logf("Validating Analytics data accuracy for: %s", filePath)
+	
+	expected := suite.expectedMetrics[filePath]
+	require.NotNil(t, expected, "Expected metrics not found for file: %s", filePath)
+	
+	// Test dashboard analytics
+	dashboardReq := &analytics.DashboardQueryRequest{
+		LogPaths:  []string{filePath},
+		StartTime: expected.TimeRange.StartTime.Unix(),
+		EndTime:   expected.TimeRange.EndTime.Unix(),
+	}
+	
+	dashboard, err := suite.analytics.GetDashboardAnalytics(suite.ctx, dashboardReq)
+	require.NoError(t, err, "Failed to get dashboard data for: %s", filePath)
+	
+	// Validate basic metrics
+	tolerance := float64(expected.TotalRecords) * 0.01 // 1% tolerance
+	assert.InDelta(t, expected.TotalRecords, dashboard.Summary.TotalPV, tolerance,
+		"Total requests mismatch for %s", filePath)
+	
+	t.Logf("✓ Dashboard validation completed for: %s", filePath)
+	t.Logf("  Total requests: expected=%d, actual=%d", expected.TotalRecords, dashboard.Summary.TotalPV)
+	t.Logf("  Unique visitors: %d", dashboard.Summary.TotalUV)
+	t.Logf("  Average daily PV: %f", dashboard.Summary.AvgDailyPV)
+}
+
+// ValidatePaginationFunctionality validates pagination works correctly using searcher
+func (suite *IntegrationTestSuite) ValidatePaginationFunctionality(t *testing.T, filePath string) {
+	t.Logf("Validating pagination functionality for: %s", filePath)
+	
+	expected := suite.expectedMetrics[filePath]
+	require.NotNil(t, expected, "Expected metrics not found for file: %s", filePath)
+	
+	startTime := expected.TimeRange.StartTime.Unix()
+	endTime := expected.TimeRange.EndTime.Unix()
+	
+	// Test first page
+	searchReq1 := &searcher.SearchRequest{
+		Query:     "*",
+		LogPaths:  []string{filePath},
+		StartTime: &startTime,
+		EndTime:   &endTime,
+		Limit:     100,
+		Offset:    0,
+		SortBy:    "timestamp",
+		SortOrder: "desc",
+	}
+	
+	result1, err := suite.searcher.Search(suite.ctx, searchReq1)
+	require.NoError(t, err, "Failed to get page 1 for: %s", filePath)
+	assert.Equal(t, 100, len(result1.Hits), "First page should have 100 entries")
+	assert.Equal(t, expected.TotalRecords, result1.TotalHits, "Total count mismatch")
+	
+	// Test second page
+	searchReq2 := &searcher.SearchRequest{
+		Query:     "*",
+		LogPaths:  []string{filePath},
+		StartTime: &startTime,
+		EndTime:   &endTime,
+		Limit:     100,
+		Offset:    100,
+		SortBy:    "timestamp",
+		SortOrder: "desc",
+	}
+	
+	result2, err := suite.searcher.Search(suite.ctx, searchReq2)
+	require.NoError(t, err, "Failed to get page 2 for: %s", filePath)
+	assert.Equal(t, 100, len(result2.Hits), "Second page should have 100 entries")
+	assert.Equal(t, expected.TotalRecords, result2.TotalHits, "Total count should be consistent")
+	
+	// Ensure different pages return different entries
+	if len(result1.Hits) > 0 && len(result2.Hits) > 0 {
+		firstPageFirstEntry := result1.Hits[0].ID
+		secondPageFirstEntry := result2.Hits[0].ID
+		assert.NotEqual(t, firstPageFirstEntry, secondPageFirstEntry,
+			"Different pages should return different entries")
+	}
+	
+	t.Logf("✓ Pagination validation completed for: %s", filePath)
+	t.Logf("  Page 1 entries: %d", len(result1.Hits))
+	t.Logf("  Page 2 entries: %d", len(result2.Hits))
+	t.Logf("  Total entries: %d", result1.TotalHits)
+}
+
+// TestNginxLogIntegration is the main integration test function
+func TestNginxLogIntegration(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	defer suite.cleanup()
+	
+	t.Log("=== Starting Nginx Log Integration Test ===")
+	
+	// Step 1: Generate test data
+	suite.GenerateTestData(t)
+	
+	// Step 2: Initialize services
+	suite.InitializeServices(t)
+	
+	// Step 3: Perform global index rebuild and validate during indexing
+	t.Log("\n=== Testing Global Index Rebuild ===")
+	suite.PerformGlobalIndexRebuild(t)
+	
+	// Step 4: Validate all files after global rebuild
+	for _, filePath := range suite.logFilePaths {
+		t.Logf("\n--- Validating file after global rebuild: %s ---", filepath.Base(filePath))
+		suite.ValidateCardinalityCounter(t, filePath)
+		suite.ValidateAnalyticsData(t, filePath)
+		suite.ValidatePaginationFunctionality(t, filePath)
+	}
+	
+	// Step 5: Test single file rebuild
+	t.Log("\n=== Testing Single File Index Rebuild ===")
+	targetFile := suite.logFilePaths[1] // Rebuild second file
+	suite.PerformSingleFileIndexRebuild(t, targetFile)
+	
+	// Step 6: Validate all files after single file rebuild
+	for _, filePath := range suite.logFilePaths {
+		t.Logf("\n--- Validating file after single file rebuild: %s ---", filepath.Base(filePath))
+		suite.ValidateCardinalityCounter(t, filePath)
+		suite.ValidateAnalyticsData(t, filePath)
+		suite.ValidatePaginationFunctionality(t, filePath)
+	}
+	
+	t.Log("\n=== Integration Test Completed Successfully ===")
+}
+
+// TestConcurrentIndexingAndQuerying tests querying while indexing is in progress
+func TestConcurrentIndexingAndQuerying(t *testing.T) {
+	suite := NewIntegrationTestSuite(t)
+	defer suite.cleanup()
+	
+	t.Log("=== Starting Concurrent Indexing and Querying Test ===")
+	
+	// Generate test data and initialize services
+	suite.GenerateTestData(t)
+	suite.InitializeServices(t)
+	
+	var wg sync.WaitGroup
+	
+	// Start indexing in background
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		suite.PerformGlobalIndexRebuild(t)
+	}()
+	
+	// Wait a bit for indexing to start
+	time.Sleep(2 * time.Second)
+	
+	// Query while indexing is in progress
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		
+		for i := 0; i < 10; i++ {
+			time.Sleep(1 * time.Second)
+			
+			// Test search functionality
+			if suite.searcher.IsHealthy() {
+				searchReq := &searcher.SearchRequest{
+					Query:    "GET",
+					LogPaths: []string{suite.logFilePaths[0]},
+					Limit:    10,
+				}
+				
+				result, err := suite.searcher.Search(suite.ctx, searchReq)
+				if err == nil {
+					t.Logf("Concurrent query %d: found %d results", i+1, result.TotalHits)
+				}
+			}
+		}
+	}()
+	
+	wg.Wait()
+	
+	// Final validation
+	for _, filePath := range suite.logFilePaths {
+		suite.ValidateCardinalityCounter(t, filePath)
+		suite.ValidateAnalyticsData(t, filePath)
+	}
+	
+	t.Log("=== Concurrent Test Completed Successfully ===")
+}

internal/nginx_log/modern_services.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/0xJacky/Nginx-UI/internal/nginx_log/analytics"
 	"github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
@@ -205,9 +206,9 @@ type NginxLogWithIndex = indexer.NginxLogWithIndex
 
 // Constants for backward compatibility
 const (
-	IndexStatusIndexed    = indexer.IndexStatusIndexed
-	IndexStatusIndexing   = indexer.IndexStatusIndexing
-	IndexStatusNotIndexed = indexer.IndexStatusNotIndexed
+	IndexStatusIndexed    = string(indexer.IndexStatusIndexed)
+	IndexStatusIndexing   = string(indexer.IndexStatusIndexing)
+	IndexStatusNotIndexed = string(indexer.IndexStatusNotIndexed)
 )
 
 // Legacy compatibility functions for log cache system
@@ -276,8 +277,23 @@ func GetIndexingFiles() []string {
 // Uses Bleve IndexAlias.Swap() for atomic shard replacement without recreating the searcher.
 // This function is safe for concurrent use and maintains service availability during index rebuilds.
 func UpdateSearcherShards() {
-	servicesMutex.Lock() // Use a write lock as we are modifying a global variable
-	defer servicesMutex.Unlock()
+	// Schedule async update to avoid blocking indexing operations
+	logger.Debugf("UpdateSearcherShards: Scheduling async shard update")
+	go updateSearcherShardsAsync()
+}
+
+// updateSearcherShardsAsync performs the actual shard update asynchronously
+func updateSearcherShardsAsync() {
+	// Small delay to let indexing operations complete
+	time.Sleep(500 * time.Millisecond)
+	
+	logger.Debugf("updateSearcherShardsAsync: Attempting to acquire write lock...")
+	servicesMutex.Lock()
+	logger.Debugf("updateSearcherShardsAsync: Write lock acquired")
+	defer func() {
+		logger.Debugf("updateSearcherShardsAsync: Releasing write lock...")
+		servicesMutex.Unlock()
+	}()
 	updateSearcherShardsLocked()
 }
 
@@ -322,12 +338,14 @@ func updateSearcherShardsLocked() {
 	// This follows Bleve best practices for zero-downtime index updates
 	if ds, ok := globalSearcher.(*searcher.DistributedSearcher); ok {
 		oldShards := ds.GetShards()
+		logger.Debugf("updateSearcherShardsLocked: About to call SwapShards...")
 		
 		// Perform atomic shard swap using IndexAlias
 		if err := ds.SwapShards(newShards); err != nil {
 			logger.Errorf("Failed to swap shards atomically: %v", err)
 			return
 		}
+		logger.Debugf("updateSearcherShardsLocked: SwapShards completed successfully")
 		
 		logger.Infof("Successfully swapped %d old shards with %d new shards using IndexAlias", 
 			len(oldShards), len(newShards))

internal/nginx_log/preflight_service.go

@@ -0,0 +1,200 @@
+package nginx_log
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/0xJacky/Nginx-UI/internal/event"
+	"github.com/0xJacky/Nginx-UI/internal/nginx"
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
+	"github.com/uozi-tech/cosy/logger"
+)
+
+// FileInfo represents basic file information for internal use
+type FileInfo struct {
+	Exists       bool  `json:"exists"`
+	Readable     bool  `json:"readable"`
+	Size         int64 `json:"size,omitempty"`
+	LastModified int64 `json:"last_modified,omitempty"`
+}
+
+// TimeRange represents a time range for log data for internal use
+type TimeRange struct {
+	Start int64 `json:"start"`
+	End   int64 `json:"end"`
+}
+
+// PreflightResponse represents the response from preflight checks for internal use
+type PreflightResponse struct {
+	Available   bool       `json:"available"`
+	IndexStatus string     `json:"index_status"`
+	Message     string     `json:"message,omitempty"`
+	TimeRange   *TimeRange `json:"time_range,omitempty"`
+	FileInfo    *FileInfo  `json:"file_info,omitempty"`
+}
+
+// PreflightService handles preflight checks for log files
+type PreflightService struct{}
+
+// NewPreflightService creates a new preflight service
+func NewPreflightService() *PreflightService {
+	return &PreflightService{}
+}
+
+// CheckLogPreflight performs preflight checks for a log file
+func (ps *PreflightService) CheckLogPreflight(logPath string) (*PreflightResponse, error) {
+	// Use default access log path if logPath is empty
+	if logPath == "" {
+		defaultLogPath := nginx.GetAccessLogPath()
+		if defaultLogPath != "" {
+			logPath = defaultLogPath
+			logger.Debugf("Using default access log path for preflight: %s", logPath)
+		}
+	}
+
+	// Get searcher to check index status
+	searcherService := GetModernSearcher()
+	if searcherService == nil {
+		return nil, ErrModernSearcherNotAvailable
+	}
+
+	// Check if the specific file is currently being indexed
+	processingManager := event.GetProcessingStatusManager()
+	currentStatus := processingManager.GetCurrentStatus()
+
+	// First check if the file exists and get file info
+	var fileInfo *os.FileInfo
+	if logPath != "" {
+		if stat, err := os.Stat(logPath); os.IsNotExist(err) {
+			// File doesn't exist - check for historical data
+			return ps.handleMissingFile(logPath, searcherService)
+		} else if err != nil {
+			// Permission or other file system error - map to error status
+			return &PreflightResponse{
+				Available:   false,
+				IndexStatus: string(indexer.IndexStatusError),
+				Message:     fmt.Sprintf("Cannot access log file %s: %v", logPath, err),
+				FileInfo: &FileInfo{
+					Exists:   true,
+					Readable: false,
+				},
+			}, nil
+		} else {
+			fileInfo = &stat
+		}
+	}
+
+	// Check if searcher is healthy
+	searcherHealthy := searcherService.IsHealthy()
+
+	// Get detailed file status from log file manager
+	return ps.buildPreflightResponse(logPath, fileInfo, searcherHealthy, &currentStatus)
+}
+
+// handleMissingFile handles the case when a log file doesn't exist
+func (ps *PreflightService) handleMissingFile(logPath string, searcherService interface{}) (*PreflightResponse, error) {
+	searcherHealthy := searcherService.(interface{ IsHealthy() bool }).IsHealthy()
+	logFileManager := GetLogFileManager()
+
+	if logFileManager != nil {
+		logGroup, err := logFileManager.GetLogByPath(logPath)
+		if err == nil && logGroup != nil && logGroup.LastIndexed > 0 {
+			// File has historical index data
+			response := &PreflightResponse{
+				Available:   searcherHealthy,
+				IndexStatus: string(indexer.IndexStatusIndexed),
+				Message:     "File indexed (historical data available)",
+				FileInfo: &FileInfo{
+					Exists:   false,
+					Readable: false,
+				},
+			}
+			if logGroup.HasTimeRange {
+				response.TimeRange = &TimeRange{
+					Start: logGroup.TimeRangeStart,
+					End:   logGroup.TimeRangeEnd,
+				}
+			}
+			return response, nil
+		}
+	}
+
+	// File doesn't exist and no historical data
+	return &PreflightResponse{
+		Available:   false,
+		IndexStatus: string(indexer.IndexStatusNotIndexed),
+		Message:     "Log file does not exist",
+		FileInfo: &FileInfo{
+			Exists:   false,
+			Readable: false,
+		},
+	}, nil
+}
+
+// buildPreflightResponse builds the preflight response for existing files
+func (ps *PreflightService) buildPreflightResponse(logPath string, fileInfo *os.FileInfo, searcherHealthy bool, currentStatus *event.ProcessingStatusData) (*PreflightResponse, error) {
+	logFileManager := GetLogFileManager()
+	var indexStatus string = string(indexer.IndexStatusNotIndexed)
+	var available bool = false
+
+	response := &PreflightResponse{}
+
+	if logFileManager != nil && logPath != "" {
+		logGroup, err := logFileManager.GetLogByPath(logPath)
+		if err == nil && logGroup != nil {
+			// Determine status based on indexing state
+			if logGroup.LastIndexed > 0 {
+				indexStatus = string(indexer.IndexStatusIndexed)
+				available = searcherHealthy
+			} else if currentStatus.NginxLogIndexing {
+				indexStatus = string(indexer.IndexStatusIndexing)
+				available = false
+			} else {
+				indexStatus = string(indexer.IndexStatusNotIndexed)
+				available = false
+			}
+
+			response.Available = available
+			response.IndexStatus = indexStatus
+
+			// Add time range if available
+			if logGroup.HasTimeRange {
+				response.TimeRange = &TimeRange{
+					Start: logGroup.TimeRangeStart,
+					End:   logGroup.TimeRangeEnd,
+				}
+			}
+		} else {
+			// File not in database or error getting it
+			if currentStatus.NginxLogIndexing {
+				indexStatus = string(indexer.IndexStatusQueued)
+			} else {
+				indexStatus = string(indexer.IndexStatusNotIndexed)
+			}
+			available = false
+
+			response.Available = available
+			response.IndexStatus = indexStatus
+			response.Message = "Log file not indexed yet"
+		}
+	} else {
+		// Fallback to basic status
+		response.Available = searcherHealthy
+		response.IndexStatus = string(indexer.IndexStatusNotIndexed)
+	}
+
+	// Add file information if available
+	if fileInfo != nil {
+		response.FileInfo = &FileInfo{
+			Exists:       true,
+			Readable:     true,
+			Size:         (*fileInfo).Size(),
+			LastModified: (*fileInfo).ModTime().Unix(),
+		}
+	}
+
+	logger.Debugf("Preflight response: log_path=%s, available=%v, index_status=%s",
+		logPath, response.Available, response.IndexStatus)
+
+	return response, nil
+}

internal/nginx_log/searcher/cardinality_counter.go

@@ -2,6 +2,7 @@ package searcher
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sync"
 
@@ -129,22 +130,33 @@ func (cc *CardinalityCounter) collectTermsUsingIndexAlias(ctx context.Context, r
 func (cc *CardinalityCounter) collectTermsUsingLargeFacet(ctx context.Context, req *CardinalityRequest) (map[string]struct{}, uint64, error) {
 	terms := make(map[string]struct{})
 	
-	// Build search request using IndexAlias
-	searchReq := bleve.NewSearchRequest(bleve.NewMatchAllQuery())
-	searchReq.Size = 0 // We don't need documents, just facets
-
+	// Build search request using IndexAlias with proper filtering
+	boolQuery := bleve.NewBooleanQuery()
+	boolQuery.AddMust(bleve.NewMatchAllQuery())
+	
 	// Add time range filter if specified
 	if req.StartTime != nil && req.EndTime != nil {
 		startTime := float64(*req.StartTime)
 		endTime := float64(*req.EndTime)
 		timeQuery := bleve.NewNumericRangeQuery(&startTime, &endTime)
 		timeQuery.SetField("timestamp")
-		
-		boolQuery := bleve.NewBooleanQuery()
-		boolQuery.AddMust(searchReq.Query)
 		boolQuery.AddMust(timeQuery)
-		searchReq.Query = boolQuery
 	}
+	
+	// CRITICAL FIX: Add log path filters (this was missing!)
+	if len(req.LogPaths) > 0 {
+		logPathQuery := bleve.NewBooleanQuery()
+		for _, logPath := range req.LogPaths {
+			termQuery := bleve.NewTermQuery(logPath)
+			termQuery.SetField("file_path")
+			logPathQuery.AddShould(termQuery)
+		}
+		logPathQuery.SetMinShould(1)
+		boolQuery.AddMust(logPathQuery)
+	}
+	
+	searchReq := bleve.NewSearchRequest(boolQuery)
+	searchReq.Size = 0 // We don't need documents, just facets
 
 	// Use very large facet size - we're back to this approach but using IndexAlias
 	// which should handle it more efficiently than individual shards
@@ -152,11 +164,18 @@ func (cc *CardinalityCounter) collectTermsUsingLargeFacet(ctx context.Context, r
 	facet := bleve.NewFacetRequest(req.Field, facetSize)
 	searchReq.AddFacet(req.Field, facet)
 
+	// Debug: Log the constructed query
+	if queryBytes, err := json.Marshal(searchReq.Query); err == nil {
+		logger.Debugf("CardinalityCounter query: %s", string(queryBytes))
+	}
+	
 	// Execute search using IndexAlias with global scoring context
 	result, err := cc.indexAlias.SearchInContext(ctx, searchReq)
 	if err != nil {
 		return terms, 0, fmt.Errorf("IndexAlias facet search failed: %w", err)
 	}
+	
+	logger.Debugf("CardinalityCounter facet search result: Total=%d, Facets=%v", result.Total, result.Facets != nil)
 
 	// Extract terms from facet result
 	if facetResult, ok := result.Facets[req.Field]; ok && facetResult.Terms != nil {
@@ -188,23 +207,35 @@ func (cc *CardinalityCounter) collectTermsUsingPagination(ctx context.Context, r
 	logger.Infof("Starting IndexAlias pagination for field '%s' (pageSize=%d)", req.Field, pageSize)
 	
 	for page := 0; page < maxPages; page++ {
-		searchReq := bleve.NewSearchRequest(bleve.NewMatchAllQuery())
-		searchReq.Size = pageSize
-		searchReq.From = page * pageSize
-		searchReq.Fields = []string{req.Field}
-
+		// Build proper query with all filters
+		boolQuery := bleve.NewBooleanQuery()
+		boolQuery.AddMust(bleve.NewMatchAllQuery())
+		
 		// Add time range filter if specified
 		if req.StartTime != nil && req.EndTime != nil {
 			startTime := float64(*req.StartTime)
 			endTime := float64(*req.EndTime)
 			timeQuery := bleve.NewNumericRangeQuery(&startTime, &endTime)
 			timeQuery.SetField("timestamp")
-			
-			boolQuery := bleve.NewBooleanQuery()
-			boolQuery.AddMust(searchReq.Query)
 			boolQuery.AddMust(timeQuery)
-			searchReq.Query = boolQuery
 		}
+		
+		// CRITICAL FIX: Add log path filters (this was missing!)
+		if len(req.LogPaths) > 0 {
+			logPathQuery := bleve.NewBooleanQuery()
+			for _, logPath := range req.LogPaths {
+				termQuery := bleve.NewTermQuery(logPath)
+				termQuery.SetField("file_path")
+				logPathQuery.AddShould(termQuery)
+			}
+			logPathQuery.SetMinShould(1)
+			boolQuery.AddMust(logPathQuery)
+		}
+		
+		searchReq := bleve.NewSearchRequest(boolQuery)
+		searchReq.Size = pageSize
+		searchReq.From = page * pageSize
+		searchReq.Fields = []string{req.Field}
 
 		// Execute with IndexAlias and global scoring
 		result, err := cc.indexAlias.SearchInContext(ctx, searchReq)

internal/nginx_log/searcher/distributed_searcher.go

@@ -2,6 +2,7 @@ package searcher
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sync"
 	"sync/atomic"
@@ -217,6 +218,12 @@ func (ds *DistributedSearcher) executeGlobalScoringSearch(ctx context.Context, q
 	// This is the key fix from Bleve documentation for distributed search
 	globalCtx := context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring)
 	
+	// Debug: Log the constructed query for comparison
+	if queryBytes, err := json.Marshal(searchReq.Query); err == nil {
+		logger.Debugf("Main search query: %s", string(queryBytes))
+		logger.Debugf("Main search Size=%d, From=%d, Fields=%v", searchReq.Size, searchReq.From, searchReq.Fields)
+	}
+	
 	// Execute search using Bleve's IndexAlias with global scoring
 	result, err := ds.indexAlias.SearchInContext(globalCtx, searchReq)
 	if err != nil {
@@ -486,11 +493,35 @@ func (ds *DistributedSearcher) SwapShards(newShards []bleve.Index) error {
 	
 	// Perform atomic swap using IndexAlias - this is the key Bleve operation
 	// that provides zero-downtime index updates
+	logger.Debugf("SwapShards: Starting atomic swap - old=%d, new=%d", len(oldShards), len(newShards))
+	
+	swapStartTime := time.Now()
 	ds.indexAlias.Swap(newShards, oldShards)
+	swapDuration := time.Since(swapStartTime)
+	
+	logger.Infof("IndexAlias.Swap completed in %v (old=%d shards, new=%d shards)", 
+		swapDuration, len(oldShards), len(newShards))
 	
 	// Update internal shards reference to match the IndexAlias
 	ds.shards = newShards
 	
+	// Clear cache after shard swap to prevent stale results
+	// Use goroutine to avoid potential deadlock during shard swap
+	if ds.cache != nil {
+		// Capture cache reference to avoid race condition
+		cache := ds.cache
+		go func() {
+			// Add a small delay to ensure shard swap is fully completed
+			time.Sleep(100 * time.Millisecond)
+			
+			// Double-check cache is still valid before clearing
+			if cache != nil {
+				cache.Clear()
+				logger.Infof("Cache cleared after shard swap to prevent stale results")
+			}
+		}()
+	}
+	
 	// Update shard stats for the new shards
 	ds.stats.mutex.Lock()
 	// Clear old shard stats

internal/nginx_log/searcher/optimized_cache.go

@@ -120,7 +120,9 @@ func (osc *OptimizedSearchCache) Put(req *SearchRequest, result *SearchResult, t
 
 // Clear clears all cached entries
 func (osc *OptimizedSearchCache) Clear() {
-	osc.cache.Clear()
+	if osc != nil && osc.cache != nil {
+		osc.cache.Clear()
+	}
 }
 
 // GetStats returns cache statistics

internal/nginx_log/task_recovery.go

@@ -0,0 +1,186 @@
+package nginx_log
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/0xJacky/Nginx-UI/internal/nginx_log/indexer"
+	"github.com/uozi-tech/cosy/logger"
+)
+
+// TaskRecovery handles the recovery of incomplete indexing tasks after restart
+type TaskRecovery struct {
+	logFileManager *indexer.LogFileManager
+	modernIndexer  *indexer.ParallelIndexer
+}
+
+// NewTaskRecovery creates a new task recovery manager
+func NewTaskRecovery() *TaskRecovery {
+	return &TaskRecovery{
+		logFileManager: GetLogFileManager(),
+		modernIndexer:  GetModernIndexer(),
+	}
+}
+
+// RecoverUnfinishedTasks recovers indexing tasks that were incomplete at last shutdown
+func (tr *TaskRecovery) RecoverUnfinishedTasks(ctx context.Context) error {
+	if tr.logFileManager == nil || tr.modernIndexer == nil {
+		logger.Warn("Cannot recover tasks: services not available")
+		return nil
+	}
+
+	logger.Info("Starting recovery of unfinished indexing tasks")
+
+	// Get all logs with their index status
+	allLogs := GetAllLogsWithIndexGrouped(func(log *NginxLogWithIndex) bool {
+		// Only process access logs
+		return log.Type == "access"
+	})
+
+	var incompleteTasksCount int
+	var queuePosition int = 1
+
+	for _, log := range allLogs {
+		if tr.needsRecovery(log) {
+			incompleteTasksCount++
+			
+			// Reset to queued status and assign queue position
+			if err := tr.recoverTask(ctx, log.Path, queuePosition); err != nil {
+				logger.Errorf("Failed to recover task for %s: %v", log.Path, err)
+			} else {
+				queuePosition++
+			}
+		}
+	}
+
+	if incompleteTasksCount > 0 {
+		logger.Infof("Recovered %d incomplete indexing tasks", incompleteTasksCount)
+	} else {
+		logger.Info("No incomplete indexing tasks found")
+	}
+
+	return nil
+}
+
+// needsRecovery determines if a log file has an incomplete indexing task that needs recovery
+func (tr *TaskRecovery) needsRecovery(log *NginxLogWithIndex) bool {
+	// Check for incomplete states that indicate interrupted operations
+	switch log.IndexStatus {
+	case string(indexer.IndexStatusIndexing):
+		// Task was in progress during last shutdown
+		logger.Debugf("Found incomplete indexing task: %s", log.Path)
+		return true
+		
+	case string(indexer.IndexStatusQueued):
+		// Task was queued but may not have started
+		logger.Debugf("Found queued indexing task: %s", log.Path)
+		return true
+		
+	case string(indexer.IndexStatusError):
+		// Check if error is recent (within last hour before restart)
+		if log.LastIndexed > 0 {
+			lastIndexTime := time.Unix(log.LastIndexed, 0)
+			if time.Since(lastIndexTime) < time.Hour {
+				logger.Debugf("Found recent error task for retry: %s", log.Path)
+				return true
+			}
+		}
+		
+	case string(indexer.IndexStatusPartial):
+		// Partial indexing should be resumed
+		logger.Debugf("Found partial indexing task: %s", log.Path)
+		return true
+	}
+	
+	return false
+}
+
+// recoverTask recovers a single indexing task
+func (tr *TaskRecovery) recoverTask(ctx context.Context, logPath string, queuePosition int) error {
+	logger.Infof("Recovering indexing task for: %s (queue position: %d)", logPath, queuePosition)
+	
+	// Set status to queued with queue position
+	if err := tr.setTaskStatus(logPath, string(indexer.IndexStatusQueued), queuePosition); err != nil {
+		return err
+	}
+	
+	// Queue the recovery task asynchronously
+	go tr.executeRecoveredTask(ctx, logPath)
+	
+	return nil
+}
+
+// executeRecoveredTask executes a recovered indexing task
+func (tr *TaskRecovery) executeRecoveredTask(ctx context.Context, logPath string) {
+	// Add a small delay to stagger recovery tasks
+	time.Sleep(time.Second * 2)
+	
+	logger.Infof("Executing recovered indexing task: %s", logPath)
+	
+	// Set status to indexing
+	if err := tr.setTaskStatus(logPath, string(indexer.IndexStatusIndexing), 0); err != nil {
+		logger.Errorf("Failed to set indexing status for recovered task %s: %v", logPath, err)
+		return
+	}
+	
+	// Execute the indexing
+	startTime := time.Now()
+	docsCountMap, minTime, maxTime, err := tr.modernIndexer.IndexLogGroupWithProgress(logPath, nil)
+	
+	if err != nil {
+		logger.Errorf("Failed to execute recovered indexing task %s: %v", logPath, err)
+		// Set error status
+		if statusErr := tr.setTaskStatus(logPath, string(indexer.IndexStatusError), 0); statusErr != nil {
+			logger.Errorf("Failed to set error status for recovered task %s: %v", logPath, statusErr)
+		}
+		return
+	}
+	
+	// Calculate total documents indexed
+	var totalDocsIndexed uint64
+	for _, docCount := range docsCountMap {
+		totalDocsIndexed += docCount
+	}
+	
+	// Save indexing metadata using the log file manager
+	duration := time.Since(startTime)
+	if err := tr.logFileManager.SaveIndexMetadata(logPath, totalDocsIndexed, startTime, duration, minTime, maxTime); err != nil {
+		logger.Errorf("Failed to save recovered index metadata for %s: %v", logPath, err)
+	}
+	
+	// Set status to indexed (completed)
+	if err := tr.setTaskStatus(logPath, string(indexer.IndexStatusIndexed), 0); err != nil {
+		logger.Errorf("Failed to set completed status for recovered task %s: %v", logPath, err)
+	}
+	
+	// Update searcher shards
+	UpdateSearcherShards()
+	
+	logger.Infof("Successfully completed recovered indexing task: %s, Documents: %d", logPath, totalDocsIndexed)
+}
+
+// setTaskStatus updates the task status in the database using the enhanced persistence layer
+func (tr *TaskRecovery) setTaskStatus(logPath, status string, queuePosition int) error {
+	// Get persistence manager
+	persistence := tr.logFileManager.GetPersistence()
+	if persistence == nil {
+		return fmt.Errorf("persistence manager not available")
+	}
+	
+	// Use enhanced SetIndexStatus method
+	return persistence.SetIndexStatus(logPath, status, queuePosition, "")
+}
+
+// InitTaskRecovery initializes the task recovery system - called during application startup
+func InitTaskRecovery(ctx context.Context) {
+	logger.Info("Initializing task recovery system")
+	
+	// Wait a bit for services to fully initialize
+	time.Sleep(3 * time.Second)
+	
+	recoveryManager := NewTaskRecovery()
+	if err := recoveryManager.RecoverUnfinishedTasks(ctx); err != nil {
+		logger.Errorf("Failed to recover unfinished tasks: %v", err)
+	}
+}

model/nginx_log_index.go

@@ -20,6 +20,14 @@ type NginxLogIndex struct {
 	DocumentCount  uint64     `gorm:"default:0" json:"document_count"`           // Total documents indexed from this file
 	Enabled        bool       `gorm:"default:true" json:"enabled"`               // Whether indexing is enabled for this file
 	HasTimeRange   bool       `gorm:"-" json:"has_timerange"`                    // Whether a time range is available (not persisted)
+	
+	// Extended status fields
+	IndexStatus    string     `gorm:"default:'not_indexed';size:50" json:"index_status"`  // Current index status
+	ErrorMessage   string     `gorm:"type:text" json:"error_message,omitempty"`           // Last error message
+	ErrorTime      *time.Time `json:"error_time,omitempty"`                               // When error occurred
+	RetryCount     int        `gorm:"default:0" json:"retry_count"`                       // Number of retry attempts
+	QueuePosition  int        `gorm:"default:0" json:"queue_position,omitempty"`          // Position in indexing queue
+	PartialOffset  int64      `gorm:"default:0" json:"partial_offset,omitempty"`          // Offset for partial indexing
 }
 
 // NeedsIndexing checks if the file needs incremental indexing
@@ -99,4 +107,55 @@ func (nli *NginxLogIndex) Reset() {
 	nli.DocumentCount = 0
 	nli.TimeRangeStart = nil
 	nli.TimeRangeEnd = nil
+	
+	// Reset status fields
+	nli.IndexStatus = "not_indexed"
+	nli.ErrorMessage = ""
+	nli.ErrorTime = nil
+	nli.RetryCount = 0
+	nli.QueuePosition = 0
+	nli.PartialOffset = 0
+}
+
+// SetIndexingStatus updates the indexing status
+func (nli *NginxLogIndex) SetIndexingStatus(status string) {
+	nli.IndexStatus = status
+	if status == "indexing" {
+		now := time.Now()
+		nli.IndexStartTime = &now
+	}
+}
+
+// SetErrorStatus records an error status with message
+func (nli *NginxLogIndex) SetErrorStatus(errorMessage string) {
+	nli.IndexStatus = "error"
+	nli.ErrorMessage = errorMessage
+	now := time.Now()
+	nli.ErrorTime = &now
+	nli.RetryCount++
+}
+
+// SetCompletedStatus marks indexing as completed successfully
+func (nli *NginxLogIndex) SetCompletedStatus() {
+	nli.IndexStatus = "indexed"
+	nli.ErrorMessage = ""
+	nli.ErrorTime = nil
+	nli.QueuePosition = 0
+	nli.PartialOffset = 0
+}
+
+// SetQueuedStatus marks the file as queued for indexing
+func (nli *NginxLogIndex) SetQueuedStatus(position int) {
+	nli.IndexStatus = "queued"
+	nli.QueuePosition = position
+}
+
+// IsHealthy returns true if the index is in a good state
+func (nli *NginxLogIndex) IsHealthy() bool {
+	return nli.IndexStatus == "indexed" || nli.IndexStatus == "indexing"
+}
+
+// CanRetry returns true if the file can be retried for indexing
+func (nli *NginxLogIndex) CanRetry() bool {
+	return nli.IndexStatus == "error"
 }