nginx-ui/internal/nginx_log/indexer_file_full.go

package nginx_log

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/blevesearch/bleve/v2"
	"github.com/uozi-tech/cosy/logger"
)

// IndexLogFileFull performs full reindexing of a log file and its related log group
func (li *LogIndexer) IndexLogFileFull(filePath string) error {
	logger.Infof("Starting full reindex of log file and related group: %s", filePath)

	// Determine main log path for group operations
	mainLogPath := li.getMainLogPath(filePath)
	logDir := filepath.Dir(mainLogPath)
	baseLogName := filepath.Base(mainLogPath)

	// Get or create progress tracker for this log group
	progressTracker := GetProgressTracker(mainLogPath)

	// Find all related log files in the group
	relatedFiles, err := li.findRelatedLogFiles(logDir, baseLogName)
	if err != nil {
		// Fallback to single file if related file discovery fails
		logger.Warnf("Failed to find related files for %s, processing single file: %v", filePath, err)
		relatedFiles = []string{filePath}
	}

	logger.Infof("Full reindexing log group %s with %d files: %v", mainLogPath, len(relatedFiles), relatedFiles)

	// Initialize progress tracker with all files
	for _, file := range relatedFiles {
		info, err := li.safeGetFileInfo(file)
		if err != nil {
			logger.Warnf("Failed to get file info for %s: %v", file, err)
			continue
		}

		isCompressed := strings.HasSuffix(file, ".gz") || strings.HasSuffix(file, ".bz2")
		progressTracker.AddFile(file, isCompressed)

		// Estimate lines in this file
		estimatedLines := EstimateFileLines(file, info.Size(), isCompressed)
		progressTracker.SetFileEstimate(file, estimatedLines)
	}

	// Delete existing index data for the entire log group
	if err := li.DeleteLogGroupFromIndex(mainLogPath); err != nil {
		logger.Warnf("Failed to delete existing index data for log group %s: %v", mainLogPath, err)
	}

	// Index all files in the group
	for _, file := range relatedFiles {
		if err := li.indexSingleFileForGroup(file, mainLogPath, progressTracker); err != nil {
			logger.Errorf("Failed to index file %s in group %s: %v", file, mainLogPath, err)
			// Continue with other files rather than failing completely
		} else {
			logger.Infof("Successfully indexed file %s in group %s", file, mainLogPath)
		}
	}

	// Clean up progress tracker
	RemoveProgressTracker(mainLogPath)

	// Clear indexing status for all files in the group
	for _, file := range relatedFiles {
		SetIndexingStatus(file, false)
	}

	logger.Infof("Completed full reindex of log group %s with %d files", mainLogPath, len(relatedFiles))
	return nil
}

// indexSingleFileForGroup indexes a single file as part of a log group
func (li *LogIndexer) indexSingleFileForGroup(filePath, mainLogPath string, progressTracker *ProgressTracker) error {
	// Get or create log index record
	logIndex, err := li.persistence.GetLogIndex(filePath)
	if err != nil {
		return fmt.Errorf("failed to get log index record: %w", err)
	}

	// Get current file info using safe method
	currentInfo, err := li.safeGetFileInfo(filePath)
	if err != nil {
		return fmt.Errorf("failed to safely stat file %s: %w", filePath, err)
	}

	logger.Infof("Indexing file in group: %s -> %s (size: %d, mod: %v)", filePath, mainLogPath, currentInfo.Size(), currentInfo.ModTime())

	// Start file processing in progress tracker
	progressTracker.StartFile(filePath)

	// Reset log index position for full reindex
	logIndex.Reset()

	// Index the entire file with specified mainLogPath for grouping
	return li.indexFileFromPositionWithMainLogPath(filePath, mainLogPath, 0, logIndex, progressTracker)
}

// DeleteLogGroupFromIndex removes all index entries for a given log group
func (li *LogIndexer) DeleteLogGroupFromIndex(mainLogPath string) error {
	logger.Infof("Deleting all index entries for log group: %s", mainLogPath)
	query := bleve.NewTermQuery(mainLogPath)
	query.SetField("file_path")
	searchReq := bleve.NewSearchRequest(query)
	searchReq.Size = 10000 // Process in batches

	for {
		searchResult, err := li.index.Search(searchReq)
		if err != nil {
			return fmt.Errorf("failed to search existing entries for log group %s: %w", mainLogPath, err)
		}

		if len(searchResult.Hits) == 0 {
			break
		}

		batch := li.index.NewBatch()
		for _, hit := range searchResult.Hits {
			batch.Delete(hit.ID)
		}

		if err := li.index.Batch(batch); err != nil {
			return fmt.Errorf("failed to delete entries for log group %s: %w", mainLogPath, err)
		}
		logger.Infof("Deleted %d entries for log group %s", len(searchResult.Hits), mainLogPath)
	}
	return nil
}