nginx-ui/internal/nginx_log/log_cache_grouping.go

package nginx_log

import (
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/0xJacky/Nginx-UI/model"
	"github.com/uozi-tech/cosy/logger"
)

// GetAllLogsWithIndexGrouped returns logs grouped by their base name (e.g., access.log includes access.log.1, access.log.2.gz etc.)
func GetAllLogsWithIndexGrouped(filters ...func(*NginxLogWithIndex) bool) []*NginxLogWithIndex {
	cacheMutex.RLock()
	defer cacheMutex.RUnlock()

	// Get all logs from both cache (config files) and persistence (indexed files)
	allLogsMap := make(map[string]*NginxLogWithIndex)
	
	// First, get logs from the cache (these are from nginx config)
	for _, cache := range logCache {
		logWithIndex := &NginxLogWithIndex{
			Path:         cache.Path,
			Type:         cache.Type,
			Name:         cache.Name,
			ConfigFile:   cache.ConfigFile,
			IndexStatus:  IndexStatusNotIndexed,
			IsCompressed: false,
			HasTimeRange: false,
		}
		allLogsMap[cache.Path] = logWithIndex
	}

	// Get persistence manager for database index records
	persistence := NewPersistenceManager()
	persistenceIndexes, err := persistence.GetAllLogIndexes()
	if err != nil {
		logger.Warnf("Failed to get persistence indexes: %v", err)
		persistenceIndexes = []*model.NginxLogIndex{}
	}

	// Add all indexed files from persistence (including rotated files)
	for _, idx := range persistenceIndexes {
		if _, exists := allLogsMap[idx.Path]; !exists {
			// This is a rotated file not in config cache, create entry for it
			logType := "access"
			if strings.Contains(idx.Path, "error") {
				logType = "error"
			}
			
			logWithIndex := &NginxLogWithIndex{
				Path:         idx.Path,
				Type:         logType,
				Name:         filepath.Base(idx.Path),
				ConfigFile:   "", // Rotated files don't have config
				IndexStatus:  IndexStatusNotIndexed,
				IsCompressed: strings.HasSuffix(idx.Path, ".gz") || strings.HasSuffix(idx.Path, ".bz2"),
				HasTimeRange: false,
			}
			allLogsMap[idx.Path] = logWithIndex
		}
	}

	// Now populate index information for all logs
	persistenceMap := make(map[string]*model.NginxLogIndex)
	for _, idx := range persistenceIndexes {
		persistenceMap[idx.Path] = idx
	}

	// Get analytics service for index status
	service := GetAnalyticsService()
	var indexStatus *IndexStatus
	if service != nil {
		status, err := service.GetIndexStatus()
		if err == nil {
			indexStatus = status
		}
	}

	// Create a map of indexed files for quick lookup
	indexedFiles := make(map[string]*FileStatus)
	if indexStatus != nil && indexStatus.Files != nil {
		for i := range indexStatus.Files {
			file := &indexStatus.Files[i]
			indexedFiles[file.Path] = file
		}
	}

	// Update index information for all logs
	for _, log := range allLogsMap {
		// Check if this file is currently being indexed
		if IsFileIndexing(log.Path) {
			log.IndexStatus = IndexStatusIndexing
		}

		// Check persistence data first (more accurate)
		if persistenceIndex, ok := persistenceMap[log.Path]; ok {
			// Set status based on persistence and current indexing state
			if log.IndexStatus != IndexStatusIndexing {
				if !persistenceIndex.LastIndexed.IsZero() {
					log.IndexStatus = IndexStatusIndexed
				}
			}
			
			// Use persistence data
			if !persistenceIndex.LastModified.IsZero() {
				log.LastModified = persistenceIndex.LastModified.Unix()
			}
			log.LastSize = persistenceIndex.LastSize
			if !persistenceIndex.LastIndexed.IsZero() {
				log.LastIndexed = persistenceIndex.LastIndexed.Unix()
			}
			if persistenceIndex.IndexStartTime != nil {
				log.IndexStartTime = persistenceIndex.IndexStartTime.Unix()
			}
			if persistenceIndex.IndexDuration != nil {
				log.IndexDuration = *persistenceIndex.IndexDuration
			}
			if persistenceIndex.TimeRangeStart != nil {
				log.TimeRangeStart = persistenceIndex.TimeRangeStart.Unix()
				log.HasTimeRange = true
			}
			if persistenceIndex.TimeRangeEnd != nil {
				log.TimeRangeEnd = persistenceIndex.TimeRangeEnd.Unix()
				log.HasTimeRange = true
			}
			log.DocumentCount = persistenceIndex.DocumentCount
		} else if fileStatus, ok := indexedFiles[log.Path]; ok {
			// Fallback to old index status system
			if log.IndexStatus != IndexStatusIndexing {
				log.IndexStatus = IndexStatusIndexed
			}
			if fileStatus.LastModified != 0 {
				log.LastModified = fileStatus.LastModified
			}
			log.LastSize = fileStatus.LastSize
			if fileStatus.LastIndexed != 0 {
				log.LastIndexed = fileStatus.LastIndexed
			}
			log.IsCompressed = fileStatus.IsCompressed
			log.HasTimeRange = fileStatus.HasTimeRange
			if fileStatus.TimeRangeStart != 0 {
				log.TimeRangeStart = fileStatus.TimeRangeStart
			}
			if fileStatus.TimeRangeEnd != 0 {
				log.TimeRangeEnd = fileStatus.TimeRangeEnd
			}
		}
	}

	// Convert map to slice
	allLogs := make([]*NginxLogWithIndex, 0, len(allLogsMap))
	for _, log := range allLogsMap {
		allLogs = append(allLogs, log)
	}
	
	// Group logs by their base log name
	logGroups := make(map[string][]*NginxLogWithIndex)
	for _, log := range allLogs {
		baseLogName := getBaseLogName(log.Path)
		logGroups[baseLogName] = append(logGroups[baseLogName], log)
	}
	
	result := make([]*NginxLogWithIndex, 0, len(logGroups))
	
	// Process each group
	for baseLogName, group := range logGroups {
		// Find the main log file (the one without rotation suffix)
		var mainLog *NginxLogWithIndex
		for _, log := range group {
			if isMainLogFile(log.Path, baseLogName) {
				mainLog = log
				break
			}
		}
		
		// If no main log file found, create one based on the base name
		if mainLog == nil {
			// Create a virtual main log based on the group's characteristics
			// Use the first log in the group as a template
			template := group[0]
			mainLog = &NginxLogWithIndex{
				Path:         baseLogName,
				Type:         template.Type,
				Name:         filepath.Base(baseLogName),
				ConfigFile:   template.ConfigFile,
				IndexStatus:  IndexStatusNotIndexed,
				IsCompressed: false,
				HasTimeRange: false,
			}
		}
		
		// Aggregate statistics from all files in the group
		aggregateLogGroupStats(mainLog, group)
		
		// Apply filters
		flag := true
		if len(filters) > 0 {
			for _, filter := range filters {
				if !filter(mainLog) {
					flag = false
					break
				}
			}
		}

		if flag {
			result = append(result, mainLog)
		}
	}
	
	return result
}

// getBaseLogName extracts the base log name from a rotated log file path
// Examples:
//   /var/log/nginx/access.log.1 -> /var/log/nginx/access.log
//   /var/log/nginx/access.log.10.gz -> /var/log/nginx/access.log
//   /var/log/nginx/access.20231201.gz -> /var/log/nginx/access.log
func getBaseLogName(logPath string) string {
	dir := filepath.Dir(logPath)
	filename := filepath.Base(logPath)
	
	// Remove .gz compression suffix if present
	if strings.HasSuffix(filename, ".gz") {
		filename = strings.TrimSuffix(filename, ".gz")
	}
	
	// Handle numbered rotation (access.log.1, access.log.2, etc.)
	// Use a more specific pattern to avoid matching date patterns like "20231201"
	if match := regexp.MustCompile(`^(.+)\.(\d{1,3})$`).FindStringSubmatch(filename); len(match) > 1 {
		// Only match if the number is reasonable for rotation (1-999)
		baseFilename := match[1]
		return filepath.Join(dir, baseFilename)
	}
	
	// Handle date-based rotation (access.20231201, access.2023-12-01, etc.)
	// Check if filename itself contains date patterns that we should strip
	// Example: access.2023-12-01 -> access.log, access.20231201 -> access.log
	parts := strings.Split(filename, ".")
	if len(parts) >= 2 {
		lastPart := parts[len(parts)-1]
		if isDatePattern(lastPart) {
			baseFilename := strings.Join(parts[:len(parts)-1], ".")
			// If the base doesn't end with .log, add it
			if !strings.HasSuffix(baseFilename, ".log") {
				baseFilename += ".log"
			}
			return filepath.Join(dir, baseFilename)
		}
	}
	
	// No rotation pattern found, return as-is
	return logPath
}

// isMainLogFile checks if the given path is the main log file (no rotation suffix)
func isMainLogFile(logPath, baseLogName string) bool {
	return logPath == baseLogName
}

// aggregateLogGroupStats aggregates statistics from all files in a log group
func aggregateLogGroupStats(aggregatedLog *NginxLogWithIndex, group []*NginxLogWithIndex) {
	var totalSize int64
	var totalDocuments uint64
	var earliestTimeStart *time.Time
	var latestTimeEnd *time.Time
	var mostRecentIndexed *time.Time
	var indexingInProgress bool
	var hasIndexedFiles bool
	var earliestIndexStartTime *time.Time
	var totalIndexDuration *int64
	
	for _, log := range group {
		// Aggregate file sizes
		totalSize += log.LastSize
		
		// Aggregate document counts
		totalDocuments += log.DocumentCount
		
		// Check for indexing status
		if log.IndexStatus == IndexStatusIndexing {
			indexingInProgress = true
		} else if log.IndexStatus == IndexStatusIndexed {
			hasIndexedFiles = true
		}
		
		// Find the most recent indexed time
		if log.LastIndexed != 0 {
			indexedTime := time.Unix(log.LastIndexed, 0)
			if mostRecentIndexed == nil || indexedTime.After(*mostRecentIndexed) {
				mostRecentIndexed = &indexedTime
			}
		}
		
		// Aggregate time ranges
		if log.TimeRangeStart != 0 {
			startTime := time.Unix(log.TimeRangeStart, 0)
			if earliestTimeStart == nil || startTime.Before(*earliestTimeStart) {
				earliestTimeStart = &startTime
			}
		}
		
		if log.TimeRangeEnd != 0 {
			endTime := time.Unix(log.TimeRangeEnd, 0)
			if latestTimeEnd == nil || endTime.After(*latestTimeEnd) {
				latestTimeEnd = &endTime
			}
		}
		
		// Use properties from the most recent file
		if log.LastModified != 0 && (aggregatedLog.LastModified == 0 || log.LastModified > aggregatedLog.LastModified) {
			aggregatedLog.LastModified = log.LastModified
		}
		
		// Find the EARLIEST IndexStartTime for the log group (when the group indexing started)
		if log.IndexStartTime != 0 {
			startTime := time.Unix(log.IndexStartTime, 0)
			if earliestIndexStartTime == nil || startTime.Before(*earliestIndexStartTime) {
				earliestIndexStartTime = &startTime
			}
		}
		
		// Sum up individual file durations to get total group duration
		if log.IndexDuration != 0 {
			if totalIndexDuration == nil {
				totalIndexDuration = new(int64)
			}
			*totalIndexDuration += log.IndexDuration
		}
	}
	
	// Set aggregated values
	if earliestIndexStartTime != nil {
		aggregatedLog.IndexStartTime = earliestIndexStartTime.Unix()
	}
	aggregatedLog.LastSize = totalSize
	aggregatedLog.DocumentCount = totalDocuments
	if mostRecentIndexed != nil {
		aggregatedLog.LastIndexed = mostRecentIndexed.Unix()
	}
	if totalIndexDuration != nil {
		aggregatedLog.IndexDuration = *totalIndexDuration
	}
	
	// Set index status based on group status
	if indexingInProgress {
		aggregatedLog.IndexStatus = IndexStatusIndexing
	} else if hasIndexedFiles {
		aggregatedLog.IndexStatus = IndexStatusIndexed
	} else {
		aggregatedLog.IndexStatus = IndexStatusNotIndexed
	}
	
	// Set time range
	if earliestTimeStart != nil && latestTimeEnd != nil {
		aggregatedLog.TimeRangeStart = earliestTimeStart.Unix()
		aggregatedLog.TimeRangeEnd = latestTimeEnd.Unix()
		aggregatedLog.HasTimeRange = true
	}
}