search_test.go 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267
  1. package nginx_log
  2. import (
  3. "context"
  4. "os"
  5. "path/filepath"
  6. "testing"
  7. "time"
  8. "github.com/dgraph-io/ristretto/v2"
  9. )
  10. func TestLogIndexer_SearchFunctionality(t *testing.T) {
  11. // Create temporary directory for test index
  12. tempDir, err := os.MkdirTemp("", "nginx_log_search_test")
  13. if err != nil {
  14. t.Fatalf("Failed to create temp dir: %v", err)
  15. }
  16. defer os.RemoveAll(tempDir)
  17. // Create test log file
  18. logFile := filepath.Join(tempDir, "access.log")
  19. logContent := `192.168.1.1 - - [10/Oct/2023:13:55:36 +0000] "GET /api/test HTTP/1.1" 200 1234 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15"
  20. 192.168.1.2 - - [10/Oct/2023:13:56:36 +0000] "POST /api/login HTTP/1.1" 401 567 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
  21. 192.168.1.3 - - [10/Oct/2023:13:57:36 +0000] "GET /api/data HTTP/1.1" 500 890 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"`
  22. err = os.WriteFile(logFile, []byte(logContent), 0644)
  23. if err != nil {
  24. t.Fatalf("Failed to write test log file: %v", err)
  25. }
  26. // Create indexer with custom index path for testing
  27. // We need to create it manually since NewLogIndexer uses config directory
  28. indexPath := filepath.Join(tempDir, "index")
  29. index, err := createOrOpenIndex(indexPath)
  30. if err != nil {
  31. t.Fatalf("Failed to create index: %v", err)
  32. }
  33. uaParser := NewSimpleUserAgentParser()
  34. parser := NewOptimizedLogParser(uaParser)
  35. // Initialize cache
  36. cache, err := ristretto.NewCache(&ristretto.Config[string, *CachedSearchResult]{
  37. NumCounters: 1e7, // number of keys to track frequency of (10M)
  38. MaxCost: 1 << 27, // maximum cost of cache (128MB)
  39. BufferItems: 64, // number of keys per Get buffer
  40. })
  41. if err != nil {
  42. t.Fatalf("Failed to create cache: %v", err)
  43. }
  44. indexer := &LogIndexer{
  45. index: index,
  46. indexPath: indexPath,
  47. parser: parser,
  48. logPaths: make(map[string]*LogFileInfo),
  49. indexBatch: 10000,
  50. cache: cache,
  51. // Note: We skip watcher initialization for testing
  52. }
  53. defer indexer.Close()
  54. // Add log path and index
  55. err = indexer.AddLogPath(logFile)
  56. if err != nil {
  57. t.Fatalf("Failed to add log path: %v", err)
  58. }
  59. err = indexer.IndexLogFile(logFile)
  60. if err != nil {
  61. t.Fatalf("Failed to index log file: %v", err)
  62. }
  63. // Wait a bit for indexing to complete
  64. time.Sleep(100 * time.Millisecond)
  65. // Test 1: Search all entries
  66. t.Run("Search all entries", func(t *testing.T) {
  67. req := &QueryRequest{
  68. Limit: 10,
  69. }
  70. result, err := indexer.SearchLogs(context.Background(), req)
  71. if err != nil {
  72. t.Fatalf("Search failed: %v", err)
  73. }
  74. if len(result.Entries) != 3 {
  75. t.Errorf("Expected 3 entries, got %d", len(result.Entries))
  76. }
  77. })
  78. // Test 2: Search by IP
  79. t.Run("Search by IP", func(t *testing.T) {
  80. req := &QueryRequest{
  81. IP: "192.168.1.1",
  82. Limit: 10,
  83. }
  84. result, err := indexer.SearchLogs(context.Background(), req)
  85. if err != nil {
  86. t.Fatalf("Search failed: %v", err)
  87. }
  88. if len(result.Entries) != 1 {
  89. t.Errorf("Expected 1 entry for IP search, got %d", len(result.Entries))
  90. }
  91. if len(result.Entries) > 0 && result.Entries[0].IP != "192.168.1.1" {
  92. t.Errorf("Expected IP 192.168.1.1, got %s", result.Entries[0].IP)
  93. }
  94. })
  95. // Test 3: Search by method
  96. t.Run("Search by method", func(t *testing.T) {
  97. req := &QueryRequest{
  98. Method: "POST",
  99. Limit: 10,
  100. }
  101. result, err := indexer.SearchLogs(context.Background(), req)
  102. if err != nil {
  103. t.Fatalf("Search failed: %v", err)
  104. }
  105. if len(result.Entries) != 1 {
  106. t.Errorf("Expected 1 entry for POST method, got %d", len(result.Entries))
  107. }
  108. if len(result.Entries) > 0 && result.Entries[0].Method != "POST" {
  109. t.Errorf("Expected method POST, got %s", result.Entries[0].Method)
  110. }
  111. })
  112. // Test 4: Search by status
  113. t.Run("Search by status", func(t *testing.T) {
  114. req := &QueryRequest{
  115. Status: []int{200},
  116. Limit: 10,
  117. }
  118. result, err := indexer.SearchLogs(context.Background(), req)
  119. if err != nil {
  120. t.Fatalf("Search failed: %v", err)
  121. }
  122. if len(result.Entries) != 1 {
  123. t.Errorf("Expected 1 entry for status 200, got %d", len(result.Entries))
  124. }
  125. if len(result.Entries) > 0 && result.Entries[0].Status != 200 {
  126. t.Errorf("Expected status 200, got %d", result.Entries[0].Status)
  127. }
  128. })
  129. // Test 5: Search by path
  130. t.Run("Search by path", func(t *testing.T) {
  131. req := &QueryRequest{
  132. Path: "/api/test",
  133. Limit: 10,
  134. }
  135. result, err := indexer.SearchLogs(context.Background(), req)
  136. if err != nil {
  137. t.Fatalf("Search failed: %v", err)
  138. }
  139. if len(result.Entries) != 1 {
  140. t.Errorf("Expected 1 entry for path /api/test, got %d", len(result.Entries))
  141. }
  142. if len(result.Entries) > 0 && result.Entries[0].Path != "/api/test" {
  143. t.Errorf("Expected path /api/test, got %s", result.Entries[0].Path)
  144. }
  145. })
  146. // Test 6: Complex search with multiple criteria
  147. t.Run("Complex search", func(t *testing.T) {
  148. req := &QueryRequest{
  149. Method: "GET",
  150. Status: []int{200, 500},
  151. Limit: 10,
  152. }
  153. result, err := indexer.SearchLogs(context.Background(), req)
  154. if err != nil {
  155. t.Fatalf("Search failed: %v", err)
  156. }
  157. // Should find 2 entries: one with status 200 and one with status 500, both GET
  158. if len(result.Entries) != 2 {
  159. t.Errorf("Expected 2 entries for complex search, got %d", len(result.Entries))
  160. }
  161. for _, entry := range result.Entries {
  162. if entry.Method != "GET" {
  163. t.Errorf("Expected method GET, got %s", entry.Method)
  164. }
  165. if entry.Status != 200 && entry.Status != 500 {
  166. t.Errorf("Expected status 200 or 500, got %d", entry.Status)
  167. }
  168. }
  169. })
  170. }
  171. func TestLogIndexer_GetIndexStatus(t *testing.T) {
  172. // Create temporary directory for test index
  173. tempDir, err := os.MkdirTemp("", "nginx_log_status_test")
  174. if err != nil {
  175. t.Fatalf("Failed to create temp dir: %v", err)
  176. }
  177. defer os.RemoveAll(tempDir)
  178. // Create indexer with custom index path for testing
  179. indexPath := filepath.Join(tempDir, "index")
  180. index, err := createOrOpenIndex(indexPath)
  181. if err != nil {
  182. t.Fatalf("Failed to create index: %v", err)
  183. }
  184. uaParser := NewSimpleUserAgentParser()
  185. parser := NewOptimizedLogParser(uaParser)
  186. // Initialize cache
  187. cache, err := ristretto.NewCache(&ristretto.Config[string, *CachedSearchResult]{
  188. NumCounters: 1e7, // number of keys to track frequency of (10M)
  189. MaxCost: 1 << 27, // maximum cost of cache (128MB)
  190. BufferItems: 64, // number of keys per Get buffer
  191. })
  192. if err != nil {
  193. t.Fatalf("Failed to create cache: %v", err)
  194. }
  195. indexer := &LogIndexer{
  196. index: index,
  197. indexPath: indexPath,
  198. parser: parser,
  199. logPaths: make(map[string]*LogFileInfo),
  200. indexBatch: 10000,
  201. cache: cache,
  202. // Note: We skip watcher initialization for testing
  203. }
  204. defer indexer.Close()
  205. // Get status for empty index
  206. status, err := indexer.GetIndexStatus()
  207. if err != nil {
  208. t.Fatalf("Failed to get index status: %v", err)
  209. }
  210. if status.DocumentCount != 0 {
  211. t.Errorf("Expected document count 0, got %v", status.DocumentCount)
  212. }
  213. if status.LogPathsCount != 0 {
  214. t.Errorf("Expected log paths count 0, got %v", status.LogPathsCount)
  215. }
  216. if status.TotalFiles != 0 {
  217. t.Errorf("Expected total files 0, got %v", status.TotalFiles)
  218. }
  219. if len(status.Files) != 0 {
  220. t.Errorf("Expected empty files array, got %d files", len(status.Files))
  221. }
  222. }