nginx-ui/cmd/ngx_dir_index/ngx_dir_index.go

//go:generate go run .
package main

import (
	"encoding/json"
	"net/http"
	"os"
	"runtime"
	"strings"

	"path/filepath"

	"github.com/uozi-tech/cosy/logger"
	"golang.org/x/net/html"
)

type Directive struct {
	Links []string `json:"links"`
}

const (
	targetPath = "internal/nginx/nginx_directives.json"
	nginxURL   = "https://nginx.org/en/docs/dirindex.html"
)

func main() {
	logger.Init("release")

	_, file, _, ok := runtime.Caller(0)
	if !ok {
		logger.Error("Unable to get the current file")
		return
	}
	basePath := filepath.Join(filepath.Dir(file), "../../")

	outputPath := filepath.Join(basePath, targetPath)
	// Fetch page content
	resp, err := http.Get(nginxURL)
	if err != nil {
		logger.Errorf("fetching page: %v", err)
		return
	}
	defer resp.Body.Close()

	// Parse HTML
	doc, err := html.Parse(resp.Body)
	if err != nil {
		logger.Errorf("parsing HTML: %v", err)
		return
	}

	// Change storage structure to map
	directives := make(map[string]Directive)

	// Find node with id="content"
	var content *html.Node
	var findContent func(*html.Node)
	findContent = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "div" {
			for _, attr := range n.Attr {
				if attr.Key == "id" && attr.Val == "content" {
					content = n
					return
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			findContent(c)
		}
	}
	findContent(doc)

	// Extract all a tags from content
	if content != nil {
		var extractLinks func(*html.Node)
		extractLinks = func(n *html.Node) {
			if n.Type == html.ElementNode && n.Data == "a" {
				var href string
				for _, attr := range n.Attr {
					if attr.Key == "href" {
						href = attr.Val
						break
					}
				}
				if href != "" && n.FirstChild != nil {
					name := strings.TrimSpace(n.FirstChild.Data)
					if name != "" {
						fullLink := "https://nginx.org/en/docs/" + href
						directive, exists := directives[name]
						if !exists {
							directives[name] = Directive{
								Links: []string{fullLink},
							}
						} else {
							// Check if link already exists to avoid duplicates
							linkExists := false
							for _, existingLink := range directive.Links {
								if existingLink == fullLink {
									linkExists = true
									break
								}
							}
							if !linkExists {
								directive.Links = append(directive.Links, fullLink)
								directives[name] = directive
							}
						}
					}
				}
			}
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				extractLinks(c)
			}
		}
		extractLinks(content)
	}

	// Write results to JSON file
	jsonData, err := json.MarshalIndent(directives, "", "  ")
	if err != nil {
		logger.Errorf("marshaling JSON: %v", err)
		return
	}

	err = os.WriteFile(outputPath, jsonData, 0644)
	if err != nil {
		logger.Errorf("writing file: %v", err)
		return
	}

	logger.Infof("Successfully parsed %d directives and saved to %s\n", len(directives), targetPath)
}