Merge pull request #65 from cstanislawski/preserve_folder_structure

Preserve folder structure

.github/workflows/ci.yaml

@@ -7,18 +7,25 @@ concurrency:
 on:
   pull_request:
     branches: [main]
+    paths-ignore:
+      - "**.md"
+      - "examples/**"
+      - "LICENSE.md"
 
 jobs:
-  build:
-    name: build
+  lint:
+    name: lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: docker/build-push-action@v6
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
-          context: .
-          file: ./docker/grafana-db-exporter/Dockerfile
-          push: false
+          go-version: 1.23.2
+      - uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.61.0
+          args: --timeout 5m
   test:
     name: test
     runs-on: ubuntu-latest
@@ -30,16 +37,13 @@ jobs:
           go-version: 1.23.2
       - name: Run go test
         run: go test ./...
-  lint:
-    name: lint
+  build:
+    name: build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 1.23.2
-      - uses: golangci/golangci-lint-action@v6
+      - uses: docker/build-push-action@v6
         with:
-          version: v1.61.0
-          args: --timeout 5m
+          context: .
+          file: ./docker/grafana-db-exporter/Dockerfile
+          push: false

README.md

@@ -32,6 +32,7 @@ Optional:
 - `DELETE_MISSING` - Whether to delete the dashboards from the target directory if they were not fetched (i.e. missing) from the Grafana instance, bool, defaults to `true`
 - `ADD_MISSING_NEWLINES` - Whether to add newlines to the end of the fetched dashboards, bool, defaults to `true`
 - `DRY_RUN` - Whether to run in dry run mode (run up to the point of committing the changes, but don't push), bool, defaults to `false`
+- `IGNORE_FOLDER_STRUCTURE` - Whether to ignore the Grafana folder structure and save all dashboards directly in `REPO_SAVE_PATH`, bool, defaults to `false`
 
 ### Examples
 

cmd/exporter/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -68,7 +69,7 @@ func run(ctx context.Context) error {
 	logger.Log.Info().Int("count", len(dashboards)).Msg("Fetched dashboards")
 
 	if cfg.DeleteMissing {
-		if err := deleteMissingDashboards(cfg.RepoSavePath, dashboards); err != nil {
+		if err := deleteMissingDashboards(cfg.RepoSavePath, dashboards, cfg); err != nil {
 			return fmt.Errorf("failed to delete missing dashboards: %w", err)
 		}
 	}
@@ -96,47 +97,92 @@ func run(ctx context.Context) error {
 	return nil
 }
 
-func deleteMissingDashboards(repoSavePath string, fetchedDashboards []grafana.Dashboard) error {
-	existingFiles, err := listDashboardFiles(repoSavePath)
+func deleteMissingDashboards(repoSavePath string, fetchedDashboards []grafana.Dashboard, cfg *config.Config) error {
+	existingFiles := make(map[string]bool)
+	fetchedPaths := make(map[string]bool)
+
+	err := filepath.Walk(repoSavePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && strings.HasSuffix(info.Name(), ".json") {
+			relPath, err := filepath.Rel(repoSavePath, path)
+			if err != nil {
+				return fmt.Errorf("failed to get relative path: %w", err)
+			}
+			existingFiles[relPath] = true
+		}
+		return nil
+	})
 	if err != nil {
-		return fmt.Errorf("failed to list existing dashboard files: %w", err)
+		return fmt.Errorf("failed to walk repository directory: %w", err)
 	}
 
-	fetchedUIDs := make(map[string]struct{})
 	for _, dashboard := range fetchedDashboards {
-		fetchedUIDs[dashboard.UID] = struct{}{}
+		relPath, err := filepath.Rel(
+			repoSavePath,
+			grafana.GetDashboardPath(repoSavePath, dashboard, cfg.IgnoreFolderStructure),
+		)
+		if err != nil {
+			return fmt.Errorf("failed to get relative path: %w", err)
+		}
+		fetchedPaths[relPath] = true
+	}
+
+	for existingPath := range existingFiles {
+		if !fetchedPaths[existingPath] {
+			fullPath := filepath.Join(repoSavePath, existingPath)
+			if err := os.Remove(fullPath); err != nil {
+				return fmt.Errorf("failed to delete file %s: %w", existingPath, err)
+			}
+			logger.Log.Info().Str("file", existingPath).Msg("Deleted missing dashboard file")
+		}
+	}
+
+	if cfg.IgnoreFolderStructure {
+		return nil
 	}
 
-	for _, file := range existingFiles {
-		fileUID := extractUIDFromFilename(file)
-		if _, exists := fetchedUIDs[fileUID]; !exists {
-			if err := os.Remove(filepath.Join(repoSavePath, file)); err != nil {
-				return fmt.Errorf("failed to delete file %s: %w", file, err)
+	err = filepath.Walk(repoSavePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if path == repoSavePath {
+			return nil
+		}
+		if info.IsDir() {
+			empty, err := isDirEmpty(path)
+			if err != nil {
+				return fmt.Errorf("failed to check if directory is empty: %w", err)
+			}
+			if empty {
+				if err := os.Remove(path); err != nil {
+					return fmt.Errorf("failed to remove empty directory %s: %w", path, err)
+				}
+				logger.Log.Info().Str("directory", path).Msg("Removed empty directory")
 			}
-			logger.Log.Info().Str("file", file).Msg("Deleted missing dashboard file")
 		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to clean up empty directories: %w", err)
 	}
 
 	return nil
 }
 
-func listDashboardFiles(dir string) ([]string, error) {
-	files, err := os.ReadDir(dir)
+func isDirEmpty(path string) (bool, error) {
+	f, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return false, err
 	}
+	defer f.Close()
 
-	var dashboardFiles []string
-	for _, file := range files {
-		if !file.IsDir() && strings.HasSuffix(file.Name(), ".json") {
-			dashboardFiles = append(dashboardFiles, file.Name())
-		}
+	_, err = f.Readdirnames(1)
+	if err == io.EOF {
+		return true, nil
 	}
-	return dashboardFiles, nil
-}
-
-func extractUIDFromFilename(filename string) string {
-	return strings.TrimSuffix(filename, ".json")
+	return false, err
 }
 
 func setupGitClient(cfg *config.Config) (*git.Client, error) {
@@ -155,36 +201,53 @@ func fetchDashboards(ctx context.Context, grafanaClient *grafana.Client) ([]graf
 }
 
 func saveDashboards(ctx context.Context, dashboards []grafana.Dashboard, cfg *config.Config) (int, error) {
-	logger.Log.Debug().Int("dashboardCount", len(dashboards)).Str("savePath", cfg.RepoSavePath).Msg("Saving dashboards")
+	logger.Log.Debug().
+		Int("dashboardCount", len(dashboards)).
+		Str("savePath", cfg.RepoSavePath).
+		Bool("ignoreFolderStructure", cfg.IgnoreFolderStructure).
+		Msg("Saving dashboards")
+
 	savedCount := 0
 	for _, dashboard := range dashboards {
 		select {
 		case <-ctx.Done():
 			return savedCount, ctx.Err()
 		default:
-			if err := saveDashboard(dashboard, cfg); err != nil {
+			fullPath := grafana.GetDashboardPath(cfg.RepoSavePath, dashboard, cfg.IgnoreFolderStructure)
+			dirPath := filepath.Dir(fullPath)
+
+			if err := os.MkdirAll(dirPath, 0755); err != nil {
+				return savedCount, fmt.Errorf("failed to create directory %s: %w", dirPath, err)
+			}
+
+			if err := saveDashboard(dashboard, fullPath, cfg); err != nil {
 				return savedCount, fmt.Errorf("failed to save dashboard %s: %w", dashboard.UID, err)
 			}
 			savedCount++
-			logger.Log.Debug().Str("dashboardUID", dashboard.UID).Msg("Dashboard saved")
+			logger.Log.Debug().
+				Str("dashboardUID", dashboard.UID).
+				Str("folder", dashboard.FolderTitle).
+				Bool("ignoredFolderStructure", cfg.IgnoreFolderStructure).
+				Msg("Dashboard saved")
 		}
 	}
 	return savedCount, nil
 }
 
-func saveDashboard(dashboard grafana.Dashboard, cfg *config.Config) error {
-	filePath := filepath.Join(cfg.RepoSavePath, fmt.Sprintf("%s.json", dashboard.UID))
+func saveDashboard(dashboard grafana.Dashboard, filePath string, cfg *config.Config) error {
 	logger.Log.Debug().Str("filePath", filePath).Msg("Saving dashboard to file")
 
 	data, err := json.MarshalIndent(dashboard.Data, "", "  ")
 	if err != nil {
 		return fmt.Errorf("failed to marshal dashboard data: %w", err)
 	}
 
-	if cfg.AddMissingNewlines {
-		if len(data) > 0 && data[len(data)-1] != '\n' {
-			data = append(data, '\n')
-		}
+	if cfg.AddMissingNewlines && len(data) > 0 && data[len(data)-1] != '\n' {
+		data = append(data, '\n')
+	}
+
+	if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
+		return fmt.Errorf("failed to create directories: %w", err)
 	}
 
 	if err := os.WriteFile(filePath, data, 0644); err != nil {

internal/config/config.go

@@ -34,8 +34,9 @@ type Config struct {
 	NumOfRetries   uint `env:"NUM_OF_RETRIES,default=3"`
 	RetriesBackoff uint `env:"RETRIES_BACKOFF,default=5"`
 
-	AddMissingNewlines bool `env:"ADD_MISSING_NEWLINES,default=true"`
-	DryRun             bool `env:"DRY_RUN,default=false"`
+	AddMissingNewlines    bool `env:"ADD_MISSING_NEWLINES,default=true"`
+	DryRun                bool `env:"DRY_RUN,default=false"`
+	IgnoreFolderStructure bool `env:"IGNORE_FOLDER_STRUCTURE,default=false"`
 }
 
 func Load() (*Config, error) {

internal/grafana/grafana.go

@@ -3,16 +3,21 @@ package grafana
 import (
 	"context"
 	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
 
 	"grafana-db-exporter/internal/logger"
 
 	"github.com/grafana-tools/sdk"
 )
 
 type Dashboard struct {
-	UID   string
-	Title string
-	Data  interface{}
+	UID         string
+	Title       string
+	FolderID    int
+	FolderTitle string
+	Data        interface{}
 }
 
 type Client struct {
@@ -30,6 +35,17 @@ func New(url, apiKey string) (*Client, error) {
 
 func (gc *Client) ListAndExportDashboards(ctx context.Context) ([]Dashboard, error) {
 	logger.Log.Debug().Msg("Starting dashboard list and export operation")
+
+	folders, err := gc.client.GetAllFolders(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch folders: %w", err)
+	}
+
+	folderMap := make(map[int]string)
+	for _, folder := range folders {
+		folderMap[folder.ID] = folder.Title
+	}
+
 	boardLinks, err := gc.client.SearchDashboards(ctx, "", false)
 	if err != nil {
 		return nil, fmt.Errorf("failed to search dashboards: %w", err)
@@ -38,20 +54,59 @@ func (gc *Client) ListAndExportDashboards(ctx context.Context) ([]Dashboard, err
 
 	var dashboards []Dashboard
 	for _, link := range boardLinks {
-		logger.Log.Debug().Str("dashboardUID", link.UID).Msg("Fetching dashboard")
+		logger.Log.Debug().
+			Str("dashboardUID", link.UID).
+			Int("folderID", link.FolderID).
+			Msg("Fetching dashboard")
+
 		board, _, err := gc.client.GetDashboardByUID(ctx, link.UID)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get dashboard by UID: %w", err)
 		}
-		logger.Log.Debug().Str("dashboardUID", link.UID).Str("title", board.Title).Msg("Dashboard retrieved")
+
+		var folderTitle string
+		if link.FolderID != 0 {
+			var ok bool
+			folderTitle, ok = folderMap[link.FolderID]
+			if !ok {
+				logger.Log.Warn().
+					Int("folderID", link.FolderID).
+					Str("dashboardUID", link.UID).
+					Msg("Folder not found, using ID as name")
+				folderTitle = fmt.Sprintf("folder-%d", link.FolderID)
+			}
+		}
 
 		dashboards = append(dashboards, Dashboard{
-			UID:   board.UID,
-			Title: board.Title,
-			Data:  board,
+			UID:         board.UID,
+			Title:       board.Title,
+			FolderID:    link.FolderID,
+			FolderTitle: folderTitle,
+			Data:        board,
 		})
+
+		logger.Log.Debug().
+			Str("dashboardUID", link.UID).
+			Str("title", board.Title).
+			Str("folder", folderTitle).
+			Msg("Dashboard retrieved")
 	}
 
 	logger.Log.Debug().Int("exportedDashboards", len(dashboards)).Msg("Completed dashboard list and export operation")
 	return dashboards, nil
 }
+
+func SanitizeFolderPath(path string) string {
+	invalidChars := regexp.MustCompile(`[<>:"/\\|?*\x00-\x1F]`)
+	sanitized := invalidChars.ReplaceAllString(path, "-")
+	return strings.TrimSpace(sanitized)
+}
+
+func GetDashboardPath(basePath string, dashboard Dashboard, ignoreFolderStructure bool) string {
+	if ignoreFolderStructure || dashboard.FolderID == 0 {
+		return filepath.Join(basePath, fmt.Sprintf("%s.json", dashboard.UID))
+	}
+
+	folderPath := SanitizeFolderPath(dashboard.FolderTitle)
+	return filepath.Join(basePath, folderPath, fmt.Sprintf("%s.json", dashboard.UID))
+}