Added automatic rlimit handling

Signed-off-by: AbstractionFactory <[email protected]>

backend/cmd/generate/main.go

@@ -87,6 +87,11 @@ func main() {
 
 	ctx := context.Background()
 
+	if err := setRLimit(ctx, mainLogger); err != nil {
+		mainLogger.Error(ctx, err.Error())
+		os.Exit(1)
+	}
+
 	approvedLicenses, err := readLicensesFile(ctx, licensesFile)
 	if err != nil {
 		mainLogger.Error(ctx, err.Error())

backend/cmd/generate/rlimit_nonwindows.go

@@ -0,0 +1,21 @@
+//go:build !windows
+
+package main
+
+import (
+	"context"
+	"syscall"
+
+	"github.com/opentofu/libregistry/logger"
+)
+
+func setRLimit(ctx context.Context, log logger.Logger) error {
+	log.Info(ctx, "Setting maximum number of file descriptors to 50000...")
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &syscall.Rlimit{
+		Cur: 50000,
+		Max: 50000,
+	}); err != nil {
+		log.Warn(ctx, "Failed to set rlimit, generation may fail on larger repositories (%v)", err)
+	}
+	return nil
+}

backend/cmd/generate/rlimit_windows.go

@@ -0,0 +1,7 @@
+//go:build windows
+
+package main
+
+func setRLimit(ctx context.Context, log logger.Logger) error {
+	return nil
+}