Test ungraceful node(s) reboot(test part)

tests/internal/params/const.go

@@ -1,6 +1,39 @@
 package systemtestsparams
 
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+)
+
 const (
 	// Label represents label that can be used for test cases selection.
 	Label = "system"
 )
+
+// BMCDetails structure to hold BMC details.
+type BMCDetails struct {
+	Username   string `json:"username"`
+	Password   string `json:"password"`
+	BMCAddress string `json:"bmc"`
+}
+
+// NodesBMCMap holds info about BMC connection for a specific node.
+type NodesBMCMap map[string]BMCDetails
+
+// Decode - method for envconfig package to parse JSON encoded environment variables.
+func (nad *NodesBMCMap) Decode(value string) error {
+	nodesAuthMap := new(map[string]BMCDetails)
+
+	err := json.Unmarshal([]byte(value), nodesAuthMap)
+
+	if err != nil {
+		log.Printf("Error to parse data %v", err)
+
+		return fmt.Errorf("invalid map json: %w", err)
+	}
+
+	*nad = *nodesAuthMap
+
+	return nil
+}

tests/internal/reboot/reboot.go

@@ -1,8 +1,12 @@
 package reboot
 
 import (
+	"context"
+	"fmt"
+	"sync"
 	"time"
 
+	"github.com/bmc-toolbox/bmclib/v2"
 	"github.com/golang/glog"
 	"github.com/openshift-kni/eco-goinfra/pkg/deployment"
 	"github.com/openshift-kni/eco-goinfra/pkg/pod"
@@ -14,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // SoftRebootNode executes systemctl reboot on a node.
@@ -29,6 +34,129 @@ func SoftRebootNode(nodeName string) error {
 	return nil
 }
 
+// powerCycleNode is used to inovke 'power cycle' command over BMC session.
+func powerCycleNode(
+	waitingGroup *sync.WaitGroup,
+	nodeName string,
+	client *bmclib.Client,
+	lock *sync.Mutex,
+	resMap map[string][]string) {
+	glog.V(90).Infof(
+		fmt.Sprintf("Starting go routine for %s", nodeName))
+
+	defer waitingGroup.Done()
+
+	glog.V(90).Infof(
+		fmt.Sprintf("[%s] Setting timeout for context", nodeName))
+
+	bmcCtx, cancel := context.WithTimeout(context.Background(), 6*time.Minute)
+	defer cancel()
+
+	glog.V(90).Infof(
+		fmt.Sprintf("[%s] Starting BMC session", nodeName))
+
+	err := client.Open(bmcCtx)
+
+	if err != nil {
+		glog.V(90).Infof("Failed to open BMC session to %s", nodeName)
+		lock.Lock()
+		if value, ok := resMap["error"]; !ok {
+			resMap["error"] = []string{fmt.Sprintf("Failed to open BMC session to %s", nodeName)}
+		} else {
+			value = append(value, fmt.Sprintf("Failed to open BMC session to %s", nodeName))
+			resMap["error"] = value
+		}
+		lock.Unlock()
+
+		return
+	}
+
+	defer client.Close(bmcCtx)
+
+	glog.V(90).Infof(fmt.Sprintf("Checking power state on %s", nodeName))
+
+	err = wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 5*time.Minute, true,
+		func(ctx context.Context) (bool, error) {
+			if _, err := client.SetPowerState(bmcCtx, "cycle"); err != nil {
+				glog.V(90).Infof(
+					fmt.Sprintf("Failed to power cycle %s -> %v", nodeName, err))
+
+				return false, err
+			}
+
+			glog.V(90).Infof(
+				fmt.Sprintf("Successfully powered cycle %s", nodeName))
+
+			return true, nil
+		})
+
+	if err != nil {
+		glog.V(90).Infof("Failed to reboot node %s due to %v", nodeName, err)
+		lock.Lock()
+		if value, ok := resMap["error"]; !ok {
+			resMap["error"] = []string{fmt.Sprintf("Failed to reboot node %s due to %v", nodeName, err)}
+		} else {
+			value = append(value, fmt.Sprintf("Failed to reboot node %s due to %v", nodeName, err))
+			resMap["error"] = value
+		}
+		lock.Unlock()
+
+		return
+	}
+}
+
+// HardRebootNodeBMC reboots node(s) via BMC
+// returns false if there were issues power cycling node(s).
+func HardRebootNodeBMC(nodesBMCMap systemtestsparams.NodesBMCMap) error {
+	clientOpts := []bmclib.Option{}
+
+	glog.V(90).Infof(fmt.Sprintf("BMC options %v", clientOpts))
+
+	glog.V(90).Infof(
+		fmt.Sprintf("NodesCredentialsMap:\n\t%#v", nodesBMCMap))
+
+	var bmcMap = make(map[string]*bmclib.Client)
+
+	for node, auth := range nodesBMCMap {
+		glog.V(90).Infof(
+			fmt.Sprintf("Creating BMC client for node %s", node))
+		glog.V(90).Infof(
+			fmt.Sprintf("BMC Auth %#v", auth))
+
+		bmcClient := bmclib.NewClient(auth.BMCAddress, auth.Username, auth.Password, clientOpts...)
+		bmcMap[node] = bmcClient
+	}
+
+	var (
+		waitGroup sync.WaitGroup
+		mapMutex  sync.Mutex
+		resultMap = make(map[string][]string)
+	)
+
+	for node, client := range bmcMap {
+		waitGroup.Add(1)
+
+		go powerCycleNode(&waitGroup, node, client, &mapMutex, resultMap)
+	}
+
+	glog.V(90).Infof("Wait for all reboots to finish")
+	waitGroup.Wait()
+
+	glog.V(90).Infof("Finished waiting for go routines to finish")
+
+	if len(resultMap) != 0 {
+		glog.V(90).Infof("There were errors power cycling nodes")
+
+		for _, msg := range resultMap["error"] {
+			glog.V(90).Infof("\tError: %v", msg)
+		}
+
+		return fmt.Errorf("there were errors power cycling nodes")
+	}
+
+	return nil
+}
+
 // HardRebootNode executes ipmitool chassis power cycle on a node.
 func HardRebootNode(nodeName string, nsName string) error {
 	err := systemtestsscc.AddPrivilegedSCCtoDefaultSA(nsName)

tests/reboot/internal/rebootconfig/config.go

@@ -0,0 +1,94 @@
+package rebootconfig
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/kelseyhightower/envconfig"
+	"github.com/openshift-kni/eco-gosystem/tests/internal/config"
+	systemtestsparams "github.com/openshift-kni/eco-gosystem/tests/internal/params"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	// PathToDefaultRebootParamsFile path to config file with default ran du parameters.
+	PathToDefaultRebootParamsFile = "./default.yaml"
+)
+
+// RebootConfig type keeps ran du configuration.
+type RebootConfig struct {
+	*config.GeneralConfig
+	NodesCredentialsMap systemtestsparams.NodesBMCMap `yaml:"nodes_bmc_map" envconfig:"ECO_SYSTEM_NODES_CREDENTIALS_MAP"`
+	//
+	ControlPlaneLabelStr string `yaml:"control_plane_nodes_label" envconfig:"ECO_REBOOT_CONTROL_PLANE_NODES_LABEL"`
+	MasterNodesLabelStr  string `yaml:"master_nodes_label" envconfig:"ECO_REBOOT_MASTER_NODES_LABEL"`
+	WorkerNodesLabelStr  string `yaml:"worker_nodes_label" envconfig:"ECO_REBOOT_WORKER_NODES_LABEL"`
+}
+
+// NewRebootConfig returns instance of RebootConfig config type.
+func NewRebootConfig() *RebootConfig {
+	log.Print("Creating new RebootConfig struct")
+
+	var rebootConf RebootConfig
+	rebootConf.GeneralConfig = config.NewConfig()
+
+	var confFile string
+
+	if fileFromEnv, exists := os.LookupEnv("ECO_SYSTEM_REBOOT_CONFIG_FILE_PATH"); !exists {
+		_, filename, _, _ := runtime.Caller(0)
+		baseDir := filepath.Dir(filename)
+		confFile = filepath.Join(baseDir, PathToDefaultRebootParamsFile)
+	} else {
+		confFile = fileFromEnv
+	}
+
+	log.Printf("Open config file %s", confFile)
+
+	err := readFile(&rebootConf, confFile)
+	if err != nil {
+		log.Printf("Error to read config file %s", confFile)
+
+		return nil
+	}
+
+	err = readEnv(&rebootConf)
+
+	if err != nil {
+		log.Print("Error to read environment variables")
+
+		return nil
+	}
+
+	return &rebootConf
+}
+
+func readFile(rebootConfig *RebootConfig, cfgFile string) error {
+	openedCfgFile, err := os.Open(cfgFile)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = openedCfgFile.Close()
+	}()
+
+	decoder := yaml.NewDecoder(openedCfgFile)
+	err = decoder.Decode(&rebootConfig)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func readEnv(rebootConfig *RebootConfig) error {
+	err := envconfig.Process("", rebootConfig)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

tests/reboot/internal/rebootconfig/default.yaml

@@ -0,0 +1,10 @@
+---
+# default configurations.
+verbose_level: 0
+dump_failed_tests: false
+reports_dump_dir: "/tmp/reports"
+polarion_report: true
+dry_run: false
+control_plane_nodes_label: "node-role.kubernetes.io/control-plane=''"
+master_nodes_label: "node-role.kubernetes.io/master=''"
+worker_nodes_label: "node-role.kubernetes.io/worker=''"

tests/reboot/internal/rebootinittools/rebootinittools.go

@@ -0,0 +1,21 @@
+package rebootinittools
+
+import (
+	"github.com/openshift-kni/eco-goinfra/pkg/clients"
+	"github.com/openshift-kni/eco-gosystem/tests/internal/inittools"
+	"github.com/openshift-kni/eco-gosystem/tests/reboot/internal/rebootconfig"
+)
+
+var (
+	// APIClient provides API access to cluster.
+	APIClient *clients.Settings
+	// RebootTestConfig provides access to tests configuration parameters.
+	RebootTestConfig *rebootconfig.RebootConfig
+)
+
+// init loads all variables automatically when this package is imported. Once package is imported a user has full
+// access to all vars within init function. It is recommended to import this package using dot import.
+func init() {
+	RebootTestConfig = rebootconfig.NewRebootConfig()
+	APIClient = inittools.APIClient
+}

tests/reboot/internal/rebootparams/const.go

@@ -0,0 +1,22 @@
+package rebootparams
+
+const (
+	// Label is used for 'reboot' test cases selection.
+	Label = "reboot"
+
+	// LabelValidateReboot is used to select tests that reboot cluster.
+	LabelValidateReboot = "validate_reboots"
+
+	// RebootLogLevel configures logging level for reboot related tests.
+	RebootLogLevel = 90
+)
+
+// BMCDetails structure to hold BMC details.
+type BMCDetails struct {
+	Username   string `json:"username"`
+	Password   string `json:"password"`
+	BMCAddress string `json:"bmc"`
+}
+
+// NodesBMCMap holds info about BMC connection for a specific node.
+type NodesBMCMap map[string]BMCDetails

tests/reboot/internal/rebootparams/rebootvars.go

@@ -0,0 +1,22 @@
+package rebootparams
+
+import (
+	systemtestsparams "github.com/openshift-kni/eco-gosystem/tests/internal/params"
+	"github.com/openshift-kni/k8sreporter"
+	v1 "k8s.io/api/core/v1"
+)
+
+var (
+	// Labels represents the range of labels that can be used for test cases selection.
+	Labels = []string{systemtestsparams.Label, Label}
+
+	// ReporterNamespacesToDump tells to the reporter from where to collect logs.
+	ReporterNamespacesToDump = map[string]string{
+		"openshift-machine-api": "openshift-machine-api",
+	}
+
+	// ReporterCRDsToDump tells to the reporter what CRs to dump.
+	ReporterCRDsToDump = []k8sreporter.CRData{
+		{Cr: &v1.PodList{}},
+	}
+)