integrated and enabled scripted risk rules

cmd/raa/main.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"io"
 	"os"
 	"sort"
@@ -34,23 +34,23 @@ func main() {
 		}
 	}
 
-	//	_ = os.WriteFile("raa_in.json", data, 0644)
+	// _ = os.WriteFile("raa_in.yaml", data, 0644)
 
 	var input types.Model
-	parseError := json.Unmarshal(data, &input)
+	parseError := yaml.Unmarshal(data, &input)
 	if parseError != nil {
 		_, _ = fmt.Fprintf(os.Stderr, "failed to parse model: %v\n", parseError)
 		os.Exit(-2)
 	}
 
 	text := CalculateRAA(&input)
-	outData, marshalError := json.MarshalIndent(input, "", "  ")
+	outData, marshalError := yaml.Marshal(input)
 	if marshalError != nil {
 		_, _ = fmt.Fprintf(os.Stderr, "failed to print model: %v\n", marshalError)
 		os.Exit(-2)
 	}
 
-	//	_ = os.WriteFile("raa_out.json", outData, 0644)
+	// _ = os.WriteFile("raa_out.yaml", outData, 0644)
 
 	var outputFile io.Writer = os.Stdout
 	if len(*outputFilename) > 0 {

cmd/script/main.go

@@ -3,13 +3,22 @@ package main
 import (
 	"fmt"
 	"github.com/threagile/threagile/pkg/script"
+	"github.com/threagile/threagile/pkg/security/risks"
 	"github.com/threagile/threagile/pkg/security/types"
 	"gopkg.in/yaml.v3"
 	"os"
 	"path/filepath"
 )
 
 func main() {
+	rules, loadError := risks.GetScriptRiskRules()
+	if loadError != nil {
+		fmt.Printf("error loading risk rules: %v\n", loadError)
+		return
+	}
+
+	_ = rules
+
 	scriptFilename := filepath.Clean(filepath.Join("test", "risk-category.yaml"))
 	ruleData, ruleReadError := os.ReadFile(scriptFilename)
 	if ruleReadError != nil {

pkg/model/custom-risk-category.go

@@ -2,7 +2,6 @@ package model
 
 import (
 	"fmt"
-	"github.com/threagile/threagile/pkg/security/risks"
 	"strings"
 
 	"github.com/threagile/threagile/pkg/security/types"
@@ -46,9 +45,9 @@ func (what *CustomRiskCategory) GenerateRisks(parsedModel *types.Model) ([]*type
 	return generatedRisks, nil
 }
 
-func LoadCustomRiskRules(pluginFiles []string, reporter types.ProgressReporter) risks.RiskRules {
+func LoadCustomRiskRules(pluginFiles []string, reporter types.ProgressReporter) types.RiskRules {
 	customRiskRuleList := make([]string, 0)
-	customRiskRules := make(risks.RiskRules)
+	customRiskRules := make(types.RiskRules)
 	if len(pluginFiles) > 0 {
 		reporter.Info("Loading custom risk rules:", strings.Join(pluginFiles, ", "))
 

pkg/model/parse.go

@@ -4,15 +4,14 @@ import (
 	"fmt"
 	"github.com/threagile/threagile/pkg/common"
 	"github.com/threagile/threagile/pkg/input"
-	"github.com/threagile/threagile/pkg/security/risks"
 	"github.com/threagile/threagile/pkg/security/types"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
 )
 
-func ParseModel(config *common.Config, modelInput *input.Model, builtinRiskRules risks.RiskRules, customRiskRules risks.RiskRules) (*types.Model, error) {
+func ParseModel(config *common.Config, modelInput *input.Model, builtinRiskRules types.RiskRules, customRiskRules types.RiskRules) (*types.Model, error) {
 	technologies := make(types.TechnologyMap)
 	technologiesLoadError := technologies.LoadWithConfig(config, "technologies.yaml")
 	if technologiesLoadError != nil {

pkg/model/parse_test.go

@@ -12,12 +12,11 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/threagile/threagile/pkg/input"
-	"github.com/threagile/threagile/pkg/security/risks"
 	"github.com/threagile/threagile/pkg/security/types"
 )
 
 func TestDefaultInputNotFail(t *testing.T) {
-	parsedModel, err := ParseModel(&common.Config{}, createInputModel(make(map[string]input.TechnicalAsset), make(map[string]input.DataAsset)), make(risks.RiskRules), make(risks.RiskRules))
+	parsedModel, err := ParseModel(&common.Config{}, createInputModel(make(map[string]input.TechnicalAsset), make(map[string]input.DataAsset)), make(types.RiskRules), make(types.RiskRules))
 
 	assert.NoError(t, err)
 	assert.NotNil(t, parsedModel)
@@ -27,7 +26,7 @@ func TestInferConfidentiality_NotSet_NoOthers_ExpectTODO(t *testing.T) {
 	ta := make(map[string]input.TechnicalAsset)
 	da := make(map[string]input.DataAsset)
 
-	_, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(risks.RiskRules), make(risks.RiskRules))
+	_, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(types.RiskRules), make(types.RiskRules))
 	// TODO: rename test and check if everyone agree that by default it should be public if there are no other assets
 
 	assert.NoError(t, err)
@@ -58,7 +57,7 @@ func TestInferConfidentiality_ExpectHighestConfidentiality(t *testing.T) {
 	taWithPublicConfidentialityDataAsset.DataAssetsProcessed = append(taWithPublicConfidentialityDataAsset.DataAssetsProcessed, daPublicConfidentiality.ID)
 	ta[taWithPublicConfidentialityDataAsset.ID] = taWithPublicConfidentialityDataAsset
 
-	parsedModel, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(risks.RiskRules), make(risks.RiskRules))
+	parsedModel, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(types.RiskRules), make(types.RiskRules))
 
 	assert.NoError(t, err)
 	assert.Equal(t, types.Confidential, parsedModel.TechnicalAssets[taWithConfidentialConfidentialityDataAsset.ID].Confidentiality)
@@ -70,7 +69,7 @@ func TestInferIntegrity_NotSet_NoOthers_ExpectTODO(t *testing.T) {
 	ta := make(map[string]input.TechnicalAsset)
 	da := make(map[string]input.DataAsset)
 
-	_, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(risks.RiskRules), make(risks.RiskRules))
+	_, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(types.RiskRules), make(types.RiskRules))
 	// TODO: rename test and check if everyone agree that by default it should be public if there are no other assets
 
 	assert.NoError(t, err)
@@ -101,7 +100,7 @@ func TestInferIntegrity_ExpectHighestIntegrity(t *testing.T) {
 	taWithArchiveIntegrityDataAsset.DataAssetsProcessed = append(taWithArchiveIntegrityDataAsset.DataAssetsProcessed, daArchiveIntegrity.ID)
 	ta[taWithArchiveIntegrityDataAsset.ID] = taWithArchiveIntegrityDataAsset
 
-	parsedModel, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(risks.RiskRules), make(risks.RiskRules))
+	parsedModel, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(types.RiskRules), make(types.RiskRules))
 
 	assert.NoError(t, err)
 	assert.Equal(t, types.Critical, parsedModel.TechnicalAssets[taWithCriticalIntegrityDataAsset.ID].Integrity)
@@ -113,7 +112,7 @@ func TestInferAvailability_NotSet_NoOthers_ExpectTODO(t *testing.T) {
 	ta := make(map[string]input.TechnicalAsset)
 	da := make(map[string]input.DataAsset)
 
-	_, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(risks.RiskRules), make(risks.RiskRules))
+	_, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(types.RiskRules), make(types.RiskRules))
 
 	assert.NoError(t, err)
 }
@@ -143,7 +142,7 @@ func TestInferAvailability_ExpectHighestAvailability(t *testing.T) {
 	taWithArchiveAvailabilityDataAsset.DataAssetsProcessed = append(taWithArchiveAvailabilityDataAsset.DataAssetsProcessed, daArchiveAvailability.ID)
 	ta[taWithArchiveAvailabilityDataAsset.ID] = taWithArchiveAvailabilityDataAsset
 
-	parsedModel, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(risks.RiskRules), make(risks.RiskRules))
+	parsedModel, err := ParseModel(&common.Config{}, createInputModel(ta, da), make(types.RiskRules), make(types.RiskRules))
 
 	assert.NoError(t, err)
 	assert.Equal(t, types.Critical, parsedModel.TechnicalAssets[taWithCriticalAvailabilityDataAsset.ID].Availability)

pkg/model/read.go

@@ -15,8 +15,8 @@ type ReadResult struct {
 	ModelInput       *input.Model
 	ParsedModel      *types.Model
 	IntroTextRAA     string
-	BuiltinRiskRules risks.RiskRules
-	CustomRiskRules  risks.RiskRules
+	BuiltinRiskRules types.RiskRules
+	CustomRiskRules  types.RiskRules
 }
 
 func (what ReadResult) ExplainRisk(cfg *common.Config, risk string, reporter common.DefaultProgressReporter) error {
@@ -73,7 +73,7 @@ func ReadAndAnalyzeModel(config *common.Config, progressReporter types.ProgressR
 	}, nil
 }
 
-func applyRiskGeneration(parsedModel *types.Model, rules risks.RiskRules,
+func applyRiskGeneration(parsedModel *types.Model, rules types.RiskRules,
 	skipRiskRules []string,
 	progressReporter types.ProgressReporter) {
 	progressReporter.Info("Applying risk generation")

pkg/model/runner.go

@@ -4,8 +4,8 @@ package model
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"os"
 	"os/exec"
 )
@@ -61,9 +61,9 @@ func (p *runner) Run(in any, out any, parameters ...string) error {
 		return startError
 	}
 
-	inData, inError := json.MarshalIndent(p.In, "", "  ")
+	inData, inError := yaml.Marshal(p.In)
 	if inError != nil {
-		return inError
+		return fmt.Errorf("error encoding input data: %w", inError)
 	}
 
 	_, writeError := stdin.Write(inData)
@@ -83,10 +83,12 @@ func (p *runner) Run(in any, out any, parameters ...string) error {
 	}
 
 	stdout := stdoutBuf.Bytes()
-	unmarshalError := json.Unmarshal(stdout, &p.Out)
+	unmarshalError := yaml.Unmarshal(stdout, p.Out)
 	if unmarshalError != nil {
 		return unmarshalError
 	}
 
+	// _ = os.WriteFile(fmt.Sprintf("%v.yaml", rand.Int31()), stdout, 0644)
+
 	return nil
 }

pkg/report/report.go

@@ -57,7 +57,7 @@ func (r *pdfReporter) WriteReportPDF(reportFilename string,
 	buildTimestamp string,
 	modelHash string,
 	introTextRAA string,
-	customRiskRules risks.RiskRules,
+	customRiskRules types.RiskRules,
 	tempFolder string,
 	model *types.Model) error {
 	defer func() {
@@ -4001,7 +4001,7 @@ func (r *pdfReporter) createSharedRuntimes(parsedModel *types.Model) {
 	}
 }
 
-func (r *pdfReporter) createRiskRulesChecked(parsedModel *types.Model, modelFilename string, skipRiskRules []string, buildTimestamp string, modelHash string, customRiskRules risks.RiskRules) {
+func (r *pdfReporter) createRiskRulesChecked(parsedModel *types.Model, modelFilename string, skipRiskRules []string, buildTimestamp string, modelHash string, customRiskRules types.RiskRules) {
 	r.pdf.SetTextColor(0, 0, 0)
 	title := "Risk Rules Checked by Threagile"
 	r.addHeadline(title, false)

pkg/script/common/scope.go

@@ -38,12 +38,12 @@ func (what *Scope) Init(risk *types.RiskCategory, methods map[string]Statement)
 
 func (what *Scope) SetModel(model *types.Model) error {
 	if model != nil {
-		data, marshalError := json.Marshal(model)
+		data, marshalError := yaml.Marshal(model)
 		if marshalError != nil {
 			return marshalError
 		}
 
-		unmarshalError := json.Unmarshal(data, &what.Model)
+		unmarshalError := yaml.Unmarshal(data, &what.Model)
 		if unmarshalError != nil {
 			return unmarshalError
 		}

pkg/script/risk-rule.go

@@ -1,16 +1,18 @@
 package script
 
 import (
+	"embed"
 	"fmt"
 	"github.com/threagile/threagile/pkg/input"
-	"github.com/threagile/threagile/pkg/security/risks"
 	"github.com/threagile/threagile/pkg/security/types"
 	"gopkg.in/yaml.v3"
+	"io/fs"
+	"path/filepath"
 	"strings"
 )
 
 type RiskRule struct {
-	risks.RiskRule
+	types.RiskRule
 	category      types.RiskCategory
 	supportedTags []string
 	script        *Script
@@ -81,3 +83,31 @@ func (what *RiskRule) GenerateRisks(parsedModel *types.Model) ([]*types.Risk, er
 
 	return newRisks, nil
 }
+
+func (what *RiskRule) Load(fileSystem embed.FS, path string, entry fs.DirEntry) error {
+	if entry.IsDir() {
+		return nil
+	}
+
+	loadError := what.loadRiskRule(fileSystem, path)
+	if loadError != nil {
+		return loadError
+	}
+
+	return nil
+}
+
+func (what *RiskRule) loadRiskRule(fileSystem embed.FS, filename string) error {
+	scriptFilename := filepath.Clean(filename)
+	ruleData, ruleReadError := fileSystem.ReadFile(scriptFilename)
+	if ruleReadError != nil {
+		return fmt.Errorf("error reading risk category: %w\n", ruleReadError)
+	}
+
+	_, parseError := what.ParseFromData(ruleData)
+	if parseError != nil {
+		return fmt.Errorf("error parsing scripts from %q: %w\n", scriptFilename, parseError)
+	}
+
+	return nil
+}

pkg/security/risks/risks.go

@@ -1,12 +1,17 @@
 package risks
 
 import (
+	"embed"
+	"fmt"
+	"github.com/threagile/threagile/pkg/script"
 	"github.com/threagile/threagile/pkg/security/risks/builtin"
+	"github.com/threagile/threagile/pkg/security/types"
+	"io/fs"
 )
 
-func GetBuiltInRiskRules() RiskRules {
-	rules := make(RiskRules)
-	for _, rule := range []RiskRule{
+func GetBuiltInRiskRules() types.RiskRules {
+	rules := make(types.RiskRules)
+	for _, rule := range []types.RiskRule{
 		builtin.NewAccidentalSecretLeakRule(),
 		builtin.NewCodeBackdooringRule(),
 		builtin.NewContainerBaseImageBackdooringRule(),
@@ -53,5 +58,57 @@ func GetBuiltInRiskRules() RiskRules {
 		rules[rule.Category().ID] = rule
 	}
 
+	scriptRules, scriptError := GetScriptRiskRules()
+	if scriptError != nil {
+		fmt.Printf("error loading script risk rules: %v\n", scriptError)
+		return rules
+	}
+
+	for id, rule := range scriptRules {
+		builtinRule, ok := rules[id]
+		if ok && builtinRule != nil {
+			fmt.Printf("WARNING: script risk rule %q shadows built-in risk rule\n", id)
+		}
+
+		rules[id] = rule
+	}
+
 	return rules
 }
+
+//go:embed scripts/*.yaml
+var ruleScripts embed.FS
+
+type RiskRules types.RiskRules
+
+func GetScriptRiskRules() (RiskRules, error) {
+	return make(RiskRules).LoadRiskRules()
+}
+
+func (what RiskRules) LoadRiskRules() (RiskRules, error) {
+	fileSystem := ruleScripts
+	walkError := fs.WalkDir(fileSystem, "scripts", func(path string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		newRule := new(script.RiskRule).Init()
+		loadError := newRule.Load(fileSystem, path, entry)
+		if loadError != nil {
+			return loadError
+		}
+
+		if newRule.Category().ID == "" {
+			return nil
+		}
+
+		what[newRule.Category().ID] = newRule
+		return nil
+	})
+
+	if walkError != nil {
+		return nil, walkError
+	}
+
+	return what, nil
+}