Fix text cleaning procedure - do not add space

Fixed panic in tag processor
airenas · Sep 18, 2023 · ca0cb87 · ca0cb87
1 parent 98eeefa
commit ca0cb87
Show file tree

Hide file tree

Showing 11 changed files with 28 additions and 21 deletions.
diff --git a/examples/docker-compose/.env.in b/examples/docker-compose/.env.in
@@ -1,6 +1,7 @@
 SERVICE_VERSION=${tts_version}
 SERVICE_PORT=${service_port}
 acronyms_version=${acronyms_version}
+TAGGER_URL=http://tagger:8000/tag
 
 MONGO_USER=list
 MONGO_PASS=${mongo_pass}

diff --git a/examples/docker-compose/config.yaml b/examples/docker-compose/config.yaml
@@ -25,13 +25,13 @@ acousticModel:
   hasVocoder: true
 
 validator:
-  maxChars: 1000
+  maxChars: 2500
 
 splitter:
-  maxChars: 50
+  maxChars: 200
 
 partRunner:
-  workers: 1  
+  workers: 5  
 
 suffixLoader:
   path: /suffixes    
diff --git a/examples/docker-compose/docker-compose.yml b/examples/docker-compose/docker-compose.yml
@@ -28,7 +28,7 @@ services:
       - ./config.yaml:/app/config.yaml:ro  
 
   text-clean:
-    image: airenas/tts-text-clean:0.2.197
+    image: airenas/tts-text-clean:0.5.7
     ports:
       - "8011:8000"  
 
@@ -41,7 +41,7 @@ services:
       - "8017:80"     
 
   normalizer:
-    image: intelektikalt/text-normalizer:0.1.28
+    image: intelektikalt/text-normalizer:0.1.33
     restart: on-failure  
 
   audioconverter:
@@ -50,7 +50,7 @@ services:
       - "8013:8000"
 
   comparator:
-    image: airenas/resynth-validator:0.1.2
+    image: intelektikalt/resynth-validator:0.1.8
     ports:
       - "8014:3000"
 
@@ -83,7 +83,7 @@ services:
 
   obscene:
     container_name: obscene
-    image: intelektikalt/obscene-filter:0.1.12 
+    image: intelektikalt/obscene-filter:0.1.14
     ports:
       - "8016:3000"   
 

diff --git a/internal/pkg/processor/cleaner.go b/internal/pkg/processor/cleaner.go
@@ -67,9 +67,14 @@ func processCleanOutput(text string, textPart []*synthesizer.TTSTextPart) ([]str
 		return []string{text}, nil
 	}
 	splits := strings.Split(text, splitIndicator)
-	if len(textPart) != len(splits) - 1 {
+	if len(textPart) != len(splits)-1 {
 		return nil, fmt.Errorf("can't restore after clean, returned count of parts is not the same")
 	}
+	for i, s := range splits {
+		if strings.HasSuffix(s, " ") {
+			splits[i] = s[:len(s)-1]
+		}
+	}
 	return splits[1:], nil
 }
 

diff --git a/internal/pkg/processor/cleaner_test.go b/internal/pkg/processor/cleaner_test.go
@@ -136,8 +136,9 @@ func Test_processCleanOutput(t *testing.T) {
 	}{
 		{name: "empty", args: args{text: splitIndicator + "", textPart: []*synthesizer.TTSTextPart{{Text: ""}}}, want: []string{""}, wantErr: false},
 		{name: "one", args: args{text: splitIndicator + "olia", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}}}, want: []string{"olia"}, wantErr: false},
-		{name: "one", args: args{text: splitIndicator + "olia" + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}, {Text: "cleaned olia2"}}}, want: []string{"olia", "olia2"}, wantErr: false},
-		{name: "error", args: args{text: splitIndicator + "olia" + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}}}, wantErr: true},
+		{name: "one", args: args{text: splitIndicator + "olia " + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}, {Text: "cleaned olia2"}}}, want: []string{"olia", "olia2"}, wantErr: false},
+		{name: "unicode", args: args{text: splitIndicator + "ąąolia " + splitIndicator + "čęolia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}, {Text: "cleaned olia2"}}}, want: []string{"ąąolia", "čęolia2"}, wantErr: false},
+		{name: "error", args: args{text: splitIndicator + "olia " + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}}}, wantErr: true},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/internal/pkg/processor/normalizer.go b/internal/pkg/processor/normalizer.go
@@ -32,7 +32,7 @@ func (p *normalizer) Process(data *synthesizer.TTSData) error {
 		return nil
 	}
 	defer goapp.Estimate("Normalize")()
-	txt := strings.Join(data.CleanedText, "")
+	txt := strings.Join(data.CleanedText, " ")
 	utils.LogData("Input: ", txt)
 	inData := &normRequestData{Orig: txt}
 	var output normResponseData
@@ -45,7 +45,7 @@ func (p *normalizer) Process(data *synthesizer.TTSData) error {
 	if err != nil {
 		return err
 	}
-	utils.LogData("Output: ", strings.Join(data.NormalizedText, ""))
+	utils.LogData("Output: ", strings.Join(data.NormalizedText, " "))
 	return nil
 }
 

diff --git a/internal/pkg/processor/numberReplace.go b/internal/pkg/processor/numberReplace.go
@@ -78,7 +78,7 @@ func (p *ssmlNumberReplace) Process(data *synthesizer.TTSData) error {
 		return nil
 	}
 	res := ""
-	err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.Text, "")), &res)
+	err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.Text, " ")), &res)
 	if err != nil {
 		return err
 	}
@@ -132,7 +132,7 @@ func mapAccentsBack(new string, origArr []string) ([]string, error) {
 		if wTo < len(alignIDs) {
 			nTo = alignIDs[wTo]
 		}
-		for nTo == -1 && wTo < len(alignIDs) {
+		for nTo == -1 && wTo < (len(alignIDs) - 1) {
 			wTo++
 			nTo = alignIDs[wTo]
 		}

diff --git a/internal/pkg/processor/saveData.go b/internal/pkg/processor/saveData.go
@@ -45,9 +45,9 @@ func getText(data *synthesizer.TTSData, t utils.RequestTypeEnum) string {
 	case utils.RequestOriginal:
 		return data.OriginalText
 	case utils.RequestCleaned:
-		return strings.Join(data.Text, "")
+		return strings.Join(data.Text, " ")
 	case utils.RequestNormalized:
-		return strings.Join(data.TextWithNumbers, "")
+		return strings.Join(data.TextWithNumbers, " ")
 	case utils.RequestUser:
 		return data.OriginalText
 	case utils.RequestOriginalSSML:

diff --git a/internal/pkg/processor/tag.go b/internal/pkg/processor/tag.go
@@ -116,7 +116,7 @@ func NewTaggerAccents(urlStr string) (synthesizer.Processor, error) {
 func (p *taggerAccents) Process(data *synthesizer.TTSData) error {
 	var output []*TaggedWord
 	data.TextWithNumbers = []string{data.OriginalText}
-	err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.TextWithNumbers, "")), &output)
+	err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.TextWithNumbers, " ")), &output)
 	if err != nil {
 		return err
 	}
@@ -229,7 +229,7 @@ func NewSSMLTagger(urlStr string) (synthesizer.Processor, error) {
 func (p *ssmlTagger) Process(data *synthesizer.TTSData) error {
 	var output []*TaggedWord
 	data.TextWithNumbers = addSpaces(data.TextWithNumbers)
-	txt := accent.ClearAccents(strings.Join(data.TextWithNumbers, ""))
+	txt := accent.ClearAccents(strings.Join(data.TextWithNumbers, " "))
 	err := p.httpWrap.InvokeText(txt, &output)
 	if err != nil {
 		return err

diff --git a/internal/pkg/processor/urlReplacer.go b/internal/pkg/processor/urlReplacer.go
@@ -37,13 +37,13 @@ func (p *urlReplacer) Process(data *synthesizer.TTSData) error {
 		return nil
 	}
 	defer goapp.Estimate("URL replace")()
-	text := strings.Join(data.NormalizedText, "")
+	text := strings.Join(data.NormalizedText, " ")
 	utils.LogData("Input: ", text)
 	data.Text = nil
 	for _, s := range data.NormalizedText{
 		data.Text = append(data.Text, p.replaceURLs(s))
 	}
-	utils.LogData("Output: ", strings.Join(data.Text, ""))
+	utils.LogData("Output: ", strings.Join(data.Text, " "))
 	return nil
 }
 

diff --git a/internal/pkg/synthesizer/processor.go b/internal/pkg/synthesizer/processor.go
@@ -124,7 +124,7 @@ func mapResult(data *TTSData) (*api.Result, error) {
 			res.RequestID = data.RequestID
 		}
 		if data.Input.OutputTextFormat == api.TextNormalized {
-			res.Text = strings.Join(data.TextWithNumbers, "")
+			res.Text = strings.Join(data.TextWithNumbers, " ")
 		} else if data.Input.OutputTextFormat == api.TextTranscribed {
 			res.Text = mapTranscribed(data)
 		} else if data.Input.OutputTextFormat == api.TextAccented {