From ca0cb872aac1436b22cf884c723dc79ee69fa526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Airenas=20Vai=C4=8Di=C5=ABnas?= Date: Mon, 18 Sep 2023 16:32:09 +0300 Subject: [PATCH] Fix text cleaning procedure - do not add space Fixed panic in tag processor --- examples/docker-compose/.env.in | 1 + examples/docker-compose/config.yaml | 6 +++--- examples/docker-compose/docker-compose.yml | 8 ++++---- internal/pkg/processor/cleaner.go | 7 ++++++- internal/pkg/processor/cleaner_test.go | 5 +++-- internal/pkg/processor/normalizer.go | 4 ++-- internal/pkg/processor/numberReplace.go | 4 ++-- internal/pkg/processor/saveData.go | 4 ++-- internal/pkg/processor/tag.go | 4 ++-- internal/pkg/processor/urlReplacer.go | 4 ++-- internal/pkg/synthesizer/processor.go | 2 +- 11 files changed, 28 insertions(+), 21 deletions(-) diff --git a/examples/docker-compose/.env.in b/examples/docker-compose/.env.in index 114af7e..818908a 100644 --- a/examples/docker-compose/.env.in +++ b/examples/docker-compose/.env.in @@ -1,6 +1,7 @@ SERVICE_VERSION=${tts_version} SERVICE_PORT=${service_port} acronyms_version=${acronyms_version} +TAGGER_URL=http://tagger:8000/tag MONGO_USER=list MONGO_PASS=${mongo_pass} diff --git a/examples/docker-compose/config.yaml b/examples/docker-compose/config.yaml index d1a1dab..23f9c6f 100644 --- a/examples/docker-compose/config.yaml +++ b/examples/docker-compose/config.yaml @@ -25,13 +25,13 @@ acousticModel: hasVocoder: true validator: - maxChars: 1000 + maxChars: 2500 splitter: - maxChars: 50 + maxChars: 200 partRunner: - workers: 1 + workers: 5 suffixLoader: path: /suffixes diff --git a/examples/docker-compose/docker-compose.yml b/examples/docker-compose/docker-compose.yml index b64ea79..6de7445 100644 --- a/examples/docker-compose/docker-compose.yml +++ b/examples/docker-compose/docker-compose.yml @@ -28,7 +28,7 @@ services: - ./config.yaml:/app/config.yaml:ro text-clean: - image: airenas/tts-text-clean:0.2.197 + image: airenas/tts-text-clean:0.5.7 ports: - "8011:8000" @@ -41,7 +41,7 @@ services: - "8017:80" normalizer: - image: intelektikalt/text-normalizer:0.1.28 + image: intelektikalt/text-normalizer:0.1.33 restart: on-failure audioconverter: @@ -50,7 +50,7 @@ services: - "8013:8000" comparator: - image: airenas/resynth-validator:0.1.2 + image: intelektikalt/resynth-validator:0.1.8 ports: - "8014:3000" @@ -83,7 +83,7 @@ services: obscene: container_name: obscene - image: intelektikalt/obscene-filter:0.1.12 + image: intelektikalt/obscene-filter:0.1.14 ports: - "8016:3000" diff --git a/internal/pkg/processor/cleaner.go b/internal/pkg/processor/cleaner.go index 4f8ec22..ec6b6b7 100644 --- a/internal/pkg/processor/cleaner.go +++ b/internal/pkg/processor/cleaner.go @@ -67,9 +67,14 @@ func processCleanOutput(text string, textPart []*synthesizer.TTSTextPart) ([]str return []string{text}, nil } splits := strings.Split(text, splitIndicator) - if len(textPart) != len(splits) - 1 { + if len(textPart) != len(splits)-1 { return nil, fmt.Errorf("can't restore after clean, returned count of parts is not the same") } + for i, s := range splits { + if strings.HasSuffix(s, " ") { + splits[i] = s[:len(s)-1] + } + } return splits[1:], nil } diff --git a/internal/pkg/processor/cleaner_test.go b/internal/pkg/processor/cleaner_test.go index 83a6b48..c095e47 100644 --- a/internal/pkg/processor/cleaner_test.go +++ b/internal/pkg/processor/cleaner_test.go @@ -136,8 +136,9 @@ func Test_processCleanOutput(t *testing.T) { }{ {name: "empty", args: args{text: splitIndicator + "", textPart: []*synthesizer.TTSTextPart{{Text: ""}}}, want: []string{""}, wantErr: false}, {name: "one", args: args{text: splitIndicator + "olia", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}}}, want: []string{"olia"}, wantErr: false}, - {name: "one", args: args{text: splitIndicator + "olia" + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}, {Text: "cleaned olia2"}}}, want: []string{"olia", "olia2"}, wantErr: false}, - {name: "error", args: args{text: splitIndicator + "olia" + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}}}, wantErr: true}, + {name: "one", args: args{text: splitIndicator + "olia " + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}, {Text: "cleaned olia2"}}}, want: []string{"olia", "olia2"}, wantErr: false}, + {name: "unicode", args: args{text: splitIndicator + "ąąolia " + splitIndicator + "čęolia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}, {Text: "cleaned olia2"}}}, want: []string{"ąąolia", "čęolia2"}, wantErr: false}, + {name: "error", args: args{text: splitIndicator + "olia " + splitIndicator + "olia2", textPart: []*synthesizer.TTSTextPart{{Text: "olia"}}}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/pkg/processor/normalizer.go b/internal/pkg/processor/normalizer.go index c6e01c7..1768308 100644 --- a/internal/pkg/processor/normalizer.go +++ b/internal/pkg/processor/normalizer.go @@ -32,7 +32,7 @@ func (p *normalizer) Process(data *synthesizer.TTSData) error { return nil } defer goapp.Estimate("Normalize")() - txt := strings.Join(data.CleanedText, "") + txt := strings.Join(data.CleanedText, " ") utils.LogData("Input: ", txt) inData := &normRequestData{Orig: txt} var output normResponseData @@ -45,7 +45,7 @@ func (p *normalizer) Process(data *synthesizer.TTSData) error { if err != nil { return err } - utils.LogData("Output: ", strings.Join(data.NormalizedText, "")) + utils.LogData("Output: ", strings.Join(data.NormalizedText, " ")) return nil } diff --git a/internal/pkg/processor/numberReplace.go b/internal/pkg/processor/numberReplace.go index 9e55f58..70b3440 100644 --- a/internal/pkg/processor/numberReplace.go +++ b/internal/pkg/processor/numberReplace.go @@ -78,7 +78,7 @@ func (p *ssmlNumberReplace) Process(data *synthesizer.TTSData) error { return nil } res := "" - err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.Text, "")), &res) + err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.Text, " ")), &res) if err != nil { return err } @@ -132,7 +132,7 @@ func mapAccentsBack(new string, origArr []string) ([]string, error) { if wTo < len(alignIDs) { nTo = alignIDs[wTo] } - for nTo == -1 && wTo < len(alignIDs) { + for nTo == -1 && wTo < (len(alignIDs) - 1) { wTo++ nTo = alignIDs[wTo] } diff --git a/internal/pkg/processor/saveData.go b/internal/pkg/processor/saveData.go index 872a62f..e6ae2c0 100644 --- a/internal/pkg/processor/saveData.go +++ b/internal/pkg/processor/saveData.go @@ -45,9 +45,9 @@ func getText(data *synthesizer.TTSData, t utils.RequestTypeEnum) string { case utils.RequestOriginal: return data.OriginalText case utils.RequestCleaned: - return strings.Join(data.Text, "") + return strings.Join(data.Text, " ") case utils.RequestNormalized: - return strings.Join(data.TextWithNumbers, "") + return strings.Join(data.TextWithNumbers, " ") case utils.RequestUser: return data.OriginalText case utils.RequestOriginalSSML: diff --git a/internal/pkg/processor/tag.go b/internal/pkg/processor/tag.go index 638e2fd..cad6ecf 100644 --- a/internal/pkg/processor/tag.go +++ b/internal/pkg/processor/tag.go @@ -116,7 +116,7 @@ func NewTaggerAccents(urlStr string) (synthesizer.Processor, error) { func (p *taggerAccents) Process(data *synthesizer.TTSData) error { var output []*TaggedWord data.TextWithNumbers = []string{data.OriginalText} - err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.TextWithNumbers, "")), &output) + err := p.httpWrap.InvokeText(accent.ClearAccents(strings.Join(data.TextWithNumbers, " ")), &output) if err != nil { return err } @@ -229,7 +229,7 @@ func NewSSMLTagger(urlStr string) (synthesizer.Processor, error) { func (p *ssmlTagger) Process(data *synthesizer.TTSData) error { var output []*TaggedWord data.TextWithNumbers = addSpaces(data.TextWithNumbers) - txt := accent.ClearAccents(strings.Join(data.TextWithNumbers, "")) + txt := accent.ClearAccents(strings.Join(data.TextWithNumbers, " ")) err := p.httpWrap.InvokeText(txt, &output) if err != nil { return err diff --git a/internal/pkg/processor/urlReplacer.go b/internal/pkg/processor/urlReplacer.go index a959d55..8e2f737 100644 --- a/internal/pkg/processor/urlReplacer.go +++ b/internal/pkg/processor/urlReplacer.go @@ -37,13 +37,13 @@ func (p *urlReplacer) Process(data *synthesizer.TTSData) error { return nil } defer goapp.Estimate("URL replace")() - text := strings.Join(data.NormalizedText, "") + text := strings.Join(data.NormalizedText, " ") utils.LogData("Input: ", text) data.Text = nil for _, s := range data.NormalizedText{ data.Text = append(data.Text, p.replaceURLs(s)) } - utils.LogData("Output: ", strings.Join(data.Text, "")) + utils.LogData("Output: ", strings.Join(data.Text, " ")) return nil } diff --git a/internal/pkg/synthesizer/processor.go b/internal/pkg/synthesizer/processor.go index acb4d1f..4a32ac3 100644 --- a/internal/pkg/synthesizer/processor.go +++ b/internal/pkg/synthesizer/processor.go @@ -124,7 +124,7 @@ func mapResult(data *TTSData) (*api.Result, error) { res.RequestID = data.RequestID } if data.Input.OutputTextFormat == api.TextNormalized { - res.Text = strings.Join(data.TextWithNumbers, "") + res.Text = strings.Join(data.TextWithNumbers, " ") } else if data.Input.OutputTextFormat == api.TextTranscribed { res.Text = mapTranscribed(data) } else if data.Input.OutputTextFormat == api.TextAccented {