From 31a16af954d9f47515e71f00562bcb46bfef96c3 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 14 Nov 2023 15:56:23 -0800 Subject: [PATCH] cleanup --- pkg/config/uploads.go | 1 + pkg/pipeline/debug.go | 3 +-- pkg/pipeline/sink/segments.go | 3 +-- pkg/pipeline/sink/uploader/s3.go | 2 +- pkg/pipeline/sink/uploader/uploader.go | 14 +++++++++----- pkg/service/handler.go | 6 ++---- pkg/service/process.go | 7 +++---- pkg/service/service.go | 5 ++--- pkg/stats/{handler_monitor.go => handler.go} | 14 ++++++++++++++ 9 files changed, 34 insertions(+), 21 deletions(-) rename pkg/stats/{handler_monitor.go => handler.go} (85%) diff --git a/pkg/config/uploads.go b/pkg/config/uploads.go index 84b16dcd..8360f5bc 100644 --- a/pkg/config/uploads.go +++ b/pkg/config/uploads.go @@ -18,6 +18,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" ) diff --git a/pkg/pipeline/debug.go b/pkg/pipeline/debug.go index 641380c7..bed3b3e8 100644 --- a/pkg/pipeline/debug.go +++ b/pkg/pipeline/debug.go @@ -23,12 +23,11 @@ import ( "sync" "time" - "github.com/livekit/egress/pkg/stats" - "github.com/go-gst/go-gst/gst" "github.com/livekit/egress/pkg/errors" "github.com/livekit/egress/pkg/pipeline/sink/uploader" + "github.com/livekit/egress/pkg/stats" "github.com/livekit/egress/pkg/types" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/pprof" diff --git a/pkg/pipeline/sink/segments.go b/pkg/pipeline/sink/segments.go index f504b858..33d06442 100644 --- a/pkg/pipeline/sink/segments.go +++ b/pkg/pipeline/sink/segments.go @@ -22,8 +22,6 @@ import ( "sync" "time" - "github.com/livekit/egress/pkg/stats" - "github.com/frostbyte73/core" "github.com/livekit/egress/pkg/config" @@ -31,6 +29,7 @@ import ( "github.com/livekit/egress/pkg/gstreamer" "github.com/livekit/egress/pkg/pipeline/sink/m3u8" "github.com/livekit/egress/pkg/pipeline/sink/uploader" + "github.com/livekit/egress/pkg/stats" "github.com/livekit/egress/pkg/types" "github.com/livekit/protocol/logger" ) diff --git a/pkg/pipeline/sink/uploader/s3.go b/pkg/pipeline/sink/uploader/s3.go index 23bfc3d2..63e6f949 100644 --- a/pkg/pipeline/sink/uploader/s3.go +++ b/pkg/pipeline/sink/uploader/s3.go @@ -27,8 +27,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/livekit/egress/pkg/config" + "github.com/livekit/egress/pkg/config" "github.com/livekit/egress/pkg/types" "github.com/livekit/protocol/logger" "github.com/livekit/psrpc" diff --git a/pkg/pipeline/sink/uploader/uploader.go b/pkg/pipeline/sink/uploader/uploader.go index 987617e7..04760cd1 100644 --- a/pkg/pipeline/sink/uploader/uploader.go +++ b/pkg/pipeline/sink/uploader/uploader.go @@ -20,11 +20,10 @@ import ( "path" "time" - "github.com/livekit/egress/pkg/stats" - "github.com/pkg/errors" "github.com/livekit/egress/pkg/config" + "github.com/livekit/egress/pkg/stats" "github.com/livekit/egress/pkg/types" "github.com/livekit/protocol/livekit" ) @@ -50,6 +49,8 @@ func New(conf config.UploadConfig, backup string, monitor stats.HandlerMonitor) switch c := conf.(type) { case *config.EgressS3Upload: u, err = newS3Uploader(c) + case *livekit.S3Upload: + u, err = newS3Uploader(&config.EgressS3Upload{S3Upload: c}) case *livekit.GCPUpload: u, err = newGCPUploader(c) case *livekit.AzureBlobUpload: @@ -63,13 +64,13 @@ func New(conf config.UploadConfig, backup string, monitor stats.HandlerMonitor) return nil, err } - remoteUploader := &remoteUploader{ + remote := &remoteUploader{ uploader: u, backup: backup, monitor: monitor, } - return remoteUploader, nil + return remote, nil } type remoteUploader struct { @@ -83,6 +84,8 @@ func (u *remoteUploader) Upload(localFilepath, storageFilepath string, outputTyp start := time.Now() location, size, err := u.upload(localFilepath, storageFilepath, outputType) elapsed := time.Since(start).Milliseconds() + + // success if err == nil { u.monitor.IncUploadCountSuccess(fileType, float64(elapsed)) if deleteAfterUpload { @@ -91,8 +94,9 @@ func (u *remoteUploader) Upload(localFilepath, storageFilepath string, outputTyp return location, size, nil } - u.monitor.IncUploadCountFailure(fileType, float64(elapsed)) + // failure + u.monitor.IncUploadCountFailure(fileType, float64(elapsed)) if u.backup != "" { stat, err := os.Stat(localFilepath) if err != nil { diff --git a/pkg/service/handler.go b/pkg/service/handler.go index 6f427079..9a945484 100644 --- a/pkg/service/handler.go +++ b/pkg/service/handler.go @@ -20,10 +20,10 @@ import ( "strings" "time" + "github.com/frostbyte73/core" + "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" - - "github.com/frostbyte73/core" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -38,8 +38,6 @@ import ( "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/tracer" "github.com/livekit/psrpc" - - "github.com/prometheus/client_golang/prometheus" ) const network = "unix" diff --git a/pkg/service/process.go b/pkg/service/process.go index 38c6a03f..5ad941a3 100644 --- a/pkg/service/process.go +++ b/pkg/service/process.go @@ -16,8 +16,6 @@ package service import ( "context" - "github.com/prometheus/common/expfmt" - "golang.org/x/exp/maps" "net" "os" "os/exec" @@ -27,6 +25,9 @@ import ( "time" "github.com/frostbyte73/core" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "golang.org/x/exp/maps" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/encoding/protojson" @@ -40,8 +41,6 @@ import ( "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/tracer" "github.com/livekit/protocol/utils" - - dto "github.com/prometheus/client_model/go" ) type Process struct { diff --git a/pkg/service/service.go b/pkg/service/service.go index 60d8ee39..799d938c 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -23,11 +23,10 @@ import ( "sync" "time" - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/frostbyte73/core" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" "github.com/livekit/egress/pkg/config" "github.com/livekit/egress/pkg/stats" diff --git a/pkg/stats/handler_monitor.go b/pkg/stats/handler.go similarity index 85% rename from pkg/stats/handler_monitor.go rename to pkg/stats/handler.go index b6e8fe93..04e3a218 100644 --- a/pkg/stats/handler_monitor.go +++ b/pkg/stats/handler.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package stats import (