diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d9fa0633d..a32cfdb93c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Main (unreleased) - Add perf_schema quantile columns to collector - Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15) +- Add ignore_older option for local.file_match ### Bugfixes @@ -92,7 +93,7 @@ v1.5.1 - Fixed a crash when updating the configuration of `remote.http`. (@kinolaev) -- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum) +- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum) - Fixed an issue in the `prometheus.exporter.postgres` component that would leak goroutines when the target was not reachable (@dehaansa) @@ -292,7 +293,7 @@ v1.4.0 - Add the label `alloy_cluster` in the metric `alloy_config_hash` when the flag `cluster.name` is set to help differentiate between configs from the same alloy cluster or different alloy clusters. (@wildum) - + - Add support for discovering the cgroup path(s) of a process in `process.discovery`. (@mahendrapaipuri) ### Bugfixes diff --git a/docs/sources/reference/components/local/local.file_match.md b/docs/sources/reference/components/local/local.file_match.md index 70f036f1ba..0f8dd47821 100644 --- a/docs/sources/reference/components/local/local.file_match.md +++ b/docs/sources/reference/components/local/local.file_match.md @@ -28,6 +28,7 @@ Name | Type | Description --------------- | ------------------- | ------------------------------------------------------------------------------------------ |---------| -------- `path_targets` | `list(map(string))` | Targets to expand; looks for glob patterns on the `__path__` and `__path_exclude__` keys. | | yes `sync_period` | `duration` | How often to sync filesystem and targets. | `"10s"` | no +`ignore_older` | `duration` | Ignores files which are modified before this duration | | no `path_targets` uses [doublestar][] style paths. * `/tmp/**/*.log` will match all subfolders of `tmp` and include any files that end in `*.log`. diff --git a/internal/component/local/file_match/file.go b/internal/component/local/file_match/file.go index e5b9766e25..f757dec1ea 100644 --- a/internal/component/local/file_match/file.go +++ b/internal/component/local/file_match/file.go @@ -28,6 +28,7 @@ func init() { type Arguments struct { PathTargets []discovery.Target `alloy:"path_targets,attr"` SyncPeriod time.Duration `alloy:"sync_period,attr,optional"` + IgnoreOlder time.Duration `alloy:"ignore_older,attr,optional"` } var _ component.Component = (*Component)(nil) @@ -80,8 +81,9 @@ func (c *Component) Update(args component.Arguments) error { c.watches = c.watches[:0] for _, v := range c.args.PathTargets { c.watches = append(c.watches, watch{ - target: v, - log: c.opts.Logger, + target: v, + log: c.opts.Logger, + ignoreOlder: c.args.IgnoreOlder, }) } diff --git a/internal/component/local/file_match/file_test.go b/internal/component/local/file_match/file_test.go index bec538ce2f..4f37ab6259 100644 --- a/internal/component/local/file_match/file_test.go +++ b/internal/component/local/file_match/file_test.go @@ -63,6 +63,35 @@ func TestDirectoryFile(t *testing.T) { require.True(t, contains(foundFiles, "t1.txt")) } +func TestFileIgnoreOlder(t *testing.T) { + dir := path.Join(os.TempDir(), "alloy_testing", "t1") + err := os.MkdirAll(dir, 0755) + require.NoError(t, err) + writeFile(t, dir, "t1.txt") + t.Cleanup(func() { + os.RemoveAll(dir) + }) + c := createComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil) + ct := context.Background() + ct, ccl := context.WithTimeout(ct, 5*time.Second) + defer ccl() + c.args.SyncPeriod = 10 * time.Millisecond + c.args.IgnoreOlder = 100 * time.Millisecond + c.Update(c.args) + go c.Run(ct) + + foundFiles := c.getWatchedFiles() + require.Len(t, foundFiles, 1) + require.True(t, contains(foundFiles, "t1.txt")) + time.Sleep(150 * time.Millisecond) + + writeFile(t, dir, "t2.txt") + ct.Done() + foundFiles = c.getWatchedFiles() + require.Len(t, foundFiles, 1) + require.True(t, contains(foundFiles, "t2.txt")) +} + func TestAddingFile(t *testing.T) { dir := path.Join(os.TempDir(), "alloy_testing", "t2") err := os.MkdirAll(dir, 0755) @@ -270,6 +299,7 @@ func createComponentWithLabels(t *testing.T, dir string, paths []string, exclude }, Arguments{ PathTargets: tPaths, SyncPeriod: 1 * time.Second, + IgnoreOlder: 1 * time.Hour, }) require.NoError(t, err) diff --git a/internal/component/local/file_match/watch.go b/internal/component/local/file_match/watch.go index 709d821151..05a28935c5 100644 --- a/internal/component/local/file_match/watch.go +++ b/internal/component/local/file_match/watch.go @@ -3,6 +3,7 @@ package file_match import ( "os" "path/filepath" + "time" "github.com/bmatcuk/doublestar" "github.com/go-kit/log" @@ -14,8 +15,9 @@ import ( // watch handles a single discovery.target for file watching. type watch struct { - target discovery.Target - log log.Logger + target discovery.Target + log log.Logger + ignoreOlder time.Duration } func (w *watch) getPaths() ([]discovery.Target, error) { @@ -48,6 +50,11 @@ func (w *watch) getPaths() ([]discovery.Target, error) { } continue } + + if w.ignoreOlder != 0 && fi.ModTime().Before(time.Now().Add(-w.ignoreOlder)) { + continue + } + if fi.IsDir() { continue }