Skip to content

Fix loki.source.docker only starting log collection for containers in running state #3449

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ Main (unreleased)
### Bugfixes

- Fix `otelcol.receiver.filelog` documentation's default value for `start_at`. (@petewall)
- Fix `loki.source.docker` only starting log collection on running containers. (@RasEza)

### Other changes

Expand Down
35 changes: 17 additions & 18 deletions internal/component/loki/source/docker/docker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package docker

import (
"context"
"fmt"
"io"
"os"
"strings"
Expand Down Expand Up @@ -106,10 +107,10 @@ func TestDuplicateTargets(t *testing.T) {
}

func TestRestart(t *testing.T) {
runningState := true
var inspectError error // nil to simulate a running container.
client := clientMock{
logLine: "2024-05-02T13:11:55.879889Z caller=module_service.go:114 msg=\"module stopped\" module=distributor",
running: func() bool { return runningState },
logLine: "2024-05-02T13:11:55.879889Z caller=module_service.go:114 msg=\"module stopped\" module=distributor",
inspectErr: &inspectError,
}
expectedLogLine := "caller=module_service.go:114 msg=\"module stopped\" module=distributor"

Expand All @@ -125,14 +126,14 @@ func TestRestart(t *testing.T) {
}, time.Second, 20*time.Millisecond, "Expected log lines were not found within the time limit.")

// Stops the container.
runningState = false
inspectError = fmt.Errorf("container not found")
time.Sleep(targetRestartInterval + 10*time.Millisecond) // Sleep for a duration greater than targetRestartInterval to make sure it stops sending log lines.
entryHandler.Clear()
time.Sleep(targetRestartInterval + 10*time.Millisecond)
assert.Empty(t, entryHandler.Received()) // No log lines because the container was not running.
assert.Empty(t, entryHandler.Received()) // No log lines because a container is not found.

// Restart the container and expect log lines.
runningState = true
inspectError = nil
assert.EventuallyWithT(t, func(c *assert.CollectT) {
logLines := entryHandler.Received()
if assert.NotEmpty(c, logLines) {
Expand All @@ -142,10 +143,10 @@ func TestRestart(t *testing.T) {
}

func TestTargetNeverStarted(t *testing.T) {
runningState := false
var inspectError error = fmt.Errorf("container not found")
client := clientMock{
logLine: "2024-05-02T13:11:55.879889Z caller=module_service.go:114 msg=\"module stopped\" module=distributor",
running: func() bool { return runningState },
logLine: "2024-05-02T13:11:55.879889Z caller=module_service.go:114 msg=\"module stopped\" module=distributor",
inspectErr: &inspectError,
}

tailer, _ := setupTailer(t, client)
Expand Down Expand Up @@ -191,19 +192,17 @@ func setupTailer(t *testing.T, client clientMock) (tailer *tailer, entryHandler

type clientMock struct {
client.APIClient
logLine string
running func() bool
logLine string
inspectErr *error
}

func (mock clientMock) ContainerInspect(ctx context.Context, c string) (types.ContainerJSON, error) {
if mock.inspectErr != nil && *mock.inspectErr != nil {
return types.ContainerJSON{}, *mock.inspectErr
}
return types.ContainerJSON{
ContainerJSONBase: &types.ContainerJSONBase{
ID: c,
State: &types.ContainerState{
Running: mock.running(),
},
},
Config: &container.Config{Tty: true},
ContainerJSONBase: &types.ContainerJSONBase{},
Config: &container.Config{Tty: true},
}, nil
}

Expand Down
9 changes: 1 addition & 8 deletions internal/component/loki/source/docker/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,14 +104,7 @@ func (t *tailer) Run(ctx context.Context) {
for {
select {
case <-tickerC:
res, err := t.opts.client.ContainerInspect(ctx, t.target.Name())
if err != nil {
level.Error(t.log).Log("msg", "error inspecting Docker container", "id", t.target.Name(), "error", err)
continue
}
if res.State.Running {
t.target.StartIfNotRunning()
}
t.target.StartIfNotRunning()
case <-ctx.Done():
t.target.Stop()
ticker.Stop()
Expand Down