podWatcher: prefer channel for synchronization
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Rob Watson 2022-06-11 07:13:42 +02:00
parent 5ebeabd3c1
commit be99d25fb2
3 changed files with 14 additions and 29 deletions

View File

@ -101,10 +101,10 @@ func (pw *PodWatcher) watchPods(ctx context.Context, wg *sync.WaitGroup) error {
ticker := time.NewTicker(tickerInterval) ticker := time.NewTicker(tickerInterval)
defer ticker.Stop() defer ticker.Stop()
// streamErrors is never closed. logStream := make(chan string)
streamErrors := make(chan *streamError) streamErrors := make(chan *streamError)
resultChan := watcher.ResultChan()
resultChan := watcher.ResultChan()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -123,7 +123,7 @@ func (pw *PodWatcher) watchPods(ctx context.Context, wg *sync.WaitGroup) error {
pw.status[pod.Name] = true pw.status[pod.Name] = true
wg.Add(1) wg.Add(1)
go func() { go func() {
if err := copyPodLogs(ctx, wg, pw.client, pod, pw.container, pw.dst); err != nil { if err := copyPodLogs(ctx, wg, pw.client, pod, pw.container, logStream); err != nil {
streamErrors <- err streamErrors <- err
} }
}() }()
@ -137,6 +137,13 @@ func (pw *PodWatcher) watchPods(ctx context.Context, wg *sync.WaitGroup) error {
} }
} }
// logStream is never closed
case l := <-logStream:
if _, err := pw.dst.Write([]byte(l)); err != nil {
return fmt.Errorf("error writing logs: %v", err)
}
// streamErrors is never closed
case streamErr := <-streamErrors: case streamErr := <-streamErrors:
if streamErr.recoverable { if streamErr.recoverable {
// if the error is recoverable, we just remove the pod from the status // if the error is recoverable, we just remove the pod from the status
@ -169,19 +176,17 @@ func (pw *PodWatcher) removePod(podName string) {
delete(pw.status, podName) delete(pw.status, podName)
} }
func copyPodLogs(ctx context.Context, wg *sync.WaitGroup, client KubernetesClient, pod *corev1.Pod, container string, dst io.Writer) *streamError { func copyPodLogs(ctx context.Context, wg *sync.WaitGroup, client KubernetesClient, pod *corev1.Pod, container string, logStream chan string) *streamError {
defer wg.Done() defer wg.Done()
podLogOpts := corev1.PodLogOptions{ podLogOpts := corev1.PodLogOptions{
Follow: true, Follow: true,
Container: container, Container: container,
} }
req := client.Typed.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) req := client.Typed.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts)
logs, err := req.Stream(ctx) logs, err := req.Stream(ctx)
// If one container is still being created, do not treat this as a fatal error. // If one container is still being created, do not treat this as a fatal error.
// We try to verify the error type as strictly as possible.
var statusErr *apierrors.StatusError var statusErr *apierrors.StatusError
if errors.As(err, &statusErr) && statusErr.Status().Reason == metav1.StatusReasonBadRequest && strings.Contains(statusErr.Error(), "ContainerCreating") { if errors.As(err, &statusErr) && statusErr.Status().Reason == metav1.StatusReasonBadRequest && strings.Contains(statusErr.Error(), "ContainerCreating") {
return newRecoverableError(err, pod.Name) return newRecoverableError(err, pod.Name)
@ -193,9 +198,7 @@ func copyPodLogs(ctx context.Context, wg *sync.WaitGroup, client KubernetesClien
scanner := bufio.NewScanner(logs) scanner := bufio.NewScanner(logs)
for scanner.Scan() { for scanner.Scan() {
if _, err = dst.Write([]byte("[" + pod.Name + "] " + scanner.Text() + nl)); err != nil { logStream <- "[" + pod.Name + "] " + scanner.Text() + nl
return newStreamError(fmt.Errorf("error writing logs: %v", err), pod.Name)
}
} }
if err := scanner.Err(); err != nil { if err := scanner.Err(); err != nil {
return newStreamError(fmt.Errorf("error scanning logs: %v", err), pod.Name) return newStreamError(fmt.Errorf("error scanning logs: %v", err), pod.Name)

View File

@ -6,7 +6,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log"
"net/http" "net/http"
"strings" "strings"
"testing" "testing"
@ -134,7 +133,6 @@ func TestPodWatcher(t *testing.T) {
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
podsWatcher := watch.NewFake() podsWatcher := watch.NewFake()
defer log.Println("exiting test func")
clientset := mockClientset{ clientset := mockClientset{
getLogsRespBody: tc.getLogsRespBody, getLogsRespBody: tc.getLogsRespBody,

View File

@ -7,7 +7,6 @@ import (
"io" "io"
"log" "log"
"os" "os"
"sync"
"time" "time"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
@ -29,21 +28,6 @@ type KubernetesClient struct {
Untyped dynamic.Interface Untyped dynamic.Interface
} }
// concurrentWriter implements an io.Writer that can be safely written to from
// multiple goroutines.
type concurrentWriter struct {
w io.Writer
mu sync.Mutex
}
// Write implements io.Writer.
func (cw *concurrentWriter) Write(p []byte) (int, error) {
cw.mu.Lock()
defer cw.mu.Unlock()
return cw.w.Write(p)
}
type PodWatcherInterface interface { type PodWatcherInterface interface {
WatchPods(ctx context.Context) error WatchPods(ctx context.Context) error
Close() Close()
@ -71,7 +55,7 @@ type Watcher struct {
podWatcher PodWatcherInterface podWatcher PodWatcherInterface
podWatcherFunc PodWatcherFunc podWatcherFunc PodWatcherFunc
errChan chan error errChan chan error
dst *concurrentWriter dst io.Writer
logger *log.Logger logger *log.Logger
} }
@ -82,7 +66,7 @@ func NewWatcher(params WatcherParams, client KubernetesClient, podWatcherFunc Po
client: client, client: client,
podWatcherFunc: podWatcherFunc, podWatcherFunc: podWatcherFunc,
errChan: make(chan error), errChan: make(chan error),
dst: &concurrentWriter{w: dst}, dst: dst,
logger: logger, logger: logger,
} }
} }