Merge branch 'dev'

Author: ioppermann

.github_build/Build.alpine.env

@@ -1,5 +1,5 @@
 # CORE ALPINE BASE IMAGE
 OS_NAME=alpine
-OS_VERSION=3.15
-GOLANG_IMAGE=golang:1.18.6-alpine3.15
-CORE_VERSION=16.10.1
+OS_VERSION=3.16
+GOLANG_IMAGE=golang:1.19.3-alpine3.16
+CORE_VERSION=16.11.0

.github_build/Build.bundle.cuda.env

@@ -1,3 +1,3 @@
 # CORE NVIDIA CUDA BUNDLE
-FFMPEG_VERSION=4.4.2
-CUDA_VERSION=11.4.2
+FFMPEG_VERSION=5.1.2
+CUDA_VERSION=11.7.1

.github_build/Build.bundle.env

@@ -1,2 +1,2 @@
 # CORE BUNDLE
-FFMPEG_VERSION=4.4.2
+FFMPEG_VERSION=5.1.2

.github_build/Build.bundle.rpi.env

@@ -1,2 +1,2 @@
 # CORE RASPBERRY-PI BUNDLE
-FFMPEG_VERSION=4.4.2
+FFMPEG_VERSION=5.1.2

.github_build/Build.bundle.vaapi.env

@@ -1,2 +1,2 @@
 # CORE BUNDLE
-FFMPEG_VERSION=4.4.2
+FFMPEG_VERSION=5.1.2

.github_build/Build.ubuntu.env

@@ -1,5 +1,5 @@
 # CORE UBUNTU BASE IMAGE
 OS_NAME=ubuntu
 OS_VERSION=20.04
-GOLANG_IMAGE=golang:1.18.6-alpine3.15
-CORE_VERSION=16.10.1
+GOLANG_IMAGE=golang:1.19.3-alpine3.16
+CORE_VERSION=16.11.0

.gitignore

@@ -2,6 +2,7 @@
 .env
 /core*
 /import*
+/ffmigrate*
 /data/**
 /test/**
 .vscode

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Core
 
+### Core v16.10.1 > v16.11.0
+
+-   Add FFmpeg 4.4 to FFmpeg 5.1 migration tool
+-   Add alternative SRT streamid
+-   Mod bump FFmpeg to v5.1.2 (datarhei/core:tag bundles)
+-   Fix crash with custom SSL certificates ([restreamer/#425](https://github.com/datarhei/restreamer/issues/425))
+-   Fix proper version handling for config
+-   Fix widged session data
+-   Fix resetting process stats when process stopped
+-   Fix stale FFmpeg process detection for streams with only audio
+-   Fix wrong return status code ([#6](https://github.com/datarhei/core/issues/6)))
+-   Fix use SRT defaults for key material exchange
+
 ### Core v16.10.0 > v16.10.1
 
 -   Add email address in TLS config for Let's Encrypt
@@ -20,11 +33,11 @@
 -   Fix process cleanup on delete, remove empty directories from disk
 -   Fix SRT blocking port on restart (upgrade datarhei/gosrt)
 -   Fix RTMP communication (Blackmagic Web Presenter, thx 235 MEDIA)
--   Fix RTMP communication (Blackmagic ATEM Mini, datarhei/restreamer#385)
+-   Fix RTMP communication (Blackmagic ATEM Mini, [#385](https://github.com/datarhei/restreamer/issues/385))
 -   Fix injecting commit, branch, and build info
 -   Fix API metadata endpoints responses
 
-#### Core v16.9.0 > v16.9.1
+#### Core v16.9.0 > v16.9.1^
 
 -   Fix v1 import app
 -   Fix race condition

Dockerfile

@@ -1,23 +1,25 @@
-ARG GOLANG_IMAGE=golang:1.18.4-alpine3.15
+ARG GOLANG_IMAGE=golang:1.19.3-alpine3.16
 
-ARG BUILD_IMAGE=alpine:3.15
+ARG BUILD_IMAGE=alpine:3.16
 
 FROM $GOLANG_IMAGE as builder
 
 COPY . /dist/core
 
 RUN apk add \
-		git \
-		make && \
+	git \
+	make && \
 	cd /dist/core && \
 	go version && \
 	make release_linux && \
-	make import_linux
+	make import_linux && \
+	make ffmigrate_linux
 
 FROM $BUILD_IMAGE
 
 COPY --from=builder /dist/core/core /core/bin/core
 COPY --from=builder /dist/core/import /core/bin/import
+COPY --from=builder /dist/core/ffmigrate /core/bin/ffmigrate
 COPY --from=builder /dist/core/mime.types /core/mime.types
 COPY --from=builder /dist/core/run.sh /core/bin/run.sh
 

Dockerfile.test

@@ -1,8 +1,8 @@
-FROM golang:1.18.3-alpine3.15
+FROM golang:1.19.3-alpine3.16
 
 RUN apk add alpine-sdk
 
 COPY . /dist/core
 
 RUN cd /dist/core && \
-    go test -coverprofile=coverage.out -covermode=atomic -v ./...
+    go test -coverprofile=coverage.out -covermode=atomic -v ./...

Makefile

@@ -75,6 +75,14 @@ import:
 import_linux:
 	cd app/import && CGO_ENABLED=0 GOOS=linux GOARCH=${OSARCH} go build -o ../../import -ldflags="-s -w"
 
+## ffmigrate: Build ffmpeg migration binary
+ffmigrate:
+	cd app/ffmigrate && CGO_ENABLED=${CGO_ENABLED} GOOS=${GOOS} GOARCH=${GOARCH} go build -o ../../ffmigrate -ldflags="-s -w"
+
+# github workflow workaround
+ffmigrate_linux:
+	cd app/ffmigrate && CGO_ENABLED=0 GOOS=linux GOARCH=${OSARCH} go build -o ../../ffmigrate -ldflags="-s -w"
+
 ## coverage: Generate code coverage analysis
 coverage:
 	go test -race -coverprofile test/cover.out ./...
@@ -96,7 +104,7 @@ release_linux:
 docker:
 	docker build -t core:$(SHORTCOMMIT) .
 
-.PHONY: help init build swagger test vet fmt vulncheck vendor commit coverage lint release import update
+.PHONY: help init build swagger test vet fmt vulncheck vendor commit coverage lint release import ffmigrate update
 
 ## help: Show all commands
 help: Makefile

app/api/api.go

@@ -16,6 +16,8 @@ import (
 
 	"github.com/datarhei/core/v16/app"
 	"github.com/datarhei/core/v16/config"
+	configstore "github.com/datarhei/core/v16/config/store"
+	configvars "github.com/datarhei/core/v16/config/vars"
 	"github.com/datarhei/core/v16/ffmpeg"
 	"github.com/datarhei/core/v16/http"
 	"github.com/datarhei/core/v16/http/cache"
@@ -96,7 +98,7 @@ type api struct {
 
 	config struct {
 		path   string
-		store  config.Store
+		store  configstore.Store
 		config *config.Config
 	}
 
@@ -145,7 +147,7 @@ func (a *api) Reload() error {
 
 	logger := log.New("Core").WithOutput(log.NewConsoleWriter(a.log.writer, log.Lwarn, true))
 
-	store, err := config.NewJSONStore(a.config.path, func() {
+	store, err := configstore.NewJSON(a.config.path, func() {
 		a.errorChan <- ErrConfigReload
 	})
 	if err != nil {
@@ -157,7 +159,7 @@ func (a *api) Reload() error {
 	cfg.Merge()
 
 	if len(cfg.Host.Name) == 0 && cfg.Host.Auto {
-		cfg.SetPublicIPs()
+		cfg.Host.Name = net.GetPublicIPs(5 * time.Second)
 	}
 
 	cfg.Validate(false)
@@ -226,7 +228,7 @@ func (a *api) Reload() error {
 	logger.Info().WithFields(logfields).Log("")
 
 	configlogger := logger.WithComponent("Config")
-	cfg.Messages(func(level string, v config.Variable, message string) {
+	cfg.Messages(func(level string, v configvars.Variable, message string) {
 		configlogger = configlogger.WithFields(log.Fields{
 			"variable":    v.Name,
 			"value":       v.Value,
@@ -362,11 +364,6 @@ func (a *api) start() error {
 		a.sessions = sessions
 	}
 
-	store := store.NewJSONStore(store.JSONConfig{
-		Dir:    cfg.DB.Dir,
-		Logger: a.log.logger.core.WithComponent("ProcessStore"),
-	})
-
 	diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{
 		Dir:    cfg.Storage.Disk.Dir,
 		Size:   cfg.Storage.Disk.Size * 1024 * 1024,
@@ -481,6 +478,12 @@ func (a *api) start() error {
 		a.replacer.RegisterTemplate("srt", template)
 	}
 
+	store := store.NewJSONStore(store.JSONConfig{
+		Filepath:  cfg.DB.Dir + "/db.json",
+		FFVersion: a.ffmpeg.Skills().FFmpeg.Version,
+		Logger:    a.log.logger.core.WithComponent("ProcessStore"),
+	})
+
 	restream, err := restream.New(restream.Config{
 		ID:           cfg.ID,
 		Name:         cfg.Name,
@@ -649,98 +652,88 @@ func (a *api) start() error {
 
 	var autocertManager *certmagic.Config
 
-	if cfg.TLS.Enable && cfg.TLS.Auto {
-		if len(cfg.Host.Name) == 0 {
-			return fmt.Errorf("at least one host must be provided in host.name or RS_HOST_NAME")
-		}
+	if cfg.TLS.Enable {
+		if cfg.TLS.Auto {
+			if len(cfg.Host.Name) == 0 {
+				return fmt.Errorf("at least one host must be provided in host.name or RS_HOST_NAME")
+			}
 
-		certmagic.DefaultACME.Agreed = true
-		certmagic.DefaultACME.Email = cfg.TLS.Email
-		certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA
-		certmagic.DefaultACME.DisableHTTPChallenge = false
-		certmagic.DefaultACME.DisableTLSALPNChallenge = true
-		certmagic.DefaultACME.Logger = nil
+			certmagic.DefaultACME.Agreed = true
+			certmagic.DefaultACME.Email = cfg.TLS.Email
+			certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA
+			certmagic.DefaultACME.DisableHTTPChallenge = false
+			certmagic.DefaultACME.DisableTLSALPNChallenge = true
+			certmagic.DefaultACME.Logger = nil
 
-		certmagic.Default.Storage = &certmagic.FileStorage{
-			Path: cfg.DB.Dir + "/cert",
-		}
-		certmagic.Default.DefaultServerName = cfg.Host.Name[0]
-		certmagic.Default.Logger = nil
-		certmagic.Default.OnEvent = func(event string, data interface{}) {
-			message := ""
-
-			switch data := data.(type) {
-			case string:
-				message = data
-			case fmt.Stringer:
-				message = data.String()
+			certmagic.Default.Storage = &certmagic.FileStorage{
+				Path: cfg.DB.Dir + "/cert",
 			}
+			certmagic.Default.DefaultServerName = cfg.Host.Name[0]
+			certmagic.Default.Logger = nil
 
-			if len(message) != 0 {
-				a.log.logger.core.WithComponent("certmagic").Info().WithField("event", event).Log(message)
-			}
-		}
+			magic := certmagic.NewDefault()
+			acme := certmagic.NewACMEIssuer(magic, certmagic.DefaultACME)
 
-		magic := certmagic.NewDefault()
-		acme := certmagic.NewACMEIssuer(magic, certmagic.DefaultACME)
+			magic.Issuers = []certmagic.Issuer{acme}
 
-		magic.Issuers = []certmagic.Issuer{acme}
+			autocertManager = magic
 
-		autocertManager = magic
+			// Start temporary http server on configured port
+			tempserver := &gohttp.Server{
+				Addr: cfg.Address,
+				Handler: acme.HTTPChallengeHandler(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
+					w.WriteHeader(gohttp.StatusNotFound)
+				})),
+				ReadTimeout:    10 * time.Second,
+				WriteTimeout:   10 * time.Second,
+				MaxHeaderBytes: 1 << 20,
+			}
 
-		// Start temporary http server on configured port
-		tempserver := &gohttp.Server{
-			Addr: cfg.Address,
-			Handler: acme.HTTPChallengeHandler(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
-				w.WriteHeader(gohttp.StatusNotFound)
-			})),
-			ReadTimeout:    10 * time.Second,
-			WriteTimeout:   10 * time.Second,
-			MaxHeaderBytes: 1 << 20,
-		}
+			wg := sync.WaitGroup{}
+			wg.Add(1)
 
-		wg := sync.WaitGroup{}
-		wg.Add(1)
+			go func() {
+				tempserver.ListenAndServe()
+				wg.Done()
+			}()
 
-		go func() {
-			tempserver.ListenAndServe()
-			wg.Done()
-		}()
+			var certerror bool
 
-		var certerror bool
+			// For each domain, get the certificate
+			for _, host := range cfg.Host.Name {
+				logger := a.log.logger.core.WithComponent("Let's Encrypt").WithField("host", host)
+				logger.Info().Log("Acquiring certificate ...")
 
-		// For each domain, get the certificate
-		for _, host := range cfg.Host.Name {
-			logger := a.log.logger.core.WithComponent("Let's Encrypt").WithField("host", host)
-			logger.Info().Log("Acquiring certificate ...")
+				ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Minute))
 
-			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Minute))
+				err := autocertManager.ManageSync(ctx, []string{host})
 
-			err := autocertManager.ManageSync(ctx, []string{host})
+				cancel()
 
-			cancel()
+				if err != nil {
+					logger.Error().WithField("error", err).Log("Failed to acquire certificate")
+					certerror = true
+					break
+				}
 
-			if err != nil {
-				logger.Error().WithField("error", err).Log("Failed to acquire certificate")
-				certerror = true
-				break
+				logger.Info().Log("Successfully acquired certificate")
 			}
 
-			logger.Info().Log("Successfully acquired certificate")
-		}
-
-		// Shut down the temporary http server
-		tempserver.Close()
+			// Shut down the temporary http server
+			tempserver.Close()
 
-		wg.Wait()
+			wg.Wait()
 
-		if certerror {
-			a.log.logger.core.Warn().Log("Continuing with disabled TLS")
-			autocertManager = nil
-			cfg.TLS.Enable = false
+			if certerror {
+				a.log.logger.core.Warn().Log("Continuing with disabled TLS")
+				autocertManager = nil
+				cfg.TLS.Enable = false
+			} else {
+				cfg.TLS.CertFile = ""
+				cfg.TLS.KeyFile = ""
+			}
 		} else {
-			cfg.TLS.CertFile = ""
-			cfg.TLS.KeyFile = ""
+			a.log.logger.core.Info().Log("Enabling TLS with cert and key files")
 		}
 	}
 
@@ -756,14 +749,15 @@ func (a *api) start() error {
 			Collector: a.sessions.Collector("rtmp"),
 		}
 
-		if autocertManager != nil && cfg.RTMP.EnableTLS {
-			config.TLSConfig = &tls.Config{
-				GetCertificate: autocertManager.GetCertificate,
-			}
-
+		if cfg.RTMP.EnableTLS {
 			config.Logger = config.Logger.WithComponent("RTMP/S")
 
 			a.log.logger.rtmps = a.log.logger.core.WithComponent("RTMPS").WithField("address", cfg.RTMP.AddressTLS)
+			if autocertManager != nil {
+				config.TLSConfig = &tls.Config{
+					GetCertificate: autocertManager.GetCertificate,
+				}
+			}
 		}
 
 		rtmpserver, err := rtmp.New(config)