drsybren/Flamenco
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Work in progress on using UPnP/SSDP to make the Worker find its Manager

Browse Source 
Due to the way SSDP works, Flamenco Manager needs to know its own URL,
where the Workers can reach it. These URLs are now found, and since there
can be multiple (like IPv6 + IPv4) they are all sent in a SSDP
notification as ;-separated strings.
This commit is contained in:
Sybren A. Stüvel 2022-03-04 17:44:04 +01:00
parent 656a495652
commit d153db4280
12 changed files with 677 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/flamenco-manager/main.go
View File

@ -49,6 +49,7 @@ import (
"git.blender.org/flamenco/internal/manager/swagger_ui"
"git.blender.org/flamenco/internal/manager/task_logs"
"git.blender.org/flamenco/internal/manager/task_state_machine"
"git.blender.org/flamenco/internal/upnp_ssdp"
"git.blender.org/flamenco/pkg/api"
)
@ -84,6 +85,13 @@ func main() {
_, port, _ := net.SplitHostPort(listen)
log.Info().Str("port", port).Msg("listening")
ssdp, err := upnp_ssdp.NewServer(log.Logger)
if err != nil {
log.Error().Err(err).Msg("error creating UPnP/SSDP server")
} else {
ssdp.AddAdvertisement(listen) // TODO: convert this to an entire URL.
}
// Construct the services.
persist := openDB(*configService)
flamenco := buildFlamencoAPI(configService, persist)
@ -119,6 +127,15 @@ func main() {
}
}()
// Start the UPnP/SSDP server.
if ssdp != nil {
wg.Add(1)
go func() {
defer wg.Done()
ssdp.Run(mainCtx)
}()
}
wg.Wait()
log.Info().Msg("shutdown complete")
}

46
cmd/ssdp_client_poc/main.go Normal file
View File

@ -0,0 +1,46 @@
package main
import (
"os"
"os/signal"
"syscall"
"time"
"git.blender.org/flamenco/internal/upnp_ssdp"
"github.com/mattn/go-colorable"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"golang.org/x/net/context"
)
func main() {
output := zerolog.ConsoleWriter{Out: colorable.NewColorableStdout(), TimeFormat: time.RFC3339}
log.Logger = log.Output(output)
c, err := upnp_ssdp.NewClient(log.Logger)
if err != nil {
panic(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Handle Ctrl+C
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
signal.Notify(signals, syscall.SIGTERM)
go func() {
for signum := range signals {
log.Info().Str("signal", signum.String()).Msg("signal received, shutting down")
cancel()
}
}()
urls, err := c.Run(ctx)
if err != nil {
panic(err)
}
for _, url := range urls {
log.Info().Str("url", url).Msg("found URL")
}
}

52
cmd/ssdp_server_poc/main.go Normal file
View File

@ -0,0 +1,52 @@
package main
import (
"os"
"os/signal"
"strings"
"syscall"
"time"
"git.blender.org/flamenco/internal/own_url"
"git.blender.org/flamenco/internal/upnp_ssdp"
"github.com/mattn/go-colorable"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"golang.org/x/net/context"
)
func main() {
output := zerolog.ConsoleWriter{Out: colorable.NewColorableStdout(), TimeFormat: time.RFC3339}
log.Logger = log.Output(output)
c, err := upnp_ssdp.NewServer(log.Logger)
if err != nil {
panic(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Handle Ctrl+C
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
signal.Notify(signals, syscall.SIGTERM)
go func() {
for signum := range signals {
log.Info().Str("signal", signum.String()).Msg("signal received, shutting down")
cancel()
}
}()
urls, err := own_url.AvailableURLs(ctx, "http", ":8080", false)
urlStrings := []string{}
for _, url := range urls {
urlStrings = append(urlStrings, url.String())
}
log.Info().Strs("urls", urlStrings).Msg("URLs to try")
location := strings.Join(urlStrings, ";")
c.AddAdvertisement(location)
c.Run(ctx)
}

1
go.mod
View File

@ -28,6 +28,7 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
github.com/fromkeith/gossdp v0.0.0-20180102154144-1b2c43f6886e // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/glebarez/go-sqlite v1.14.7 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect

2
go.sum
View File

@ -19,6 +19,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7 h1:tYwu/z8Y0Nkk
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/fromkeith/gossdp v0.0.0-20180102154144-1b2c43f6886e h1:cG4ivpkHpkmWTaaLrgekDVR0xAr87V697T2c+WnUdiY=
github.com/fromkeith/gossdp v0.0.0-20180102154144-1b2c43f6886e/go.mod h1:7xQpS/YtlWo38XfIqje9GgtlPuBRatYcL23GlYBtgWM=
github.com/getkin/kin-openapi v0.80.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
github.com/getkin/kin-openapi v0.88.0 h1:BjJ2JERWJbYE1o1RGEj/5LmR5qw7ecfl3O3su4ImR+0=
github.com/getkin/kin-openapi v0.88.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=

129
internal/own_url/interfaces.go Normal file
View File

@ -0,0 +1,129 @@
package own_url
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
import (
"errors"
"fmt"
"net"
"github.com/rs/zerolog/log"
)
var (
// ErrNoInterface is returned when no network interfaces with a real IP-address were found.
ErrNoInterface = errors.New("no network interface found")
)
// networkInterfaces returns a list of interface addresses.
// Only those addresses that can be eached by a unicast TCP/IP connection are returned.
func networkInterfaces(includeLinkLocal, includeLocalhost bool) ([]net.IP, error) {
log.Debug().Msg("iterating over all network interfaces")
interfaces, err := net.Interfaces()
if err != nil {
return []net.IP{}, err
}
usableAddresses := make([]net.IP, 0)
for _, iface := range interfaces {
if iface.Flags&net.FlagUp == 0 {
log.Debug().Str("interface", iface.Name).Msg("skipping down interface")
continue
}
addrs, err := iface.Addrs()
if err != nil {
continue
}
ifaceAddresses := make([]net.IP, 0)
for k := range addrs {
var ip net.IP
switch a := addrs[k].(type) {
case *net.IPAddr:
ip = a.IP
case *net.IPNet:
ip = a.IP
default:
log.Warn().
Interface("addr", addrs[k]).
Str("type", fmt.Sprintf("%T", addrs[k])).
Msg(" - skipping unknown interface type")
continue
}
logger := log.With().
Interface("ip", ip).
Str("iface", iface.Name).
Logger()
switch {
case ip.IsMulticast():
logger.Debug().Msg(" - skipping multicast")
case ip.IsMulticast():
logger.Debug().Msg(" - skipping multicast")
case ip.IsUnspecified():
logger.Debug().Msg(" - skipping unspecified")
case !includeLinkLocal && ip.IsLinkLocalUnicast():
logger.Debug().Msg(" - skipping link-local")
case !includeLocalhost && ip.IsLoopback():
logger.Debug().Msg(" - skipping localhost")
default:
logger.Debug().Msg(" - usable")
ifaceAddresses = append(ifaceAddresses, ip)
}
}
usableAddresses = append(usableAddresses, filterAddresses(ifaceAddresses)...)
}
if len(usableAddresses) == 0 {
return usableAddresses, ErrNoInterface
}
return usableAddresses, nil
}
// filterAddresses removes "privacy extension" addresses.
// It assumes the list of addresses belong to the same network interface, and
// that the OS reports preferred (i.e. private/random) addresses before
// non-random ones.
func filterAddresses(addrs []net.IP) []net.IP {
keep := make([]net.IP, 0)
var lastSeenIP net.IP
for _, addr := range addrs {
if addr.To4() != nil {
// IPv4 addresses are always kept.
keep = append(keep, addr)
continue
}
lastSeenIP = addr
}
if len(lastSeenIP) > 0 {
keep = append(keep, lastSeenIP)
}
return keep
}

98
internal/own_url/own_url.go Normal file
View File

@ -0,0 +1,98 @@
// Package own_url provides a way for a process to find a URL on which it can be reached.
package own_url
import (
"context"
"fmt"
"net"
"net/url"
"github.com/rs/zerolog/log"
)
func AvailableURLs(ctx context.Context, schema, listen string, includeLocal bool) ([]*url.URL, error) {
var (
host, port string
portnum int
err error
)
if listen == "" {
panic("empty 'listen' parameter")
}
// Figure out which port we're supposted to listen on.
if host, port, err = net.SplitHostPort(listen); err != nil {
return nil, fmt.Errorf("unable to split host and port in address '%s': %w", listen, err)
}
if portnum, err = net.DefaultResolver.LookupPort(ctx, "listen", port); err != nil {
return nil, fmt.Errorf("unable to look up port '%s': %w", port, err)
}
// If the host is empty or ::0/0.0.0.0, show a list of URLs to connect to.
listenSpecificHost := false
var ip net.IP
if host != "" {
ip = net.ParseIP(host)
if ip == nil {
addrs, erresolve := net.DefaultResolver.LookupHost(ctx, host)
if erresolve != nil {
return nil, fmt.Errorf("unable to resolve listen host '%v': %w", host, erresolve)
}
if len(addrs) > 0 {
ip = net.ParseIP(addrs[0])
}
}
if ip != nil && !ip.IsUnspecified() {
listenSpecificHost = true
}
}
if listenSpecificHost {
// We can just construct a URL here, since we know it's a specific host anyway.
log.Debug().Str("host", ip.String()).Msg("listening on host")
link := fmt.Sprintf("%s://%s:%d/", schema, host, portnum)
myURL, errparse := url.Parse(link)
if errparse != nil {
return nil, fmt.Errorf("unable to parse listen URL %s: %w", link, errparse)
}
return []*url.URL{myURL}, nil
}
log.Debug().Str("host", host).Msg("not listening on any specific host")
addrs, err := networkInterfaces(false, includeLocal)
if err == ErrNoInterface {
addrs, err = networkInterfaces(true, includeLocal)
}
if err != nil {
return nil, err
}
log.Debug().Msg("iterating network interfaces to find possible URLs for Flamenco Manager.")
links := make([]*url.URL, 0)
for _, addr := range addrs {
var strAddr string
if ipv4 := addr.To4(); ipv4 != nil {
strAddr = ipv4.String()
} else {
strAddr = fmt.Sprintf("[%s]", addr)
}
constructedURL := fmt.Sprintf("%s://%s:%d/", schema, strAddr, portnum)
parsedURL, err := url.Parse(constructedURL)
if err != nil {
log.Warn().
Str("address", strAddr).
Str("url", constructedURL).
Err(err).
Msg("skipping address, as it results in an unparseable URL")
continue
}
links = append(links, parsedURL)
}
return links, nil
}

26
internal/own_url/own_url_test.go Normal file
View File

@ -0,0 +1,26 @@
// Package own_url provides a way for a process to find a URL on which it can be reached.
package own_url
import (
"context"
"testing"
"time"
"github.com/mattn/go-colorable"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
func TestAvailableURLs(t *testing.T) {
output := zerolog.ConsoleWriter{Out: colorable.NewColorableStdout(), TimeFormat: time.RFC3339}
log.Logger = log.Output(output)
ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer ctxCancel()
_, err := AvailableURLs(ctx, "http", ":9999", true)
if err != nil {
t.Fatal(err)
}
// t.Fatalf("urls: %v", urls)
}

127
internal/upnp_ssdp/client.go Normal file
View File

@ -0,0 +1,127 @@
package upnp_ssdp
/* ***** BEGIN GPL LICENSE BLOCK *****
*
* Original Code Copyright (C) 2022 Blender Foundation.
*
* This file is part of Flamenco.
*
* Flamenco is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 3 of the License, or (at your option) any later
* version.
*
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Flamenco. If not, see <https://www.gnu.org/licenses/>.
*
* ***** END GPL LICENSE BLOCK ***** */
import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/fromkeith/gossdp"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
type Client struct {
ssdp *gossdp.ClientSsdp
log *zerolog.Logger
mutex *sync.Mutex
urls []string
}
func NewClient(logger zerolog.Logger) (*Client, error) {
client := Client{
log: &logger,
mutex: new(sync.Mutex),
urls: make([]string, 0),
}
wrap := wrappedLogger(&logger)
ssdp, err := gossdp.NewSsdpClientWithLogger(&client, wrap)
if err != nil {
return nil, fmt.Errorf("create UPnP/SSDP client: %w", err)
}
client.ssdp = ssdp
return &client, nil
}
func (c *Client) Run(ctx context.Context) ([]string, error) {
defer c.ssdp.Stop()
log.Debug().Msg("waiting for UPnP/SSDP answer")
go c.ssdp.Start()
var waitTime time.Duration
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(waitTime):
if err := c.ssdp.ListenFor(FlamencoServiceType); err != nil {
return nil, fmt.Errorf("unable to find Manager: %w", err)
}
waitTime = 1 * time.Second
urls := c.receivedURLs()
if len(urls) > 0 {
return urls, nil
}
}
}
}
// Response is called by the gossdp library on M-SEARCH responses.
func (c *Client) Response(message gossdp.ResponseMessage) {
logger := c.log.With().
Int("maxAge", message.MaxAge).
Str("searchType", message.SearchType).
Str("deviceID", message.DeviceId).
Str("usn", message.Usn).
Str("location", message.Location).
Str("server", message.Server).
Str("urn", message.Urn).
Logger()
if message.DeviceId != FlamencoUUID {
logger.Debug().Msg("ignoring message from unknown device")
return
}
logger.Debug().Msg("UPnP/SSDP message received")
c.appendURLs(message.Location)
}
func (c *Client) appendURLs(location string) {
urls := strings.Split(location, LocationSeparator)
c.mutex.Lock()
defer c.mutex.Unlock()
c.urls = append(c.urls, urls...)
c.log.Debug().
Int("new", len(urls)).
Int("total", len(c.urls)).
Msg("new URLs received")
}
// receivedURLs takes a thread-safe copy of the URLs received so far.
func (c *Client) receivedURLs() []string {
c.mutex.Lock()
defer c.mutex.Unlock()
urls := make([]string, len(c.urls))
copy(urls, c.urls)
return urls
}

60
internal/upnp_ssdp/logging.go Normal file
View File

@ -0,0 +1,60 @@
// package upnp_ssdp allows Workers to find their Manager on the LAN.
package upnp_ssdp
/* ***** BEGIN GPL LICENSE BLOCK *****
*
* Original Code Copyright (C) 2022 Blender Foundation.
*
* This file is part of Flamenco.
*
* Flamenco is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 3 of the License, or (at your option) any later
* version.
*
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Flamenco. If not, see <https://www.gnu.org/licenses/>.
*
* ***** END GPL LICENSE BLOCK ***** */
import (
"github.com/fromkeith/gossdp"
"github.com/rs/zerolog"
)
type ssdpLogger struct {
zlog *zerolog.Logger
}
var _ gossdp.LoggerInterface = (*ssdpLogger)(nil)
// wrappedLogger returns a gossdp.LoggerInterface-compatible wrapper around the given logger.
func wrappedLogger(logger *zerolog.Logger) *ssdpLogger {
return &ssdpLogger{
zlog: logger,
}
}
func (sl *ssdpLogger) Tracef(fmt string, args ...interface{}) {
sl.zlog.Debug().Msgf("SSDP: "+fmt, args...)
}
func (sl *ssdpLogger) Infof(fmt string, args ...interface{}) {
sl.zlog.Info().Msgf("SSDP: "+fmt, args...)
}
func (sl *ssdpLogger) Warnf(fmt string, args ...interface{}) {
sl.zlog.Warn().Msgf("SSDP: "+fmt, args...)
}
func (sl *ssdpLogger) Errorf(fmt string, args ...interface{}) {
// Errors from the SSDP library are logged by that library AND returned as
// error, which then triggers our own code to log the error as well. Since our
// code can provide more context about what it's doing, demote SSDP errors to
// the warning level.
sl.zlog.Warn().Msgf("SSDP: "+fmt, args...)
}

90
internal/upnp_ssdp/server.go Normal file
View File

@ -0,0 +1,90 @@
package upnp_ssdp
/* ***** BEGIN GPL LICENSE BLOCK *****
*
* Original Code Copyright (C) 2022 Blender Foundation.
*
* This file is part of Flamenco.
*
* Flamenco is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 3 of the License, or (at your option) any later
* version.
*
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Flamenco. If not, see <https://www.gnu.org/licenses/>.
*
* ***** END GPL LICENSE BLOCK ***** */
import (
"context"
"github.com/fromkeith/gossdp"
"github.com/rs/zerolog"
)
// Server advertises services via UPnP/SSDP.
type Server struct {
ssdp *gossdp.Ssdp
log *zerolog.Logger
wrappedLog *ssdpLogger
}
func NewServer(logger zerolog.Logger) (*Server, error) {
wrap := wrappedLogger(&logger)
ssdp, err := gossdp.NewSsdpWithLogger(nil, wrap)
if err != nil {
return nil, err
}
return &Server{ssdp, &logger, wrap}, nil
}
// AddAdvertisement adds a service advertisement for Flamenco Manager.
// Must be called before calling Run().
func (s *Server) AddAdvertisement(serviceLocation string) {
// Define the service we want to advertise
serverDef := gossdp.AdvertisableServer{
ServiceType: FlamencoServiceType,
DeviceUuid: FlamencoUUID,
Location: serviceLocation,
MaxAge: 3600, // Number of seconds this advertisement is valid for.
}
s.ssdp.AdvertiseServer(serverDef)
}
// Run starts the advertisement, and blocks until the context is closed.
func (s *Server) Run(ctx context.Context) {
s.log.Info().Msg("UPnP/SSDP advertisement starting")
isStopping := false
go func() {
// There is a bug in the SSDP library, where closing the server can cause a panic.
defer func() {
if isStopping {
// Only capture a panic when we expect one.
recover()
}
}()
s.ssdp.Start()
}()
<-ctx.Done()
s.log.Debug().Msg("UPnP/SSDP advertisement stopping")
// Sneakily disable warnings when shutting down, otherwise the read operation
// from the UDP socket will cause a warning.
tempLog := s.log.Level(zerolog.ErrorLevel)
s.wrappedLog.zlog = &tempLog
isStopping = true
s.ssdp.Stop()
s.wrappedLog.zlog = s.log
s.log.Info().Msg("UPnP/SSDP advertisement stopped")
}

29
internal/upnp_ssdp/upnp_ssdp.go Normal file
View File

@ -0,0 +1,29 @@
// package upnp_ssdp allows Workers to find their Manager on the LAN.
package upnp_ssdp
/* ***** BEGIN GPL LICENSE BLOCK *****
*
* Original Code Copyright (C) 2022 Blender Foundation.
*
* This file is part of Flamenco.
*
* Flamenco is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 3 of the License, or (at your option) any later
* version.
*
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Flamenco. If not, see <https://www.gnu.org/licenses/>.
*
* ***** END GPL LICENSE BLOCK ***** */
const (
FlamencoUUID = "aa80bc5f-d0af-46b8-8630-23bd7e80ec4d"
FlamencoServiceType = "urn:flamenco:manager:0"
LocationSeparator = ";"
)