/* (c) 2019, Blender Foundation - Sybren A. Stüvel * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package shaman import ( "context" "io" "runtime" "sync" "github.com/rs/zerolog/log" "projects.blender.org/studio/flamenco/pkg/api" "projects.blender.org/studio/flamenco/pkg/shaman/checkout" "projects.blender.org/studio/flamenco/pkg/shaman/config" "projects.blender.org/studio/flamenco/pkg/shaman/fileserver" "projects.blender.org/studio/flamenco/pkg/shaman/filestore" "projects.blender.org/studio/flamenco/pkg/shaman/jwtauth" "projects.blender.org/studio/flamenco/pkg/sysinfo" "projects.blender.org/studio/flamenco/pkg/website" ) var ErrDoesNotExist = checkout.ErrDoesNotExist // Server represents a Shaman Server. type Server struct { config config.Config auther jwtauth.Authenticator fileStore *filestore.Store fileServer *fileserver.FileServer checkoutMan *checkout.Manager shutdownChan chan struct{} wg sync.WaitGroup } // NewServer creates a new Shaman server. func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server { if !conf.Enabled { log.Info().Msg("shaman server is disabled") return nil } checkPlatformSymlinkSupport() if conf.StoragePath == "" { log.Error().Interface("config", conf).Msg("shaman: no checkout path configured, unable to start") return nil } fileStore := filestore.New(conf) checkoutMan := checkout.NewManager(conf, fileStore) fileServer := fileserver.New(fileStore) shamanServer := &Server{ config: conf, auther: auther, fileStore: fileStore, fileServer: fileServer, checkoutMan: checkoutMan, shutdownChan: make(chan struct{}), wg: sync.WaitGroup{}, } return shamanServer } func checkPlatformSymlinkSupport() { canSymlink, err := sysinfo.CanSymlink() switch { case err != nil: log.Warn().AnErr("cause", err). Msgf("unable to determine whether this platform can use symlinks. "+ "Please report a bug about this: %s", website.BugReportURL) return case canSymlink: return } osDetail, err := sysinfo.Description() if err != nil { log.Warn().AnErr("cause", err). Msgf("unable to find details of your operating system. "+ "Please report a bug about this: %s", website.BugReportURL) return } log.Warn(). Str("os", runtime.GOOS). Str("arch", runtime.GOARCH). Str("osDetail", osDetail). Msgf("this platform does not reliably support symbolic links, "+ "see %s", website.ShamanRequirementsURL) } // Go starts goroutines for background operations. // After Go() has been called, use Close() to stop those goroutines. func (s *Server) Go() { log.Info().Msg("Shaman server starting") s.fileServer.Go() if s.config.GarbageCollect.Period == 0 { log.Warn().Msg("garbage collection disabled, set garbageCollect.period > 0 in configuration") } else if s.config.GarbageCollect.SilentlyDisable { log.Debug().Msg("not starting garbage collection") } else { s.wg.Add(1) go s.periodicCleanup() } } // Close shuts down the Shaman server. func (s *Server) Close() { log.Info().Msg("shutting down Shaman server") close(s.shutdownChan) s.fileServer.Close() s.checkoutMan.Close() s.wg.Wait() } func (s *Server) IsEnabled() bool { return s != nil && s.config.Enabled } // Checkout creates a directory, and symlinks the required files into it. The // files must all have been uploaded to Shaman before calling this. func (s *Server) Checkout(ctx context.Context, checkout api.ShamanCheckout) (string, error) { return s.checkoutMan.Checkout(ctx, checkout) } // Requirements checks a Shaman Requirements file, and returns the subset // containing the unknown files. func (s *Server) Requirements(ctx context.Context, requirements api.ShamanRequirementsRequest) (api.ShamanRequirementsResponse, error) { return s.checkoutMan.ReportRequirements(ctx, requirements) } var fsStatusToApiStatus = map[filestore.FileStatus]api.ShamanFileStatus{ filestore.StatusDoesNotExist: api.ShamanFileStatusUnknown, filestore.StatusUploading: api.ShamanFileStatusUploading, filestore.StatusStored: api.ShamanFileStatusStored, } // Check the status of a file on the Shaman server. // status (stored, currently being uploaded, unknown). func (s *Server) FileStoreCheck(ctx context.Context, checksum string, filesize int64) api.ShamanFileStatus { status := s.fileServer.CheckFile(checksum, filesize) apiStatus, ok := fsStatusToApiStatus[status] if !ok { log.Warn(). Str("checksum", checksum). Int64("filesize", filesize). Int("fileserverStatus", int(status)). Msg("shaman: unknown status on fileserver") return api.ShamanFileStatusUnknown } return apiStatus } // Store a new file on the Shaman server. Note that the Shaman server can return // early when another client finishes uploading the exact same file, to prevent // double uploads. func (s *Server) FileStore(ctx context.Context, file io.ReadCloser, checksum string, filesize int64, canDefer bool, originalFilename string) error { err := s.fileServer.ReceiveFile(ctx, file, checksum, filesize, canDefer, originalFilename) // TODO: Maybe translate this error into something that can be understood by // the caller without relying on types declared in the `fileserver` package? return err } // EraseCheckout deletes the symlinks and the directory structure that makes up the checkout. func (s *Server) EraseCheckout(checkoutID string) error { return s.checkoutMan.EraseCheckout(checkoutID) }