feat: Add backend gallery

This PR add support to manage backends as similar to models. There is
now available a backend gallery which can be used to install and remove
extra backends.
The backend gallery can be configured similarly as a model gallery, and
API calls allows to install and remove new backends in runtime, and as
well during the startup phase of LocalAI.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

core/application/startup.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
 	"github.com/mudler/LocalAI/core/services"
 	"github.com/mudler/LocalAI/internal"
 	"github.com/mudler/LocalAI/pkg/assets"
@@ -60,12 +61,20 @@ func New(opts ...config.AppOption) (*Application, error) {
 		log.Error().Err(err).Msg("error installing models")
 	}
 
+	if err := pkgStartup.InstallExternalBackends(options.BackendGalleries, options.BackendsPath, nil, options.ExternalBackends...); err != nil {
+		log.Error().Err(err).Msg("error installing external backends")
+	}
+
 	configLoaderOpts := options.ToConfigLoaderOptions()
 
 	if err := application.BackendLoader().LoadBackendConfigsFromPath(options.ModelPath, configLoaderOpts...); err != nil {
 		log.Error().Err(err).Msg("error loading config files")
 	}
 
+	if err := gallery.RegisterBackends(options.BackendsPath, application.ModelLoader()); err != nil {
+		log.Error().Err(err).Msg("error registering external backends")
+	}
+
 	if options.ConfigFile != "" {
 		if err := application.BackendLoader().LoadMultipleBackendConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
 			log.Error().Err(err).Msg("error loading config file")

core/cli/models.go

@@ -86,7 +86,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
 		modelURI := downloader.URI(modelName)
 
 		if !modelURI.LooksLikeOCI() {
-			model := gallery.FindModel(models, modelName, mi.ModelsPath)
+			model := gallery.FindGalleryElement(models, modelName, mi.ModelsPath)
 			if model == nil {
 				log.Error().Str("model", modelName).Msg("model not found")
 				return err

core/cli/run.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -19,6 +20,8 @@ import (
 type RunCMD struct {
 	ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"`
 
+	ExternalBackends             []string      `env:"LOCALAI_EXTERNAL_BACKENDS,EXTERNAL_BACKENDS" help:"A list of external backends to load from gallery on boot" group:"backends"`
+	BackendsPath                 string        `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
 	ModelsPath                   string        `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
 	BackendAssetsPath            string        `env:"LOCALAI_BACKEND_ASSETS_PATH,BACKEND_ASSETS_PATH" type:"path" default:"/tmp/localai/backend_data" help:"Path used to extract libraries that are required by some of the backends in runtime" group:"storage"`
 	GeneratedContentPath         string        `env:"LOCALAI_GENERATED_CONTENT_PATH,GENERATED_CONTENT_PATH" type:"path" default:"/tmp/generated/content" help:"Location for generated content (e.g. images, audio, videos)" group:"storage"`
@@ -27,8 +30,8 @@ type RunCMD struct {
 	LocalaiConfigDir             string        `env:"LOCALAI_CONFIG_DIR" type:"path" default:"${basepath}/configuration" help:"Directory for dynamic loading of certain configuration files (currently api_keys.json and external_backends.json)" group:"storage"`
 	LocalaiConfigDirPollInterval time.Duration `env:"LOCALAI_CONFIG_DIR_POLL_INTERVAL" help:"Typically the config path picks up changes automatically, but if your system has broken fsnotify events, set this to an interval to poll the LocalAI Config Dir (example: 1m)" group:"storage"`
 	// The alias on this option is there to preserve functionality with the old `--config-file` parameter
-	ModelsConfigFile string `env:"LOCALAI_MODELS_CONFIG_FILE,CONFIG_FILE" aliases:"config-file" help:"YAML file containing a list of model backend configs" group:"storage"`
-
+	ModelsConfigFile    string   `env:"LOCALAI_MODELS_CONFIG_FILE,CONFIG_FILE" aliases:"config-file" help:"YAML file containing a list of model backend configs" group:"storage"`
+	BackendGalleries    string   `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"`
 	Galleries           string   `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models" default:"${galleries}"`
 	AutoloadGalleries   bool     `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models"`
 	PreloadModels       string   `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"`
@@ -73,11 +76,15 @@ type RunCMD struct {
 }
 
 func (r *RunCMD) Run(ctx *cliContext.Context) error {
+	os.MkdirAll(r.BackendsPath, 0750)
+	os.MkdirAll(r.ModelsPath, 0750)
+
 	opts := []config.AppOption{
 		config.WithConfigFile(r.ModelsConfigFile),
 		config.WithJSONStringPreload(r.PreloadModels),
 		config.WithYAMLConfigPreload(r.PreloadModelsConfig),
 		config.WithModelPath(r.ModelsPath),
+		config.WithBackendsPath(r.BackendsPath),
 		config.WithContextSize(r.ContextSize),
 		config.WithDebug(zerolog.GlobalLevel() <= zerolog.DebugLevel),
 		config.WithGeneratedContentDir(r.GeneratedContentPath),
@@ -87,6 +94,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithDynamicConfigDirPollInterval(r.LocalaiConfigDirPollInterval),
 		config.WithF16(r.F16),
 		config.WithStringGalleries(r.Galleries),
+		config.WithBackendGalleries(r.BackendGalleries),
 		config.WithCors(r.CORS),
 		config.WithCorsAllowOrigins(r.CORSAllowOrigins),
 		config.WithCsrf(r.CSRF),
@@ -97,6 +105,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithUploadLimitMB(r.UploadLimit),
 		config.WithApiKeys(r.APIKeys),
 		config.WithModelsURL(append(r.Models, r.ModelArgs...)...),
+		config.WithExternalBackends(r.ExternalBackends...),
 		config.WithOpaqueErrors(r.OpaqueErrors),
 		config.WithEnforcedPredownloadScans(!r.DisablePredownloadScan),
 		config.WithSubtleKeyComparison(r.UseSubtleKeyComparison),

core/config/application_config.go

@@ -15,6 +15,8 @@ type ApplicationConfig struct {
 	Context                             context.Context
 	ConfigFile                          string
 	ModelPath                           string
+	BackendsPath                        string
+	ExternalBackends                    []string
 	LibPath                             string
 	UploadLimitMB, Threads, ContextSize int
 	F16                                 bool
@@ -45,7 +47,8 @@ type ApplicationConfig struct {
 	DisableGalleryEndpoint             bool
 	LoadToMemory                       []string
 
-	Galleries []Gallery
+	Galleries        []Gallery
+	BackendGalleries []Gallery
 
 	BackendAssets     *rice.Box
 	AssetsDestination string
@@ -95,6 +98,18 @@ func WithModelPath(path string) AppOption {
 	}
 }
 
+func WithBackendsPath(path string) AppOption {
+	return func(o *ApplicationConfig) {
+		o.BackendsPath = path
+	}
+}
+
+func WithExternalBackends(backends ...string) AppOption {
+	return func(o *ApplicationConfig) {
+		o.ExternalBackends = backends
+	}
+}
+
 func WithMachineTag(tag string) AppOption {
 	return func(o *ApplicationConfig) {
 		o.MachineTag = tag
@@ -218,6 +233,20 @@ func WithStringGalleries(galls string) AppOption {
 	}
 }
 
+func WithBackendGalleries(galls string) AppOption {
+	return func(o *ApplicationConfig) {
+		if galls == "" {
+			o.BackendGalleries = []Gallery{}
+			return
+		}
+		var galleries []Gallery
+		if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
+			log.Error().Err(err).Msg("failed loading galleries")
+		}
+		o.BackendGalleries = append(o.BackendGalleries, galleries...)
+	}
+}
+
 func WithGalleries(galleries []Gallery) AppOption {
 	return func(o *ApplicationConfig) {
 		o.Galleries = append(o.Galleries, galleries...)

core/gallery/backend_types.go

@@ -0,0 +1,35 @@
+package gallery
+
+import "github.com/mudler/LocalAI/core/config"
+
+type GalleryBackend struct {
+	Metadata `json:",inline" yaml:",inline"`
+	Alias    string `json:"alias,omitempty" yaml:"alias,omitempty"`
+	URI      string `json:"uri,omitempty" yaml:"uri,omitempty"`
+}
+
+type GalleryBackends []*GalleryBackend
+
+func (m *GalleryBackend) SetGallery(gallery config.Gallery) {
+	m.Gallery = gallery
+}
+
+func (m *GalleryBackend) SetInstalled(installed bool) {
+	m.Installed = installed
+}
+
+func (m *GalleryBackend) GetName() string {
+	return m.Name
+}
+
+func (m *GalleryBackend) GetGallery() config.Gallery {
+	return m.Gallery
+}
+
+func (m *GalleryBackend) GetDescription() string {
+	return m.Description
+}
+
+func (m *GalleryBackend) GetTags() []string {
+	return m.Tags
+}

core/gallery/backends.go

@@ -0,0 +1,107 @@
+package gallery
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/oci"
+)
+
+// Installs a model from the gallery
+func InstallBackendFromGallery(galleries []config.Gallery, name string, basePath string, downloadStatus func(string, string, string, float64)) error {
+	backends, err := AvailableBackends(galleries, basePath)
+	if err != nil {
+		return err
+	}
+
+	backend := FindGalleryElement(backends, name, basePath)
+	if backend == nil {
+		return fmt.Errorf("no model found with name %q", name)
+	}
+
+	return InstallBackend(basePath, backend, downloadStatus)
+}
+
+func InstallBackend(basePath string, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error {
+	// Create base path if it doesn't exist
+	err := os.MkdirAll(basePath, 0750)
+	if err != nil {
+		return fmt.Errorf("failed to create base path: %v", err)
+	}
+
+	name := config.Name
+
+	img, err := oci.GetImage(config.URI, "", nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to get image %q: %v", config.URI, err)
+	}
+
+	backendPath := filepath.Join(basePath, name)
+	if err := os.MkdirAll(backendPath, 0750); err != nil {
+		return fmt.Errorf("failed to create backend path %q: %v", backendPath, err)
+	}
+
+	if err := oci.ExtractOCIImage(img, backendPath); err != nil {
+		return fmt.Errorf("failed to extract image %q: %v", config.URI, err)
+	}
+
+	if config.Alias != "" {
+		// Write an alias file inside
+		aliasFile := filepath.Join(backendPath, "alias")
+		if err := os.WriteFile(aliasFile, []byte(config.Alias), 0644); err != nil {
+			return fmt.Errorf("failed to write alias file %q: %v", aliasFile, err)
+		}
+	}
+
+	return nil
+}
+
+func DeleteBackendFromSystem(basePath string, name string) error {
+	backendFile := filepath.Join(basePath, name)
+
+	return os.RemoveAll(backendFile)
+}
+
+func ListSystemBackends(basePath string) (map[string]string, error) {
+	backends, err := os.ReadDir(basePath)
+	if err != nil {
+		return nil, err
+	}
+
+	backendsNames := make(map[string]string)
+
+	for _, backend := range backends {
+		if backend.IsDir() {
+			runFile := filepath.Join(basePath, backend.Name(), "run.sh")
+			backendsNames[backend.Name()] = runFile
+
+			aliasFile := filepath.Join(basePath, backend.Name(), "alias")
+			if _, err := os.Stat(aliasFile); err == nil {
+				// read the alias file, and use it as key
+				alias, err := os.ReadFile(aliasFile)
+				if err != nil {
+					return nil, err
+				}
+				backendsNames[string(alias)] = runFile
+			}
+		}
+	}
+
+	return backendsNames, nil
+}
+
+func RegisterBackends(basePath string, modelLoader *model.ModelLoader) error {
+	backends, err := ListSystemBackends(basePath)
+	if err != nil {
+		return err
+	}
+
+	for name, runFile := range backends {
+		modelLoader.SetExternalBackend(name, runFile)
+	}
+
+	return nil
+}