feat(util): add util command to print GGUF informations (#2528)

Signed-off-by: Ettore Di Giacinto <[email protected]>

Author: mudler

core/cli/cli.go

@@ -13,4 +13,5 @@ var CLI struct {
 	TTS        TTSCMD        `cmd:"" help:"Convert text to speech"`
 	Transcript TranscriptCMD `cmd:"" help:"Convert audio to text"`
 	Worker     worker.Worker `cmd:"" help:"Run workers to distribute workload (llama.cpp-only)"`
+	Util       UtilCMD       `cmd:"" help:"Utility commands"`
 }

core/cli/util.go

@@ -0,0 +1,39 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/rs/zerolog/log"
+
+	cliContext "github.com/go-skynet/LocalAI/core/cli/context"
+	gguf "github.com/thxcode/gguf-parser-go"
+)
+
+type UtilCMD struct {
+	GGUFInfo GGUFInfoCMD `cmd:"" name:"gguf-info" help:"Get information about a GGUF file"`
+}
+
+type GGUFInfoCMD struct {
+	Args []string `arg:"" optional:"" name:"args" help:"Arguments to pass to the utility command"`
+}
+
+func (u *GGUFInfoCMD) Run(ctx *cliContext.Context) error {
+	if u.Args == nil || len(u.Args) == 0 {
+		return fmt.Errorf("no GGUF file provided")
+	}
+	// We try to guess only if we don't have a template defined already
+	f, err := gguf.ParseGGUFFile(u.Args[0])
+	if err != nil {
+		// Only valid for gguf files
+		log.Error().Msgf("guessDefaultsFromFile: %s", "not a GGUF file")
+		return err
+	}
+
+	log.Info().
+		Any("eosTokenID", f.Tokenizer().EOSTokenID).
+		Any("bosTokenID", f.Tokenizer().BOSTokenID).
+		Any("modelName", f.Model().Name).
+		Any("architecture", f.Architecture().Architecture).Msgf("GGUF file loaded: %s", u.Args[0])
+
+	return nil
+}

main.go

@@ -42,7 +42,7 @@ func main() {
 
 	for _, envFile := range envFiles {
 		if _, err := os.Stat(envFile); err == nil {
-			log.Info().Str("envFile", envFile).Msg("loading environment variables from file")
+			log.Info().Str("envFile", envFile).Msg("env file found, loading environment variables from file")
 			err = godotenv.Load(envFile)
 			if err != nil {
 				log.Error().Err(err).Str("envFile", envFile).Msg("failed to load environment variables from file")
@@ -117,6 +117,7 @@ Version: ${version}
 
 	// Run the thing!
 	err = ctx.Run(&cli.CLI.Context)
-
-	ctx.FatalIfErrorf(err)
+	if err != nil {
+		log.Fatal().Err(err).Msg("Error running the application")
+	}
 }

pkg/templates/cache.go

@@ -82,6 +82,7 @@ func (tc *TemplateCache) loadTemplateIfExists(templateType TemplateType, templat
 		return fmt.Errorf("template file outside path: %s", file)
 	}
 
+	// can either be a file in the system or a string with the template
 	if utils.ExistsInPath(tc.templatesPath, modelTemplateFile) {
 		d, err := os.ReadFile(file)
 		if err != nil {