8080 if e .IsDir () {
8181 continue
8282 }
83+ if strings .HasSuffix (e .Name (), ".log" ) {
84+ continue
85+ }
8386
8487 // Skip the llama.cpp variants if we are autoDetecting
8588 // But we always load the fallback variant if it exists
@@ -265,12 +268,12 @@ func selectGRPCProcess(backend, assetDir string, f16 bool) string {
265268
266269// starts the grpcModelProcess for the backend, and returns a grpc client
267270// It also loads the model
268- func (ml * ModelLoader ) grpcModel (backend string , o * Options ) func (string , string ) (ModelAddress , error ) {
269- return func (modelName , modelFile string ) (ModelAddress , error ) {
271+ func (ml * ModelLoader ) grpcModel (backend string , o * Options ) func (string , string ) (* Model , error ) {
272+ return func (modelName , modelFile string ) (* Model , error ) {
270273
271274 log .Debug ().Msgf ("Loading Model %s with gRPC (file: %s) (backend: %s): %+v" , modelName , modelFile , backend , * o )
272275
273- var client ModelAddress
276+ var client * Model
274277
275278 getFreeAddress := func () (string , error ) {
276279 port , err := freeport .GetFreePort ()
@@ -298,26 +301,26 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
298301 log .Debug ().Msgf ("external backend is file: %+v" , fi )
299302 serverAddress , err := getFreeAddress ()
300303 if err != nil {
301- return "" , fmt .Errorf ("failed allocating free ports: %s" , err .Error ())
304+ return nil , fmt .Errorf ("failed allocating free ports: %s" , err .Error ())
302305 }
303306 // Make sure the process is executable
304307 if err := ml .startProcess (uri , o .model , serverAddress ); err != nil {
305308 log .Error ().Err (err ).Str ("path" , uri ).Msg ("failed to launch " )
306- return "" , err
309+ return nil , err
307310 }
308311
309312 log .Debug ().Msgf ("GRPC Service Started" )
310313
311- client = ModelAddress (serverAddress )
314+ client = NewModel (serverAddress )
312315 } else {
313316 log .Debug ().Msg ("external backend is uri" )
314317 // address
315- client = ModelAddress (uri )
318+ client = NewModel (uri )
316319 }
317320 } else {
318321 grpcProcess := backendPath (o .assetDir , backend )
319322 if err := utils .VerifyPath (grpcProcess , o .assetDir ); err != nil {
320- return "" , fmt .Errorf ("grpc process not found in assetdir: %s" , err .Error ())
323+ return nil , fmt .Errorf ("grpc process not found in assetdir: %s" , err .Error ())
321324 }
322325
323326 if autoDetect {
@@ -329,12 +332,12 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
329332
330333 // Check if the file exists
331334 if _ , err := os .Stat (grpcProcess ); os .IsNotExist (err ) {
332- return "" , fmt .Errorf ("grpc process not found: %s. some backends(stablediffusion, tts) require LocalAI compiled with GO_TAGS" , grpcProcess )
335+ return nil , fmt .Errorf ("grpc process not found: %s. some backends(stablediffusion, tts) require LocalAI compiled with GO_TAGS" , grpcProcess )
333336 }
334337
335338 serverAddress , err := getFreeAddress ()
336339 if err != nil {
337- return "" , fmt .Errorf ("failed allocating free ports: %s" , err .Error ())
340+ return nil , fmt .Errorf ("failed allocating free ports: %s" , err .Error ())
338341 }
339342
340343 args := []string {}
@@ -344,12 +347,12 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
344347
345348 // Make sure the process is executable in any circumstance
346349 if err := ml .startProcess (grpcProcess , o .model , serverAddress , args ... ); err != nil {
347- return "" , err
350+ return nil , err
348351 }
349352
350353 log .Debug ().Msgf ("GRPC Service Started" )
351354
352- client = ModelAddress (serverAddress )
355+ client = NewModel (serverAddress )
353356 }
354357
355358 // Wait for the service to start up
@@ -369,7 +372,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
369372
370373 if ! ready {
371374 log .Debug ().Msgf ("GRPC Service NOT ready" )
372- return "" , fmt .Errorf ("grpc service not ready" )
375+ return nil , fmt .Errorf ("grpc service not ready" )
373376 }
374377
375378 options := * o .gRPCOptions
@@ -380,27 +383,16 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
380383
381384 res , err := client .GRPC (o .parallelRequests , ml .wd ).LoadModel (o .context , & options )
382385 if err != nil {
383- return "" , fmt .Errorf ("could not load model: %w" , err )
386+ return nil , fmt .Errorf ("could not load model: %w" , err )
384387 }
385388 if ! res .Success {
386- return "" , fmt .Errorf ("could not load model (no success): %s" , res .Message )
389+ return nil , fmt .Errorf ("could not load model (no success): %s" , res .Message )
387390 }
388391
389392 return client , nil
390393 }
391394}
392395
393- func (ml * ModelLoader ) resolveAddress (addr ModelAddress , parallel bool ) (grpc.Backend , error ) {
394- if parallel {
395- return addr .GRPC (parallel , ml .wd ), nil
396- }
397-
398- if _ , ok := ml .grpcClients [string (addr )]; ! ok {
399- ml .grpcClients [string (addr )] = addr .GRPC (parallel , ml .wd )
400- }
401- return ml .grpcClients [string (addr )], nil
402- }
403-
404396func (ml * ModelLoader ) BackendLoader (opts ... Option ) (client grpc.Backend , err error ) {
405397 o := NewOptions (opts ... )
406398
@@ -425,7 +417,6 @@ func (ml *ModelLoader) BackendLoader(opts ...Option) (client grpc.Backend, err e
425417 log .Error ().Err (err ).Str ("keptModel" , o .model ).Msg ("error while shutting down all backends except for the keptModel" )
426418 return nil , err
427419 }
428-
429420 }
430421
431422 var backendToConsume string
@@ -438,26 +429,28 @@ func (ml *ModelLoader) BackendLoader(opts ...Option) (client grpc.Backend, err e
438429 backendToConsume = backend
439430 }
440431
441- addr , err := ml .LoadModel (o .model , ml .grpcModel (backendToConsume , o ))
432+ model , err := ml .LoadModel (o .model , ml .grpcModel (backendToConsume , o ))
442433 if err != nil {
443434 return nil , err
444435 }
445436
446- return ml . resolveAddress ( addr , o .parallelRequests )
437+ return model . GRPC ( o .parallelRequests , ml . wd ), nil
447438}
448439
449440func (ml * ModelLoader ) GreedyLoader (opts ... Option ) (grpc.Backend , error ) {
450441 o := NewOptions (opts ... )
451442
452443 ml .mu .Lock ()
444+
453445 // Return earlier if we have a model already loaded
454446 // (avoid looping through all the backends)
455- if m := ml .CheckIsLoaded (o .model ); m != "" {
447+ if m := ml .CheckIsLoaded (o .model ); m != nil {
456448 log .Debug ().Msgf ("Model '%s' already loaded" , o .model )
457449 ml .mu .Unlock ()
458450
459- return ml . resolveAddress ( m , o .parallelRequests )
451+ return m . GRPC ( o .parallelRequests , ml . wd ), nil
460452 }
453+
461454 // If we can have only one backend active, kill all the others (except external backends)
462455 if o .singleActiveBackend {
463456 log .Debug ().Msgf ("Stopping all backends except '%s'" , o .model )
0 commit comments