Add 'Backends' init and 'XmlDecoder' register

Author: granty1

config/config.go

@@ -1,9 +1,11 @@
 package config
 
 import (
+	"errors"
 	"fmt"
 	"melody/encoding"
 	"regexp"
+	"strings"
 	"time"
 )
 
@@ -16,9 +18,10 @@ const (
 )
 
 var (
-	RoutingPattern       = ColonRouterPatternBuilder
-	debugPattern         = "^[^/]|/__debug(/.*)?$"
-	simpleURLKeysPattern = regexp.MustCompile(`\{([a-zA-Z\-_0-9]+)\}`)
+	RoutingPattern         = ColonRouterPatternBuilder
+	debugPattern           = "^[^/]|/__debug(/.*)?$"
+	simpleURLKeysPattern   = regexp.MustCompile(`\{([a-zA-Z\-_0-9]+)\}`)
+	errInvalidNoOpEncoding = errors.New("can not use NoOp encoding with more than one backends connected to the same endpoint")
 )
 
 //ServiceConfig contains all config in melody server.
@@ -29,8 +32,8 @@ type ServiceConfig struct {
 	Host        []string          `mapstructure:"host"`
 	Endpoints   []*EndpointConfig `mapstructure:"endpoints"`
 
-	OutputEncoding string `mapstructure:"output_encoding"`
-	CacheTTL time.Duration `mapstructure:"cache_ttl"`
+	OutputEncoding string        `mapstructure:"output_encoding"`
+	CacheTTL       time.Duration `mapstructure:"cache_ttl"`
 
 	MaxIdleConnsPerHost int `mapstructure:"max_idle_connections_per_host"`
 
@@ -64,6 +67,42 @@ type EndpointConfig struct {
 }
 
 type Backend struct {
+	// the name of the group the response should be moved to. If empty, the response is
+	// not changed
+	Group string `mapstructure:"group"`
+	// HTTP method of the request to send to the backend
+	Method string `mapstructure:"method"`
+	// Set of hosts of the API
+	Host []string `mapstructure:"host"`
+	// False if the hostname should be sanitized
+	HostSanitizationDisabled bool `mapstructure:"disable_host_sanitize"`
+	// URL pattern to use to locate the resource to be consumed
+	URLPattern string `mapstructure:"url_pattern"`
+	// set of response fields to remove. If empty, the filter id not used
+	Blacklist []string `mapstructure:"blacklist"`
+	// set of response fields to allow. If empty, the filter id not used
+	Whitelist []string `mapstructure:"whitelist"`
+	// map of response fields to be renamed and their new names
+	Mapping map[string]string `mapstructure:"mapping"`
+	// the encoding format
+	Encoding string `mapstructure:"encoding"`
+	// the response to process is a collection
+	IsCollection bool `mapstructure:"is_collection"`
+	// name of the field to extract to the root. If empty, the formater will do nothing
+	Target string `mapstructure:"target"`
+	// name of the service discovery driver to use
+	//SD string `mapstructure:"sd"`
+
+	// list of keys to be replaced in the URLPattern
+	URLKeys []string
+	// number of concurrent calls this endpoint must send to the API
+	ConcurrentCalls int
+	// timeout of this backend
+	Timeout time.Duration
+	// decoder to use in order to parse the received response from the API
+	Decoder encoding.Decoder `json:"-"`
+	// Backend Extra configuration for customized behaviours
+	ExtraConfig ExtraConfig `mapstructure:"extra_config"`
 }
 
 //Extra config for melody
@@ -153,11 +192,20 @@ func (s *ServiceConfig) initEndpoints() error {
 		// 初始化一些全局默认值
 		s.initDefaultEndpoints(i)
 
-		//TODO NOOP encode (目前不知道noop什么意思)
+		//判断encode为NOOP时，backends是否大于1
+		if e.OutputEncoding == encoding.NOOP && len(e.Backends) > 1 {
+			return errInvalidNoOpEncoding
+		}
 
 		e.ExtraConfig.sanitize()
 
-		//TODO 初始化 Endpoints下的Backends （*）
+		for j, b := range e.Backends {
+			s.initDefaultBackends(i, j)
+
+			//TODO 初始化、解析Backends的url以及对应的参数 （*）
+
+			b.ExtraConfig.sanitize()
+		}
 	}
 
 	return nil
@@ -213,6 +261,28 @@ func (s *ServiceConfig) initDefaultEndpoints(i int) {
 	}
 }
 
+func (s *ServiceConfig) initDefaultBackends(e, b int) {
+	endpoint := s.Endpoints[e]
+	backend := endpoint.Backends[b]
+	if len(backend.Host) == 0 {
+		backend.Host = s.Host
+	}
+
+	if !backend.HostSanitizationDisabled {
+		backend.Host = s.uriParser.CleanHosts(backend.Host)
+	}
+
+	if backend.Method == "" {
+		backend.Method = endpoint.Method
+	}
+
+	backend.Timeout = endpoint.Timeout
+	backend.ConcurrentCalls = endpoint.ConcurrentCalls
+
+	//根据配置的encoding， 加载对应的Decoder
+	backend.Decoder = encoding.Get(strings.ToLower(backend.Encoding))(backend.IsCollection)
+}
+
 func (e *EndpointConfig) validate() error {
 	matched, err := regexp.MatchString(debugPattern, e.Endpoint)
 	if err != nil {

core/melody/encoding.go

@@ -0,0 +1,8 @@
+package melody
+
+import xml "melody/middleware/melody-xml"
+
+func RegisterEncoders() {
+	xml.Register()
+	//TODO RSS register
+}

core/version.go

@@ -1,4 +1,4 @@
 package core
 
 //MelodyVersion export the project version.
-var MelodyVersion = "0.1.1"
+var MelodyVersion = "0.0.1"

docs/关键的结构体说明.md

@@ -116,3 +116,46 @@ type EndpointConfig struct {
 	OutputEncoding string `mapstructure:"output_encoding"`
 }
 ```
+
+## Backend
+
+```go
+type Backend struct {
+	// the name of the group the response should be moved to. If empty, the response is
+	// not changed
+	Group string `mapstructure:"group"`
+	// HTTP method of the request to send to the backend
+	Method string `mapstructure:"method"`
+	// Set of hosts of the API
+	Host []string `mapstructure:"host"`
+	// False if the hostname should be sanitized
+	HostSanitizationDisabled bool `mapstructure:"disable_host_sanitize"`
+	// URL pattern to use to locate the resource to be consumed
+	URLPattern string `mapstructure:"url_pattern"`
+	// set of response fields to remove. If empty, the filter id not used
+	Blacklist []string `mapstructure:"blacklist"`
+	// set of response fields to allow. If empty, the filter id not used
+	Whitelist []string `mapstructure:"whitelist"`
+	// map of response fields to be renamed and their new names
+	Mapping map[string]string `mapstructure:"mapping"`
+	// the encoding format
+	Encoding string `mapstructure:"encoding"`
+	// the response to process is a collection
+	IsCollection bool `mapstructure:"is_collection"`
+	// name of the field to extract to the root. If empty, the formater will do nothing
+	Target string `mapstructure:"target"`
+	// name of the service discovery driver to use
+	SD string `mapstructure:"sd"`
+
+	// list of keys to be replaced in the URLPattern
+	URLKeys []string
+	// number of concurrent calls this endpoint must send to the API
+	ConcurrentCalls int
+	// timeout of this backend
+	Timeout time.Duration
+	// decoder to use in order to parse the received response from the API
+	Decoder encoding.Decoder `json:"-"`
+	// Backend Extra configuration for customized behaviours
+	ExtraConfig ExtraConfig `mapstructure:"extra_config"`
+}
+```

encoding/encoding.go

@@ -0,0 +1,26 @@
+package encoding
+
+import "io"
+
+const NOOP = "no-op"
+
+// NoOpDecoder implements the Decoder interface
+func NoOpDecoder(_ io.Reader, _ *map[string]interface{}) error { return nil }
+
+func NoOpDecoderFactory(_ bool) func(io.Reader, *map[string]interface{}) error { return NoOpDecoder }
+
+// Decoder a accept a param that can be read , and a target map pointer to write
+type Decoder func(io.Reader, *map[string]interface{}) error
+
+// DecoderFactory returns a Decoder
+// 1. EntityDecoder {}
+// 2. CollectionDecoder []
+type DecoderFactory func(bool) func(io.Reader, *map[string]interface{}) error
+
+func Get(key string) DecoderFactory {
+	return decoders.Get(key)
+}
+
+func Register(key string, factory DecoderFactory) error {
+	return decoders.Register(key, factory)
+}

encoding/json.go

@@ -1,3 +1,39 @@
 package encoding
 
+import (
+	"encoding/json"
+	"io"
+)
+
 const JSON = "json"
+
+func NewJSONDecoder(isCollection bool) func(io.Reader, *map[string]interface{}) error {
+	if isCollection {
+		return JSONCollectionDecoder()
+	} else {
+		return JSONDecoder()
+	}
+
+	return nil
+}
+
+func JSONCollectionDecoder() Decoder {
+	return func(reader io.Reader, i *map[string]interface{}) error {
+		d := json.NewDecoder(reader)
+		d.UseNumber()
+		return d.Decode(i)
+	}
+}
+
+func JSONDecoder() Decoder {
+	return func(reader io.Reader, i *map[string]interface{}) error {
+		var collection []interface{}
+		d := json.NewDecoder(reader)
+		d.UseNumber()
+		if err := d.Decode(&collection); err != nil {
+			return err
+		}
+		*i = map[string]interface{}{"collection": collection}
+		return nil
+	}
+}

encoding/register.go

@@ -0,0 +1,46 @@
+package encoding
+
+import (
+	"io"
+	"melody/register"
+)
+
+var (
+	decoders        = initDecoderRegister()
+	defaultDecoders = map[string]func(bool) func(io.Reader, *map[string]interface{}) error{
+		JSON:   NewJSONDecoder,
+		STRING: NewStringDecoder,
+		NOOP:   NoOpDecoderFactory,
+	}
+)
+
+type DecoderRegister struct {
+	data register.Untyped
+}
+
+func (d *DecoderRegister) Get(s string) DecoderFactory {
+	// if can not get decoder:key = s,
+	// return json decoder
+	for _, v := range []string{s, JSON} {
+		if v, ok := d.data.Get(v); ok {
+			if decoderFactory, ok := v.(func(bool) func(io.Reader, *map[string]interface{}) error); ok {
+				return decoderFactory
+			}
+		}
+	}
+
+	return NewJSONDecoder
+}
+
+func (d *DecoderRegister) Register(name string, factory DecoderFactory) error {
+	d.data.Register(name, factory)
+	return nil
+}
+
+func initDecoderRegister() *DecoderRegister {
+	decoder := &DecoderRegister{data: register.New()}
+	for k, v := range defaultDecoders {
+		decoder.data.Register(k, v)
+	}
+	return decoder
+}

encoding/string.go

@@ -0,0 +1,23 @@
+package encoding
+
+import (
+	"io"
+	"io/ioutil"
+)
+
+const STRING = "string"
+
+func NewStringDecoder(_ bool) func(io.Reader, *map[string]interface{}) error {
+	return StringDecoder()
+}
+
+func StringDecoder() Decoder {
+	return func(reader io.Reader, i *map[string]interface{}) error {
+		data, err := ioutil.ReadAll(reader)
+		if err != nil {
+			return err
+		}
+		*i = map[string]interface{}{"content": string(data)}
+		return nil
+	}
+}

go.mod

@@ -3,6 +3,7 @@ module melody
 go 1.13
 
 require (
+	github.com/clbanning/mxj v1.8.4
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/spf13/afero v1.2.2 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
+github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

main_test.go

@@ -5,5 +5,5 @@ import (
 )
 
 func TestCommandRun(t *testing.T) {
-	
+
 }

melody.json

@@ -27,7 +27,7 @@
 	  "headers_to_pass": [
 		"*"
 	  ],
-	  "backend": [
+	  "backends": [
 		{
 		  "url_pattern": "/page",
 		  "encoding": "json",

middleware/melody-gelf/log.go

@@ -11,19 +11,17 @@ import (
 const Namespace = "melody_gelf"
 
 var (
-	tcpWriter = gelf.NewTCPWriter
-	udpWriter = gelf.NewUDPWriter
-	ErrorConfig = fmt.Errorf("not found extra config about melody-gelf module")
+	tcpWriter        = gelf.NewTCPWriter
+	udpWriter        = gelf.NewUDPWriter
+	ErrorConfig      = fmt.Errorf("not found extra config about melody-gelf module")
 	ErrorMissAddress = errors.New("miss gelf address to send log")
 )
 
 type Config struct {
-	Addr string
+	Addr      string
 	TCPEnable bool
 }
 
-
-
 func NewWriter(cfg config.ExtraConfig) (io.Writer, error) {
 	g, ok := GetConfig(cfg).(Config)
 	if !ok {
@@ -68,6 +66,3 @@ func GetConfig(cfg config.ExtraConfig) interface{} {
 		TCPEnable: enable.(bool),
 	}
 }
-
-
-