From 455d5ce196e3ea72e6451bd2546a2e1bf6f22a41 Mon Sep 17 00:00:00 2001
From: Ken Sipe <kensipe@gmail.com>
Date: Sun, 19 Apr 2020 14:06:24 -0500
Subject: [PATCH 1/7] initial step to support multi-repo include
Signed-off-by: Ken Sipe <kensipe@gmail.com>
---
pkg/kudoctl/util/repo/repo_operator.go | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/pkg/kudoctl/util/repo/repo_operator.go b/pkg/kudoctl/util/repo/repo_operator.go
index d0db1b000..3d13248ca 100644
--- a/pkg/kudoctl/util/repo/repo_operator.go
+++ b/pkg/kudoctl/util/repo/repo_operator.go
@@ -50,20 +50,27 @@ func NewClient(conf *Configuration) (*Client, error) {
// DownloadIndexFile fetches the index file from a repository.
func (c *Client) DownloadIndexFile() (*IndexFile, error) {
- var indexURL string
parsedURL, err := url.Parse(c.Config.URL)
if err != nil {
return nil, fmt.Errorf("parsing config url: %w", err)
}
+ // we need the index.yaml at the url provided
parsedURL.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(parsedURL.Path, "/"))
- indexURL = parsedURL.String()
+ return c.downloadIndexFile(parsedURL)
+}
+
+func (c *Client) downloadIndexFile(url *url.URL) (*IndexFile, error) {
var resp *bytes.Buffer
- if strings.HasPrefix(indexURL, "file:") {
- b, _ := ioutil.ReadFile(parsedURL.Path)
+ var err error
+ if strings.HasPrefix(url.String(), "file:") {
+ b, err := ioutil.ReadFile(url.Path)
+ if err != nil {
+ return nil, err
+ }
resp = bytes.NewBuffer(b)
} else {
- resp, err = c.Client.Get(indexURL)
+ resp, err = c.Client.Get(url.String())
}
if err != nil {
return nil, fmt.Errorf("getting index url: %w", err)
From ca5addffbbf3e6e53c5ee332a19901465374f8eb Mon Sep 17 00:00:00 2001
From: Ken Sipe <kensipe@gmail.com>
Date: Tue, 21 Apr 2020 21:32:38 -0500
Subject: [PATCH 2/7] working version of merge index and include of index files
with tests
Signed-off-by: Ken Sipe <kensipe@gmail.com>
---
pkg/kudoctl/cmd/repo_index.go | 16 +------
pkg/kudoctl/cmd/repo_index_test.go | 12 ++++-
pkg/kudoctl/util/repo/index.go | 1 +
pkg/kudoctl/util/repo/repo_operator.go | 44 ++++++++++++++++---
pkg/kudoctl/util/repo/repo_test.go | 26 +++++++++++
.../repo/testdata/include-index/index.yaml | 18 ++++++++
.../repo/testdata/included-repo/index.yaml | 39 ++++++++++++++++
7 files changed, 135 insertions(+), 21 deletions(-)
create mode 100644 pkg/kudoctl/util/repo/testdata/include-index/index.yaml
create mode 100644 pkg/kudoctl/util/repo/testdata/included-repo/index.yaml
diff --git a/pkg/kudoctl/cmd/repo_index.go b/pkg/kudoctl/cmd/repo_index.go
index 2d3d33ce7..b344bdf2f 100644
--- a/pkg/kudoctl/cmd/repo_index.go
+++ b/pkg/kudoctl/cmd/repo_index.go
@@ -140,7 +140,7 @@ func (ri *repoIndexCmd) run() error {
if err != nil {
return err
}
- merge(index, mergeIndex)
+ client.Merge(index, mergeIndex)
}
if err := index.WriteFile(ri.fs, target); err != nil {
@@ -150,20 +150,6 @@ func (ri *repoIndexCmd) run() error {
return nil
}
-func merge(index *repo.IndexFile, mergeIndex *repo.IndexFile) {
- // index is the master, any dups in the merged in index will have what is local replace those entries
- for _, pvs := range mergeIndex.Entries {
- for _, pv := range pvs {
- err := index.AddPackageVersion(pv)
- // this is most likely to be a duplicate pv, which we ignore (but will log at the right v)
- if err != nil {
- // todo: add verbose logging here
- continue
- }
- }
- }
-}
-
func (ri *repoIndexCmd) mergeRepoConfig() (*repo.Configuration, error) {
if ri.mergeRepoName != "" {
return ri.repoConfig(ri.mergeRepoName)
diff --git a/pkg/kudoctl/cmd/repo_index_test.go b/pkg/kudoctl/cmd/repo_index_test.go
index b6a06b46b..634c3bb48 100644
--- a/pkg/kudoctl/cmd/repo_index_test.go
+++ b/pkg/kudoctl/cmd/repo_index_test.go
@@ -2,6 +2,7 @@ package cmd
import (
"bytes"
+ "fmt"
"io/ioutil"
"path/filepath"
"testing"
@@ -92,7 +93,16 @@ func TestRepoIndexCmd_MergeIndex(t *testing.T) {
mergeFile, _ := repo.ParseIndexFile(mergeBytes)
resultBuf := &bytes.Buffer{}
- merge(indexFile, mergeFile)
+
+ p, err := filepath.Abs("testdata/include-index")
+ assert.NoError(t, err)
+ config := &repo.Configuration{
+ URL: fmt.Sprintf("file://%s", p),
+ }
+ client, err := repo.NewClient(config)
+ assert.NoError(t, err)
+
+ client.Merge(indexFile, mergeFile)
if err := indexFile.Write(resultBuf); err != nil {
t.Fatal(err)
}
diff --git a/pkg/kudoctl/util/repo/index.go b/pkg/kudoctl/util/repo/index.go
index 9b8ddbbc5..6fa5aec96 100644
--- a/pkg/kudoctl/util/repo/index.go
+++ b/pkg/kudoctl/util/repo/index.go
@@ -22,6 +22,7 @@ const defaultURL = "http://localhost/"
type IndexFile struct {
APIVersion string `json:"apiVersion"`
Entries map[string]PackageVersions `json:"entries"`
+ Includes []string `json:"includes,omitempty"`
Generated *time.Time `json:"generated"`
}
diff --git a/pkg/kudoctl/util/repo/repo_operator.go b/pkg/kudoctl/util/repo/repo_operator.go
index 3d13248ca..4221997d9 100644
--- a/pkg/kudoctl/util/repo/repo_operator.go
+++ b/pkg/kudoctl/util/repo/repo_operator.go
@@ -9,6 +9,7 @@ import (
"github.com/spf13/afero"
+ "github.com/kudobuilder/kudo/pkg/kudoctl/clog"
"github.com/kudobuilder/kudo/pkg/kudoctl/http"
"github.com/kudobuilder/kudo/pkg/kudoctl/kudohome"
)
@@ -54,16 +55,17 @@ func (c *Client) DownloadIndexFile() (*IndexFile, error) {
if err != nil {
return nil, fmt.Errorf("parsing config url: %w", err)
}
- // we need the index.yaml at the url provided
- parsedURL.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(parsedURL.Path, "/"))
- return c.downloadIndexFile(parsedURL)
+ return c.downloadIndexFile(nil, parsedURL)
}
-func (c *Client) downloadIndexFile(url *url.URL) (*IndexFile, error) {
+func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL) (*IndexFile, error) {
var resp *bytes.Buffer
var err error
- if strings.HasPrefix(url.String(), "file:") {
+ // we need the index.yaml at the url provided
+ url.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(url.Path, "/"))
+
+ if url.Scheme == "file" || strings.HasPrefix(url.String(), "file:") {
b, err := ioutil.ReadFile(url.Path)
if err != nil {
return nil, err
@@ -82,5 +84,37 @@ func (c *Client) downloadIndexFile(url *url.URL) (*IndexFile, error) {
}
indexFile, err := ParseIndexFile(indexBytes)
+ //TODO (kensipe): err handling such that error in includes are ignored (but not root)
+ //TODO (kensipe): track which includes have happened so there are no repeats
+ for _, include := range indexFile.Includes {
+ iURL, err := url.Parse(include)
+ if err != nil {
+ clog.Printf("Unable to parse include url for %s", include)
+ }
+ nextIndex, err := c.downloadIndexFile(indexFile, iURL)
+ if err != nil {
+ return nil, err
+ }
+ if parent != nil {
+ c.Merge(parent, nextIndex)
+ } else {
+ c.Merge(indexFile, nextIndex)
+ }
+ }
+
return indexFile, err
}
+
+func (c *Client) Merge(index *IndexFile, mergeIndex *IndexFile) {
+ // index is the master, any dups in the merged in index will have what is local replace those entries
+ for _, pvs := range mergeIndex.Entries {
+ for _, pv := range pvs {
+ err := index.AddPackageVersion(pv)
+ // this is most likely to be a duplicate pv, which we ignore (but will log at the right v)
+ if err != nil {
+ // todo: add verbose logging here
+ continue
+ }
+ }
+ }
+}
diff --git a/pkg/kudoctl/util/repo/repo_test.go b/pkg/kudoctl/util/repo/repo_test.go
index 7cb77b92b..cbf030b0e 100644
--- a/pkg/kudoctl/util/repo/repo_test.go
+++ b/pkg/kudoctl/util/repo/repo_test.go
@@ -42,3 +42,29 @@ func TestLoadRepositories(t *testing.T) {
assert.Equal(t, r.CurrentConfiguration().Name, Default.Name)
assert.Equal(t, r.CurrentConfiguration().URL, Default.URL)
}
+
+func TestDownloadMultiRepo(t *testing.T) {
+
+ p, err := filepath.Abs("testdata/include-index")
+ assert.NoError(t, err)
+ config := &Configuration{
+ URL: fmt.Sprintf("file://%s", p),
+ }
+ client, err := NewClient(config)
+ assert.NoError(t, err)
+ index, err := client.DownloadIndexFile()
+ assert.NoError(t, err)
+ // mysql package only there, if include worked
+ assert.NotNil(t, index.Entries["mysql"])
+
+ // the merge for flink will have 1 dup that doesn't merge
+ flink, err := index.FindFirstMatch("flink", "", "0.3.0")
+ assert.NoError(t, err)
+ assert.Equal(t, "correct flink", flink.Description)
+
+ // and a new version that does merge
+ flink, err = index.FindFirstMatch("flink", "", "0.4.0")
+ assert.NoError(t, err)
+ assert.Equal(t, "0.4.0", flink.OperatorVersion)
+
+}
diff --git a/pkg/kudoctl/util/repo/testdata/include-index/index.yaml b/pkg/kudoctl/util/repo/testdata/include-index/index.yaml
new file mode 100644
index 000000000..f28debea7
--- /dev/null
+++ b/pkg/kudoctl/util/repo/testdata/include-index/index.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+entries:
+ flink:
+ - appVersion: 0.7.0
+ description: correct flink
+ digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+ maintainers:
+ - email: <runyontr@gmail.com>
+ name: Tom Runyon
+ - email: <kensipe@gmail.com>
+ name: Ken Sipe
+ name: flink
+ operatorVersion: 0.3.0
+ urls:
+ - http://kudo.dev/flink
+includes:
+ - ../included-repo
+generated: "2020-04-19T15:04:00Z"
diff --git a/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml b/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml
new file mode 100644
index 000000000..f44166d45
--- /dev/null
+++ b/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml
@@ -0,0 +1,39 @@
+apiVersion: v1
+entries:
+ mysql:
+ - appVersion: "5.7"
+ digest: ad2451eaf68896490a2864afbb3fcd81e6fe3368e4bf001f8a23dc9cbcddf49a
+ maintainers:
+ - email: nick@dischord.org
+ name: Nick Jones
+ name: mysql
+ operatorVersion: 0.2.0
+ urls:
+ - https://kudo-repository.storage.googleapis.com/0.10.0/mysql-5.7_0.2.0.tgz
+ flink:
+ - appVersion: 0.7.0
+ description: wrong flink (should NOT be included)
+ digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+ maintainers:
+ - email: <runyontr@gmail.com>
+ name: Tom Runyon
+ - email: <kensipe@gmail.com>
+ name: Ken Sipe
+ name: flink
+ operatorVersion: 0.3.0
+ - appVersion: 0.8.0
+ description: this merges because the version doesn't exist in parent
+ digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+ maintainers:
+ - email: <runyontr@gmail.com>
+ name: Tom Runyon
+ - email: <kensipe@gmail.com>
+ name: Ken Sipe
+ name: flink
+ operatorVersion: 0.4.0
+ urls:
+ - http://kudo.dev/flink
+
+generated: "2020-04-19T15:04:00Z"
+
+
From 7e3484b31ef2dd39622dbab09c5ed1add7722f30 Mon Sep 17 00:00:00 2001
From: Ken Sipe <kensipe@gmail.com>
Date: Tue, 21 Apr 2020 21:43:47 -0500
Subject: [PATCH 3/7] adding godoc
Signed-off-by: Ken Sipe <kensipe@gmail.com>
---
pkg/kudoctl/util/repo/repo_operator.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pkg/kudoctl/util/repo/repo_operator.go b/pkg/kudoctl/util/repo/repo_operator.go
index 4221997d9..aec157c2d 100644
--- a/pkg/kudoctl/util/repo/repo_operator.go
+++ b/pkg/kudoctl/util/repo/repo_operator.go
@@ -105,6 +105,8 @@ func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL) (*IndexFile,
return indexFile, err
}
+// Merge combines the Entries of 2 index files. The first index file is the master
+// the second is merged into the first. Any duplicates are ignored.
func (c *Client) Merge(index *IndexFile, mergeIndex *IndexFile) {
// index is the master, any dups in the merged in index will have what is local replace those entries
for _, pvs := range mergeIndex.Entries {
From 536c8799419216a12c8ee9772ecbdb5a2eecc0fb Mon Sep 17 00:00:00 2001
From: Ken Sipe <kensipe@gmail.com>
Date: Thu, 23 Apr 2020 10:16:06 -0500
Subject: [PATCH 4/7] updated error handling
Signed-off-by: Ken Sipe <kensipe@gmail.com>
---
pkg/kudoctl/cmd/repo_index.go | 5 ++++-
pkg/kudoctl/cmd/repo_index_test.go | 3 ++-
pkg/kudoctl/util/repo/index.go | 20 ++++++++++++++++++--
pkg/kudoctl/util/repo/index_test.go | 4 ++--
pkg/kudoctl/util/repo/repo_operator.go | 21 +++++++++++++--------
5 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/pkg/kudoctl/cmd/repo_index.go b/pkg/kudoctl/cmd/repo_index.go
index b344bdf2f..4fc3c68fc 100644
--- a/pkg/kudoctl/cmd/repo_index.go
+++ b/pkg/kudoctl/cmd/repo_index.go
@@ -140,7 +140,10 @@ func (ri *repoIndexCmd) run() error {
if err != nil {
return err
}
- client.Merge(index, mergeIndex)
+ err = client.Merge(index, mergeIndex)
+ if err != nil {
+ return err
+ }
}
if err := index.WriteFile(ri.fs, target); err != nil {
diff --git a/pkg/kudoctl/cmd/repo_index_test.go b/pkg/kudoctl/cmd/repo_index_test.go
index 634c3bb48..bdc9c80fc 100644
--- a/pkg/kudoctl/cmd/repo_index_test.go
+++ b/pkg/kudoctl/cmd/repo_index_test.go
@@ -102,7 +102,8 @@ func TestRepoIndexCmd_MergeIndex(t *testing.T) {
client, err := repo.NewClient(config)
assert.NoError(t, err)
- client.Merge(indexFile, mergeFile)
+ err = client.Merge(indexFile, mergeFile)
+ assert.NoError(t, err)
if err := indexFile.Write(resultBuf); err != nil {
t.Fatal(err)
}
diff --git a/pkg/kudoctl/util/repo/index.go b/pkg/kudoctl/util/repo/index.go
index 6fa5aec96..6b200cac6 100644
--- a/pkg/kudoctl/util/repo/index.go
+++ b/pkg/kudoctl/util/repo/index.go
@@ -105,13 +105,29 @@ func (i IndexFile) Write(w io.Writer) error {
return err
}
+// DuplicateError is returned for a duplicate entry
+type DuplicateError struct {
+ pv *PackageVersion
+}
+
+// Implement the Error interface.
+func (e *DuplicateError) Error() string {
+ return fmt.Sprintf("operator %q version: %v_%v already exists", e.pv.Name, e.pv.AppVersion, e.pv.OperatorVersion)
+}
+
+// Defines what is is
+func (e *DuplicateError) Is(target error) bool {
+ _, ok := target.(*DuplicateError)
+ return ok
+}
+
// AddPackageVersion adds an entry to the IndexFile (does not allow dups)
func (i *IndexFile) AddPackageVersion(pv *PackageVersion) error {
name := pv.Name
appVersion := pv.AppVersion
operatorVersion := pv.OperatorVersion
if operatorVersion == "" {
- return fmt.Errorf("operator '%v' is missing operator version", name)
+ return fmt.Errorf("operator %q is missing operator version", name)
}
if i.Entries == nil {
i.Entries = make(map[string]PackageVersions)
@@ -127,7 +143,7 @@ func (i *IndexFile) AddPackageVersion(pv *PackageVersion) error {
// loop thru all... don't allow dups
for _, ver := range vs {
if ver.AppVersion == appVersion && ver.OperatorVersion == operatorVersion {
- return fmt.Errorf("operator '%v' version: %v_%v already exists", name, appVersion, operatorVersion)
+ return &DuplicateError{ver}
}
}
diff --git a/pkg/kudoctl/util/repo/index_test.go b/pkg/kudoctl/util/repo/index_test.go
index c5c86336f..8c7c9d713 100644
--- a/pkg/kudoctl/util/repo/index_test.go
+++ b/pkg/kudoctl/util/repo/index_test.go
@@ -174,8 +174,8 @@ func TestAddPackageVersionErrorConditions(t *testing.T) {
pv *PackageVersion
err string
}{
- {"duplicate version", dup, "operator 'flink' version: 0.7.0_0.3.0 already exists"},
- {"no version", &missing, "operator 'flink' is missing operator version"},
+ {"duplicate version", dup, "operator \"flink\" version: 0.7.0_0.3.0 already exists"},
+ {"no version", &missing, "operator \"flink\" is missing operator version"},
{"good additional version", &good, ""},
{"good additional package", &g2, ""},
}
diff --git a/pkg/kudoctl/util/repo/repo_operator.go b/pkg/kudoctl/util/repo/repo_operator.go
index aec157c2d..93d6f2a53 100644
--- a/pkg/kudoctl/util/repo/repo_operator.go
+++ b/pkg/kudoctl/util/repo/repo_operator.go
@@ -2,6 +2,7 @@ package repo
import (
"bytes"
+ "errors"
"fmt"
"io/ioutil"
"net/url"
@@ -84,21 +85,23 @@ func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL) (*IndexFile,
}
indexFile, err := ParseIndexFile(indexBytes)
- //TODO (kensipe): err handling such that error in includes are ignored (but not root)
//TODO (kensipe): track which includes have happened so there are no repeats
for _, include := range indexFile.Includes {
iURL, err := url.Parse(include)
if err != nil {
- clog.Printf("Unable to parse include url for %s", include)
+ return nil, clog.Errorf("unable to parse include url for %s", include)
}
nextIndex, err := c.downloadIndexFile(indexFile, iURL)
if err != nil {
return nil, err
}
if parent != nil {
- c.Merge(parent, nextIndex)
+ err = c.Merge(parent, nextIndex)
} else {
- c.Merge(indexFile, nextIndex)
+ err = c.Merge(indexFile, nextIndex)
+ }
+ if err != nil {
+ return nil, err
}
}
@@ -107,16 +110,18 @@ func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL) (*IndexFile,
// Merge combines the Entries of 2 index files. The first index file is the master
// the second is merged into the first. Any duplicates are ignored.
-func (c *Client) Merge(index *IndexFile, mergeIndex *IndexFile) {
+func (c *Client) Merge(index *IndexFile, mergeIndex *IndexFile) error {
// index is the master, any dups in the merged in index will have what is local replace those entries
for _, pvs := range mergeIndex.Entries {
for _, pv := range pvs {
err := index.AddPackageVersion(pv)
- // this is most likely to be a duplicate pv, which we ignore (but will log at the right v)
- if err != nil {
- // todo: add verbose logging here
+ if errors.Is(err, &DuplicateError{}) {
+ clog.V(1).Printf("ignoring duplicate for %q: appver: %v, opver: %v", pv.Name, pv.AppVersion, pv.OperatorVersion)
continue
+ } else if err != nil {
+ return err
}
}
}
+ return nil
}
From 8637e6ea28c6d973f40a7849a59f9fc1d6d1d116 Mon Sep 17 00:00:00 2001
From: Ken Sipe <kensipe@gmail.com>
Date: Thu, 23 Apr 2020 10:26:21 -0500
Subject: [PATCH 5/7] url tracking added
Signed-off-by: Ken Sipe <kensipe@gmail.com>
---
pkg/kudoctl/util/repo/repo_operator.go | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/pkg/kudoctl/util/repo/repo_operator.go b/pkg/kudoctl/util/repo/repo_operator.go
index 93d6f2a53..b83a27d90 100644
--- a/pkg/kudoctl/util/repo/repo_operator.go
+++ b/pkg/kudoctl/util/repo/repo_operator.go
@@ -56,15 +56,21 @@ func (c *Client) DownloadIndexFile() (*IndexFile, error) {
if err != nil {
return nil, fmt.Errorf("parsing config url: %w", err)
}
-
- return c.downloadIndexFile(nil, parsedURL)
+ h := make(map[string]bool)
+ return c.downloadIndexFile(nil, parsedURL, h)
}
-func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL) (*IndexFile, error) {
+func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL, urlHistory map[string]bool) (*IndexFile, error) {
var resp *bytes.Buffer
var err error
// we need the index.yaml at the url provided
url.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(url.Path, "/"))
+ if val, ok := urlHistory[url.String()]; ok {
+ // if we have seen the url previous we don't process it
+ clog.V(1).Printf("duplicate url %v ignored", val)
+ return parent, nil
+ }
+ urlHistory[url.String()] = true
if url.Scheme == "file" || strings.HasPrefix(url.String(), "file:") {
b, err := ioutil.ReadFile(url.Path)
@@ -85,13 +91,12 @@ func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL) (*IndexFile,
}
indexFile, err := ParseIndexFile(indexBytes)
- //TODO (kensipe): track which includes have happened so there are no repeats
for _, include := range indexFile.Includes {
iURL, err := url.Parse(include)
if err != nil {
return nil, clog.Errorf("unable to parse include url for %s", include)
}
- nextIndex, err := c.downloadIndexFile(indexFile, iURL)
+ nextIndex, err := c.downloadIndexFile(indexFile, iURL, urlHistory)
if err != nil {
return nil, err
}
From 2335f222c314b3d6329c64b0ad46377361441b4d Mon Sep 17 00:00:00 2001
From: Andreas Neumann <aneumann@mesosphere.com>
Date: Thu, 23 Apr 2020 18:22:15 +0200
Subject: [PATCH 6/7] Added test case for nested included repos
Signed-off-by: Andreas Neumann <aneumann@mesosphere.com>
---
pkg/kudoctl/util/repo/repo_test.go | 6 +++
.../repo/testdata/included-repo/index.yaml | 15 ++++++-
.../testdata/nested-included-repo/index.yaml | 39 +++++++++++++++++++
3 files changed, 59 insertions(+), 1 deletion(-)
create mode 100644 pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml
diff --git a/pkg/kudoctl/util/repo/repo_test.go b/pkg/kudoctl/util/repo/repo_test.go
index cbf030b0e..c874ada75 100644
--- a/pkg/kudoctl/util/repo/repo_test.go
+++ b/pkg/kudoctl/util/repo/repo_test.go
@@ -67,4 +67,10 @@ func TestDownloadMultiRepo(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, "0.4.0", flink.OperatorVersion)
+ // and a version that is nested in a repository
+ flink, err = index.FindFirstMatch("flink", "", "0.4.1")
+ assert.NoError(t, err)
+ assert.Equal(t, "0.4.1", flink.OperatorVersion)
+ assert.Equal(t, "this merges and overwrites the version in nested-included-repo", flink.Description)
+
}
diff --git a/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml b/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml
index f44166d45..f53a8a71f 100644
--- a/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml
+++ b/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml
@@ -33,7 +33,20 @@ entries:
operatorVersion: 0.4.0
urls:
- http://kudo.dev/flink
-
+ - appVersion: 0.8.1
+ description: this merges and overwrites the version in nested-included-repo
+ digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+ maintainers:
+ - email: <runyontr@gmail.com>
+ name: Tom Runyon
+ - email: <kensipe@gmail.com>
+ name: Ken Sipe
+ name: flink
+ operatorVersion: 0.4.1
+ urls:
+ - http://kudo.dev/flink
+includes:
+ - ../nested-included-repo
generated: "2020-04-19T15:04:00Z"
diff --git a/pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml b/pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml
new file mode 100644
index 000000000..c9f90e51b
--- /dev/null
+++ b/pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml
@@ -0,0 +1,39 @@
+apiVersion: v1
+entries:
+ mysql:
+ - appVersion: "5.7"
+ digest: ad2451eaf68896490a2864afbb3fcd81e6fe3368e4bf001f8a23dc9cbcddf49a
+ maintainers:
+ - email: nick@dischord.org
+ name: Nick Jones
+ name: mysql
+ operatorVersion: 0.2.0
+ urls:
+ - https://kudo-repository.storage.googleapis.com/0.10.0/mysql-5.7_0.2.0.tgz
+ flink:
+ - appVersion: 0.7.0
+ description: wrong flink (should NOT be included)
+ digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+ maintainers:
+ - email: <runyontr@gmail.com>
+ name: Tom Runyon
+ - email: <kensipe@gmail.com>
+ name: Ken Sipe
+ name: flink
+ operatorVersion: 0.3.0
+ - appVersion: 0.8.1
+ description: this will get overwritten by included-repo
+ digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+ maintainers:
+ - email: <runyontr@gmail.com>
+ name: Tom Runyon
+ - email: <kensipe@gmail.com>
+ name: Ken Sipe
+ name: flink
+ operatorVersion: 0.4.1
+ urls:
+ - http://kudo.dev/flink
+
+generated: "2020-04-19T15:04:00Z"
+
+
From 7a0537d7dac91da70032c4a154bf8975a2690961 Mon Sep 17 00:00:00 2001
From: Andreas Neumann <aneumann@mesosphere.com>
Date: Thu, 23 Apr 2020 18:30:32 +0200
Subject: [PATCH 7/7] Fixed nested repo include
Signed-off-by: Andreas Neumann <aneumann@mesosphere.com>
---
pkg/kudoctl/util/repo/repo_operator.go | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/pkg/kudoctl/util/repo/repo_operator.go b/pkg/kudoctl/util/repo/repo_operator.go
index b83a27d90..6021f7cd6 100644
--- a/pkg/kudoctl/util/repo/repo_operator.go
+++ b/pkg/kudoctl/util/repo/repo_operator.go
@@ -57,10 +57,10 @@ func (c *Client) DownloadIndexFile() (*IndexFile, error) {
return nil, fmt.Errorf("parsing config url: %w", err)
}
h := make(map[string]bool)
- return c.downloadIndexFile(nil, parsedURL, h)
+ return c.downloadIndexFile(parsedURL, h)
}
-func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL, urlHistory map[string]bool) (*IndexFile, error) {
+func (c *Client) downloadIndexFile(url *url.URL, urlHistory map[string]bool) (*IndexFile, error) {
var resp *bytes.Buffer
var err error
// we need the index.yaml at the url provided
@@ -68,7 +68,7 @@ func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL, urlHistory m
if val, ok := urlHistory[url.String()]; ok {
// if we have seen the url previous we don't process it
clog.V(1).Printf("duplicate url %v ignored", val)
- return parent, nil
+ return nil, nil
}
urlHistory[url.String()] = true
@@ -96,15 +96,11 @@ func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL, urlHistory m
if err != nil {
return nil, clog.Errorf("unable to parse include url for %s", include)
}
- nextIndex, err := c.downloadIndexFile(indexFile, iURL, urlHistory)
+ nextIndex, err := c.downloadIndexFile(iURL, urlHistory)
if err != nil {
return nil, err
}
- if parent != nil {
- err = c.Merge(parent, nextIndex)
- } else {
- err = c.Merge(indexFile, nextIndex)
- }
+ err = c.Merge(indexFile, nextIndex)
if err != nil {
return nil, err
}