Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 8a67a27

Browse files
authoredMar 4, 2022
Merge branch 'master' into develop
2 parents a231ce7 + ff2b5a5 commit 8a67a27

File tree

13 files changed

+73
-121
lines changed

13 files changed

+73
-121
lines changed
 

‎pkg/apiserver/api.go

Lines changed: 6 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
package apiserver
22

33
import (
4-
"bytes"
5-
"compress/gzip"
64
"context"
75
"errors"
8-
"io/ioutil"
6+
"fmt"
97
"net/http"
108
"strings"
119
"time"
@@ -52,8 +50,7 @@ func NewAPIServer(opt Options) *APIServer {
5250
router.Use(HandleCors).GET("/api/sample_types", apiServer.listSampleTypes)
5351
router.Use(HandleCors).GET("/api/group_sample_types", apiServer.listGroupSampleTypes)
5452
router.Use(HandleCors).GET("/api/profile_meta/:sample_type", apiServer.listProfileMeta)
55-
router.Use(HandleCors).GET("/api/profile/:id", apiServer.getProfile)
56-
router.Use(HandleCors).GET("/api/trace/:id", apiServer.getTrace)
53+
router.Use(HandleCors).GET("/api/download/:id", apiServer.downloadProfile)
5754

5855
// register pprof page
5956
router.Use(HandleCors).GET(pprofPath+"/*any", apiServer.webPProf)
@@ -187,9 +184,9 @@ func (s *APIServer) listProfileMeta(c *gin.Context) {
187184
c.JSON(http.StatusOK, metas)
188185
}
189186

190-
func (s *APIServer) getProfile(c *gin.Context) {
187+
func (s *APIServer) downloadProfile(c *gin.Context) {
191188
id := c.Param("id")
192-
data, err := s.store.GetProfile(id)
189+
name, data, err := s.store.GetProfile(id)
193190
if err != nil {
194191
if errors.Is(err, storage.ErrProfileNotFound) {
195192
c.String(http.StatusNotFound, "Profile not found")
@@ -198,39 +195,8 @@ func (s *APIServer) getProfile(c *gin.Context) {
198195
c.String(http.StatusInternalServerError, err.Error())
199196
return
200197
}
201-
202-
c.Writer.Header().Set("Content-Encoding", "gzip")
203-
c.Writer.Header().Set("Content-Disposition", "attachment;filename=profile_"+id+".pb.gz")
204-
c.Data(200, "application/vnd.google.protobuf+gzip", data)
205-
}
206-
207-
func (s *APIServer) getTrace(c *gin.Context) {
208-
id := c.Param("id")
209-
data, err := s.store.GetProfile(id)
210-
if err != nil {
211-
if errors.Is(err, storage.ErrProfileNotFound) {
212-
c.String(http.StatusNotFound, "Profile not found")
213-
return
214-
}
215-
c.String(http.StatusInternalServerError, err.Error())
216-
return
217-
}
218-
219-
buf := bytes.NewBuffer(data)
220-
gzipReader, err := gzip.NewReader(buf)
221-
if err != nil {
222-
c.String(http.StatusInternalServerError, err.Error())
223-
return
224-
}
225-
defer gzipReader.Close()
226-
b, err := ioutil.ReadAll(gzipReader)
227-
if err != nil && !strings.Contains(err.Error(), "unexpected EOF") {
228-
c.String(http.StatusInternalServerError, err.Error())
229-
return
230-
}
231-
232-
c.Writer.Header().Set("Content-Disposition", "attachment;filename=trace_"+id+".out")
233-
c.Data(200, "application/octet-stream", b)
198+
c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment;filename=%s-%s.out", name, id))
199+
c.Data(200, "application/octet-stream", data)
234200
}
235201

236202
func (s *APIServer) webPProf(c *gin.Context) {

‎pkg/apiserver/api_test.go

Lines changed: 11 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
package apiserver
22

33
import (
4-
"bytes"
5-
"compress/gzip"
64
"fmt"
75
"io/ioutil"
86
"net/http"
@@ -13,7 +11,6 @@ import (
1311
"github.com/gavv/httpexpect/v2"
1412
"github.com/gin-gonic/gin"
1513
"github.com/stretchr/testify/require"
16-
"github.com/xyctruth/profiler/pkg/internal/v1175/trace"
1714
"github.com/xyctruth/profiler/pkg/storage"
1815
"github.com/xyctruth/profiler/pkg/storage/badger"
1916
)
@@ -84,20 +81,20 @@ func initMateData(s storage.Store, t *testing.T) {
8481
}
8582

8683
func initProfileData(s storage.Store, t *testing.T) (string, string, string, string) {
87-
invalidId, err := s.SaveProfile([]byte{}, time.Second*10)
84+
invalidId, err := s.SaveProfile("", []byte{}, time.Second*10)
8885
require.Equal(t, nil, err)
8986

90-
invalidId2, err := s.SaveProfile([]byte("haha"), time.Second*10)
87+
invalidId2, err := s.SaveProfile("", []byte("haha"), time.Second*10)
9188
require.Equal(t, nil, err)
9289

93-
profileBytes, err := ioutil.ReadFile("./testdata/profile.gz")
90+
profileBytes, err := ioutil.ReadFile("./testdata/profile.out.testdata")
9491
require.Equal(t, nil, err)
95-
id, err := s.SaveProfile(profileBytes, time.Second*10)
92+
id, err := s.SaveProfile("", profileBytes, time.Second*10)
9693
require.Equal(t, nil, err)
9794

98-
traceBytes, err := ioutil.ReadFile("./testdata/trace.gz")
95+
traceBytes, err := ioutil.ReadFile("./testdata/trace.out.testdata")
9996
require.Equal(t, nil, err)
100-
traceID, err := s.SaveProfile(traceBytes, time.Second*3)
97+
traceID, err := s.SaveProfile("", traceBytes, time.Second*3)
10198
require.Equal(t, nil, err)
10299
return invalidId, invalidId2, id, traceID
103100
}
@@ -208,7 +205,7 @@ func TestListProfileMeta(t *testing.T) {
208205
Status(http.StatusOK).JSON().Array().Length().Equal(0)
209206
}
210207

211-
func TestGetProfile(t *testing.T) {
208+
func TestDownloadProfile(t *testing.T) {
212209
dir, err := ioutil.TempDir("./", "temp-*")
213210
require.Equal(t, nil, err)
214211
defer os.RemoveAll(dir)
@@ -218,19 +215,19 @@ func TestGetProfile(t *testing.T) {
218215
apiServer := NewAPIServer(DefaultOptions(s))
219216
e := getExpect(apiServer, t)
220217

221-
e.GET("/api/profile/999").
218+
e.GET("/api/download/999").
222219
Expect().
223220
Status(http.StatusNotFound)
224221

225-
e.GET(fmt.Sprintf("/api/profile/%s", id)).
222+
e.GET(fmt.Sprintf("/api/download/%s", id)).
226223
Expect().
227-
Status(http.StatusOK).Header("Content-Type").Equal("application/vnd.google.protobuf+gzip")
224+
Status(http.StatusOK).Header("Content-Type").Equal("application/octet-stream")
228225

229226
e.GET("/api/trace/999").
230227
Expect().
231228
Status(http.StatusNotFound)
232229

233-
e.GET(fmt.Sprintf("/api/trace/%s", traceID)).
230+
e.GET(fmt.Sprintf("/api/download/%s", traceID)).
234231
Expect().
235232
Status(http.StatusOK).Header("Content-Type").Equal("application/octet-stream")
236233
}
@@ -272,15 +269,3 @@ func getExpect(apiServer *APIServer, t *testing.T) *httpexpect.Expect {
272269
Reporter: httpexpect.NewAssertReporter(t),
273270
})
274271
}
275-
276-
func TestGZIP(t *testing.T) {
277-
f1, err := os.Open("./testdata/trace.gz")
278-
require.Equal(t, nil, err)
279-
gzipReader, _ := gzip.NewReader(f1)
280-
defer gzipReader.Close()
281-
b, err := ioutil.ReadAll(gzipReader)
282-
buf := bytes.NewReader(b)
283-
_, err = trace.Parse(buf, "")
284-
require.Equal(t, nil, err)
285-
286-
}

‎pkg/apiserver/testdata/profile.gz

-109 KB
Binary file not shown.
2.74 KB
Binary file not shown.

‎pkg/apiserver/testdata/trace.gz

-167 KB
Binary file not shown.
39.8 KB
Binary file not shown.

‎pkg/apiserver/ui/trace/trace.go

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,14 @@
11
package trace
22

33
import (
4-
"bytes"
5-
"compress/gzip"
6-
"io/ioutil"
74
"net/http"
85
"path"
9-
"strings"
106

117
"github.com/xyctruth/profiler/pkg/internal/v1175/traceui"
128
)
139

1410
func Driver(basePath string, mux *http.ServeMux, id string, data []byte) error {
15-
buf := bytes.NewBuffer(data)
16-
gzipReader, err := gzip.NewReader(buf)
17-
if err != nil {
18-
return err
19-
}
20-
defer gzipReader.Close()
21-
b, err := ioutil.ReadAll(gzipReader)
22-
if err != nil && !strings.Contains(err.Error(), "unexpected EOF") {
23-
return err
24-
}
25-
26-
ui, err := traceui.NewUI(b)
11+
ui, err := traceui.NewUI(data)
2712
if err != nil {
2813
return err
2914
}

‎pkg/apiserver/ui/ui.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) {
7979
return
8080
}
8181

82-
data, err := s.store.GetProfile(id)
82+
_, data, err := s.store.GetProfile(id)
8383
if err != nil {
8484
if errors.Is(err, storage.ErrProfileNotFound) {
8585
http.Error(w, "Profile not found", http.StatusNotFound)

‎pkg/apiserver/ui/ui_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,34 +21,34 @@ import (
2121
)
2222

2323
func initProfileData(s storage.Store, t *testing.T) (string, string, string) {
24-
invalidId, err := s.SaveProfile([]byte{}, time.Second*10)
24+
invalidId, err := s.SaveProfile("", []byte{}, time.Second*10)
2525
require.Equal(t, nil, err)
2626
require.Equal(t, "0", invalidId)
2727

28-
invalidId2, err := s.SaveProfile([]byte("haha"), time.Second*10)
28+
invalidId2, err := s.SaveProfile("", []byte("haha"), time.Second*10)
2929
require.Equal(t, nil, err)
3030
require.Equal(t, "1", invalidId2)
3131

32-
profileBytes, err := ioutil.ReadFile("../testdata/profile.gz")
32+
profileBytes, err := ioutil.ReadFile("../testdata/profile.out.testdata")
3333
require.Equal(t, nil, err)
34-
id, err := s.SaveProfile(profileBytes, time.Second*10)
34+
id, err := s.SaveProfile("", profileBytes, time.Second*10)
3535
require.Equal(t, nil, err)
3636
require.Equal(t, "2", id)
3737
return invalidId, invalidId2, id
3838
}
3939

4040
func initTraceData(s storage.Store, t *testing.T) (string, string, string) {
41-
invalidId, err := s.SaveProfile([]byte{}, time.Second*10)
41+
invalidId, err := s.SaveProfile("", []byte{}, time.Second*10)
4242
require.Equal(t, nil, err)
4343
require.Equal(t, "0", invalidId)
4444

45-
invalidId2, err := s.SaveProfile([]byte("haha"), time.Second*10)
45+
invalidId2, err := s.SaveProfile("", []byte("haha"), time.Second*10)
4646
require.Equal(t, nil, err)
4747
require.Equal(t, "1", invalidId2)
4848

49-
traceBytes, err := ioutil.ReadFile("../testdata/trace.gz")
49+
traceBytes, err := ioutil.ReadFile("../testdata/trace.out.testdata")
5050
require.Equal(t, nil, err)
51-
id, err := s.SaveProfile(traceBytes, time.Second*10)
51+
id, err := s.SaveProfile("", traceBytes, time.Second*10)
5252
require.Equal(t, nil, err)
5353
require.Equal(t, "2", id)
5454
return invalidId, invalidId2, id
@@ -156,11 +156,11 @@ func testTraceUI(e *httpexpect.Expect, store storage.Store, t *testing.T, server
156156

157157
e.GET(fmt.Sprintf("/api/trace/ui/%s", invalidId)).
158158
Expect().
159-
Status(http.StatusInternalServerError).Text().Equal("EOF\n")
159+
Status(http.StatusInternalServerError).Text().Equal("failed to parse trace: failed to read header: read 0, err EOF\n")
160160

161161
e.GET(fmt.Sprintf("/api/trace/ui/%s", invalidId2)).
162162
Expect().
163-
Status(http.StatusInternalServerError).Text().Equal("unexpected EOF\n")
163+
Status(http.StatusInternalServerError).Text().Equal("failed to parse trace: failed to read header: read 4, err unexpected EOF\n")
164164

165165
e.GET(fmt.Sprintf("/api/trace/ui/%s", id)).
166166
Expect().

‎pkg/collector/collector.go

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package collector
22

33
import (
44
"bytes"
5-
"compress/gzip"
65
"errors"
76
"fmt"
87
"io/ioutil"
@@ -168,6 +167,7 @@ func (collector *Collector) analysis(profileType string, profileBytes []byte) er
168167
return errors.New("sample type is nil")
169168
}
170169

170+
// Set profile name , Display it on the Profile UI
171171
if len(p.Mapping) > 0 {
172172
p.Mapping[0].File = collector.TargetName
173173
}
@@ -177,7 +177,7 @@ func (collector *Collector) analysis(profileType string, profileBytes []byte) er
177177
return err
178178
}
179179

180-
profileID, err := collector.store.SaveProfile(b.Bytes(), collector.Expiration)
180+
profileID, err := collector.store.SaveProfile(fmt.Sprintf("%s-%s", collector.TargetName, profileType), b.Bytes(), collector.Expiration)
181181
if err != nil {
182182
return err
183183
}
@@ -213,22 +213,11 @@ func (collector *Collector) analysis(profileType string, profileBytes []byte) er
213213
}
214214

215215
func (collector *Collector) analysisTrace(profileType string, profileBytes []byte) error {
216-
var compressData bytes.Buffer
217-
gzipWriter := gzip.NewWriter(&compressData)
218-
defer gzipWriter.Close()
219-
_, err := gzipWriter.Write(profileBytes)
220-
if err != nil {
221-
return err
222-
}
223-
err = gzipWriter.Flush()
216+
profileID, err := collector.store.SaveProfile(fmt.Sprintf("%s-%s", collector.TargetName, profileType), profileBytes, collector.Expiration)
224217
if err != nil {
225218
return err
226219
}
227220

228-
profileID, err := collector.store.SaveProfile(compressData.Bytes(), collector.Expiration)
229-
if err != nil {
230-
return err
231-
}
232221
metas := make([]*storage.ProfileMeta, 0, 1)
233222
meta := &storage.ProfileMeta{}
234223
meta.Timestamp = time.Now().UnixNano() / time.Millisecond.Nanoseconds()

‎pkg/storage/badger/store.go

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package badger
22

33
import (
4+
"bytes"
5+
"compress/gzip"
46
"errors"
7+
"io/ioutil"
58
"strconv"
69
"strings"
710
"time"
@@ -67,34 +70,58 @@ func (s *store) gc() {
6770
s.gc()
6871
}
6972

70-
func (s *store) GetProfile(id string) ([]byte, error) {
71-
var date []byte
73+
func (s *store) GetProfile(id string) (string, []byte, error) {
74+
var data []byte
7275
err := s.db.View(func(txn *badger.Txn) error {
7376
item, err := txn.Get(buildProfileKey(id))
7477
if err != nil {
7578
return err
7679
}
7780

7881
return item.Value(func(val []byte) error {
79-
date = val
82+
data = val
8083
return nil
8184
})
8285
})
8386

8487
if errors.Is(err, badger.ErrKeyNotFound) {
85-
return nil, storage.ErrProfileNotFound
88+
return "", nil, storage.ErrProfileNotFound
8689
}
87-
return date, err
90+
91+
buf := bytes.NewBuffer(data)
92+
gzipReader, err := gzip.NewReader(buf)
93+
if err != nil {
94+
return "", nil, err
95+
}
96+
defer gzipReader.Close()
97+
b, err := ioutil.ReadAll(gzipReader)
98+
if err != nil && !strings.Contains(err.Error(), "unexpected EOF") {
99+
return "", nil, err
100+
}
101+
return gzipReader.Header.Name, b, nil
88102
}
89103

90-
func (s *store) SaveProfile(profileData []byte, ttl time.Duration) (string, error) {
104+
func (s *store) SaveProfile(name string, profileData []byte, ttl time.Duration) (string, error) {
105+
var compressData bytes.Buffer
106+
gzipWriter, _ := gzip.NewWriterLevel(&compressData, gzip.BestCompression)
107+
gzipWriter.Header.Name = name
108+
defer gzipWriter.Close()
109+
_, err := gzipWriter.Write(profileData)
110+
if err != nil {
111+
return "", err
112+
}
113+
err = gzipWriter.Flush()
114+
if err != nil {
115+
return "", err
116+
}
117+
91118
id, err := s.profileSeq.Next()
92119
if err != nil {
93120
return "", err
94121
}
95122
idStr := strconv.FormatUint(id, 10)
96123
err = s.db.Update(func(txn *badger.Txn) error {
97-
return txn.SetEntry(newProfileEntry(idStr, profileData, ttl))
124+
return txn.SetEntry(newProfileEntry(idStr, compressData.Bytes(), ttl))
98125
})
99126

100127
return idStr, err

‎pkg/storage/badger/store_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -136,16 +136,16 @@ func TestProfile(t *testing.T) {
136136
defer s.Release()
137137
require.NotEqual(t, nil, s)
138138

139-
id, err := s.SaveProfile([]byte{}, 1*time.Second)
139+
id, err := s.SaveProfile("", []byte{}, 1*time.Second)
140140
require.Equal(t, nil, err)
141141
require.NotEqual(t, 0, id)
142142

143-
_, err = s.GetProfile(id)
143+
_, _, err = s.GetProfile(id)
144144
require.Equal(t, nil, err)
145145

146146
// Waiting for the overdue
147147
time.Sleep(1 * time.Second)
148-
_, err = s.GetProfile(id)
148+
_, _, err = s.GetProfile(id)
149149
require.NotEqual(t, nil, err)
150150
}
151151

@@ -384,7 +384,7 @@ func BenchmarkBadger1(b *testing.B) {
384384
}
385385

386386
for i := 0; i < b.N; i++ {
387-
_, err = s.SaveProfile(res, time.Hour*24*7)
387+
_, err = s.SaveProfile("", res, time.Hour*24*7)
388388
if err != nil {
389389
panic(err)
390390
}
@@ -428,7 +428,7 @@ func BenchmarkBadger2(b *testing.B) {
428428
}
429429

430430
for i := 0; i < b.N; i++ {
431-
_, err = s.SaveProfile(res, time.Hour*24*7)
431+
_, err = s.SaveProfile("", res, time.Hour*24*7)
432432
if err != nil {
433433
panic(err)
434434
}

‎pkg/storage/store.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ import (
99

1010
type Store interface {
1111
// GetProfile Get profile binaries by profile id, return profile binaries
12-
GetProfile(id string) ([]byte, error)
12+
GetProfile(id string) (string, []byte, error)
1313

1414
// SaveProfile Save profile,return profile id
1515
// binary data
1616
// ttl profile expiration time
17-
SaveProfile(data []byte, ttl time.Duration) (string, error)
17+
SaveProfile(name string, data []byte, ttl time.Duration) (string, error)
1818

1919
// SaveProfileMeta Save profile meta data
2020
SaveProfileMeta(metas []*ProfileMeta, ttl time.Duration) error

0 commit comments

Comments
 (0)
Please sign in to comment.