Skip to content

Commit e65cc58

Browse files
jemiahwjemiahw
authored
Add _created and _updated metalables (#89)
The code change itself is relatively small and straightforward: * Add the new metalable in etre.go * Add the validations in validate.go * Maintain the label in store.go * Update the CLI to pretty-print in a human readable form (instead of the raw Unix nano time) The test changes were more involved, particularly because the timestamps are not deterministic. Most of the tests that validate actual data now follow a pattern where they put the actual values into the expected entity for the assert.Equal(t, expected, actual), and then do a separate check that the actual time is a realistic value (in the last few seconds). Also fixed a bug with Entity.Rev(). It expected an int type, but json.Unmarshal sets it to a float64 which caused a panic. Now that is fixed to allow the various int flavors _and_ float.
1 parent 092a3ed commit e65cc58

File tree

11 files changed

+207
-94
lines changed

11 files changed

+207
-94
lines changed

‎api/api_test.go

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ type server struct {
5252
}
5353

5454
var testEntities = []etre.Entity{
55-
{"_id": "59f10d2a5669fc79103a0000", "_type": "node", "_rev": int64(0), "x": "1", "foo": "bar"},
56-
{"_id": "59f10d2a5669fc79103a1111", "_type": "node", "_rev": int64(0), "x": "2", "foo": "bar"},
57-
{"_id": "59f10d2a5669fc79103a2222", "_type": "node", "_rev": int64(0), "x": "3", "foo": "bar"},
55+
{"_id": "59f10d2a5669fc79103a0000", "_type": "node", "_rev": int64(0), "_created": int64(1000), "_updated": int64(2000), "x": "1", "foo": "bar"},
56+
{"_id": "59f10d2a5669fc79103a1111", "_type": "node", "_rev": int64(0), "_created": int64(3000), "_updated": int64(4000), "x": "2", "foo": "bar"},
57+
{"_id": "59f10d2a5669fc79103a2222", "_type": "node", "_rev": int64(0), "_created": int64(5000), "_updated": int64(6000), "x": "3", "foo": "bar"},
5858
}
5959

6060
var testEntityIds = []string{"59f10d2a5669fc79103a0000", "59f10d2a5669fc79103a1111", "59f10d2a5669fc79103a2222"}
@@ -66,9 +66,9 @@ var (
6666
)
6767

6868
var testEntitiesWithObjectIDs = []etre.Entity{
69-
{"_id": testEntityId0, "_type": "node", "_rev": int64(0), "x": "1", "foo": "bar"},
70-
{"_id": testEntityId1, "_type": "node", "_rev": int64(0), "x": "2", "foo": "bar"},
71-
{"_id": testEntityId2, "_type": "node", "_rev": int64(0), "x": "3", "foo": "bar"},
69+
{"_id": testEntityId0, "_type": "node", "_rev": int64(0), "_created": int64(1000), "_updated": int64(2000), "x": "1", "foo": "bar"},
70+
{"_id": testEntityId1, "_type": "node", "_rev": int64(0), "_created": int64(3000), "_updated": int64(4000), "x": "2", "foo": "bar"},
71+
{"_id": testEntityId2, "_type": "node", "_rev": int64(0), "_created": int64(5000), "_updated": int64(6000), "x": "3", "foo": "bar"},
7272
}
7373

7474
var defaultConfig = config.Config{
@@ -125,11 +125,13 @@ func uri(id string) string {
125125
return addr + etre.API_ROOT + "/entity/" + id
126126
}
127127

128-
func fixRev(e []etre.Entity) {
129-
for i := range e {
130-
f := e[i]["_rev"].(float64)
131-
delete(e[i], "_rev")
132-
e[i]["_rev"] = int64(f)
128+
func fixInt64(e []etre.Entity) {
129+
for _, label := range []string{"_rev", "_updated", "_created"} {
130+
for i := range e {
131+
f := e[i][label].(float64)
132+
delete(e[i], label)
133+
e[i][label] = int64(f)
134+
}
133135
}
134136
}
135137

‎api/query_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func TestQueryBasic(t *testing.T) {
5252
expectFilter := etre.QueryFilter{}
5353
assert.Equal(t, expectFilter, gotFilter)
5454

55-
fixRev(gotEntities) // JSON float64(_rev) ->, int64(_rev)
55+
fixInt64(gotEntities) // JSON float64(_rev) ->, int64(_rev)
5656
assert.Equal(t, testEntities, gotEntities)
5757

5858
// -- Metrics -----------------------------------------------------------

‎api/single_entity_read_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestGetEntityBasic(t *testing.T) {
5555
expectFilter := etre.QueryFilter{}
5656
assert.Equal(t, expectFilter, gotFilter)
5757

58-
fixRev([]etre.Entity{gotEntity}) // JSON float64(_rev) ->, int64(_rev)
58+
fixInt64([]etre.Entity{gotEntity}) // JSON float64(_rev) ->, int64(_rev)
5959
assert.Equal(t, testEntities[0], gotEntity)
6060

6161
// -- Metrics -----------------------------------------------------------

‎client_test.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ func TestQueryOK(t *testing.T) {
145145
{
146146
"_id": "abc",
147147
"hostname": "localhost",
148+
"_rev": float64(0), // json.Unmarshal will convert int64 to float64, so use float64 here so the asserts are okay
149+
"_created": float64(1000),
150+
"_updated": float64(2000),
148151
},
149152
}
150153

@@ -162,6 +165,9 @@ func TestQueryOK(t *testing.T) {
162165
assert.Equal(t, "query="+query, gotQuery)
163166
assert.Equal(t, got, respData)
164167
assert.Equal(t, ctx, httpRT.gotCtx)
168+
assert.Equal(t, int64(0), got[0].Rev())
169+
assert.Equal(t, time.Unix(0, 1000), got[0].Created())
170+
assert.Equal(t, time.Unix(0, 2000), got[0].Updated())
165171
}
166172

167173
func TestQueryNoResults(t *testing.T) {

‎entity/store.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,10 +129,13 @@ func (s store) CreateEntities(wo WriteOp, entities []etre.Entity) ([]string, err
129129
// A slice of IDs we generate to insert along with entities into DB
130130
newIds := make([]string, 0, len(entities))
131131

132+
now := time.Now().UnixNano()
132133
for i := range entities {
133134
entities[i]["_id"] = primitive.NewObjectID()
134135
entities[i]["_type"] = wo.EntityType
135136
entities[i]["_rev"] = int64(0)
137+
entities[i]["_created"] = now
138+
entities[i]["_updated"] = now
136139

137140
res, err := c.InsertOne(s.ctx, entities[i])
138141
if err != nil {
@@ -187,14 +190,15 @@ func (s store) UpdateEntities(wo WriteOp, q query.Query, patch etre.Entity) ([]e
187190
// diffs is a slice made up of a diff for each doc updated
188191
diffs := []etre.Entity{}
189192

193+
patch["_updated"] = time.Now().UnixNano()
190194
updates := bson.M{
191195
"$set": patch,
192196
"$inc": bson.M{
193197
"_rev": 1, // increment the revision
194198
},
195199
}
196200

197-
p := bson.M{"_id": 1, "_type": 1, "_rev": 1}
201+
p := bson.M{"_id": 1, "_type": 1, "_rev": 1, "_updated": 1}
198202
for label := range patch {
199203
p[label] = 1
200204
}

‎entity/store_test.go

Lines changed: 71 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package entity_test
55
import (
66
"context"
77
"testing"
8+
"time"
89

910
"github.com/stretchr/testify/assert"
1011
"github.com/stretchr/testify/require"
@@ -62,10 +63,11 @@ func setup(t *testing.T, cdcm *mock.CDCStore) entity.Store {
6263
_, err = iv.CreateOne(context.TODO(), idx)
6364
require.NoError(t, err)
6465

66+
now := time.Now().UnixNano()
6567
testNodes = []etre.Entity{
66-
{"_type": entityType, "_rev": int64(0), "x": int64(2), "y": "a", "z": int64(9), "foo": ""},
67-
{"_type": entityType, "_rev": int64(0), "x": int64(4), "y": "b", "bar": ""},
68-
{"_type": entityType, "_rev": int64(0), "x": int64(6), "y": "b", "bar": ""},
68+
{"_type": entityType, "_rev": int64(0), "x": int64(2), "y": "a", "z": int64(9), "foo": "", "_created": now, "_updated": now},
69+
{"_type": entityType, "_rev": int64(0), "x": int64(4), "y": "b", "bar": "", "_created": now, "_updated": now},
70+
{"_type": entityType, "_rev": int64(0), "x": int64(6), "y": "b", "bar": "", "_created": now, "_updated": now},
6971
}
7072
res, err := nodesColl.InsertMany(context.TODO(), docs(testNodes))
7173
require.NoError(t, err)
@@ -210,11 +212,12 @@ func TestReadEntitiesFilterReturnMetalabels(t *testing.T) {
210212
q, err := query.Translate("y=a")
211213
require.NoError(t, err)
212214

213-
actual, err := store.ReadEntities(entityType, q, etre.QueryFilter{ReturnLabels: []string{"_id", "_type", "_rev", "y"}})
215+
actual, err := store.ReadEntities(entityType, q, etre.QueryFilter{ReturnLabels: []string{"_id", "_type", "_rev", "y", "_created", "_updated"}})
214216
require.NoError(t, err)
215217

216218
expect := []etre.Entity{
217-
{"_id": testNodes[0]["_id"], "_type": entityType, "_rev": int64(0), "y": "a"},
219+
{"_id": testNodes[0]["_id"], "_type": entityType, "_rev": int64(0), "y": "a",
220+
"_created": testNodes[0]["_created"], "_updated": testNodes[0]["_updated"]},
218221
}
219222
assert.Equal(t, expect, actual)
220223
}
@@ -248,6 +251,9 @@ func TestCreateEntitiesMultiple(t *testing.T) {
248251
id1, _ := primitive.ObjectIDFromHex(ids[0])
249252
id2, _ := primitive.ObjectIDFromHex(ids[1])
250253
id3, _ := primitive.ObjectIDFromHex(ids[2])
254+
255+
createTime, ok := (*gotEvents[0].New)["_created"].(int64)
256+
require.True(t, ok, "expected _created to be int64, got %T", (*gotEvents[0].New)["_created"])
251257
expectEvents := []etre.CDCEvent{
252258
{
253259
Id: gotEvents[0].Id, // non-deterministic
@@ -258,7 +264,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
258264
Caller: username,
259265
Op: "i",
260266
Old: nil,
261-
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 7},
267+
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 7, "_created": createTime, "_updated": createTime},
262268
},
263269
{
264270
Id: gotEvents[1].Id, // non-deterministic
@@ -269,7 +275,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
269275
Caller: username,
270276
Op: "i",
271277
Old: nil,
272-
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "x": 8},
278+
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "x": 8, "_created": createTime, "_updated": createTime},
273279
},
274280
{
275281
Id: gotEvents[2].Id, // non-deterministic
@@ -280,7 +286,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
280286
Caller: username,
281287
Op: "i",
282288
Old: nil,
283-
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "x": 9, "_setId": "343", "_setOp": "something", "_setSize": 1},
289+
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "x": 9, "_setId": "343", "_setOp": "something", "_setSize": 1, "_created": createTime, "_updated": createTime},
284290
SetId: "343",
285291
SetOp: "something",
286292
SetSize: 1,
@@ -317,6 +323,7 @@ func TestCreateEntitiesMultiplePartialSuccess(t *testing.T) {
317323

318324
// Only x=5 written/inserted, so only a CDC event for it
319325
id1, _ := primitive.ObjectIDFromHex(ids[0])
326+
upd1 := (*gotEvents[0].New)["_updated"].(int64)
320327
expectEvents := []etre.CDCEvent{
321328
{
322329
Id: gotEvents[0].Id, // non-deterministic
@@ -327,9 +334,10 @@ func TestCreateEntitiesMultiplePartialSuccess(t *testing.T) {
327334
Caller: username,
328335
Op: "i",
329336
Old: nil,
330-
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 5},
337+
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 5, "_created": upd1, "_updated": upd1},
331338
},
332339
}
340+
assert.Greater(t, upd1, time.Now().Add(-10*time.Second).UnixNano(), "expected _created/_updated to be within the last 10 seconds")
333341
assert.Equal(t, expectEvents, gotEvents)
334342
}
335343

@@ -368,10 +376,11 @@ func TestUpdateEntities(t *testing.T) {
368376
assert.Len(t, gotDiffs, 1)
369377
expectDiffs := []etre.Entity{
370378
{
371-
"_id": testNodes[0]["_id"],
372-
"_type": entityType,
373-
"_rev": int64(0),
374-
"y": "a",
379+
"_id": testNodes[0]["_id"],
380+
"_type": entityType,
381+
"_rev": int64(0),
382+
"_updated": testNodes[0]["_updated"],
383+
"y": "a",
375384
},
376385
}
377386
assert.Equal(t, expectDiffs, gotDiffs)
@@ -394,16 +403,18 @@ func TestUpdateEntities(t *testing.T) {
394403
assert.Len(t, gotDiffs, 2)
395404
expectDiffs = []etre.Entity{
396405
{
397-
"_id": testNodes[1]["_id"],
398-
"_type": entityType,
399-
"_rev": int64(0),
400-
"y": "b",
406+
"_id": testNodes[1]["_id"],
407+
"_type": entityType,
408+
"_rev": int64(0),
409+
"_updated": testNodes[1]["_updated"],
410+
"y": "b",
401411
},
402412
{
403-
"_id": testNodes[2]["_id"],
404-
"_type": entityType,
405-
"_rev": int64(0),
406-
"y": "b",
413+
"_id": testNodes[2]["_id"],
414+
"_type": entityType,
415+
"_rev": int64(0),
416+
"_updated": testNodes[2]["_updated"],
417+
"y": "b",
407418
},
408419
}
409420
assert.Equal(t, expectDiffs, gotDiffs)
@@ -417,15 +428,18 @@ func TestUpdateEntities(t *testing.T) {
417428
id1, _ := testNodes[0]["_id"].(primitive.ObjectID)
418429
id2, _ := testNodes[1]["_id"].(primitive.ObjectID)
419430
id3, _ := testNodes[2]["_id"].(primitive.ObjectID)
431+
upd1 := (*gotEvents[0].New)["_updated"].(int64)
432+
upd2 := (*gotEvents[1].New)["_updated"].(int64)
433+
upd3 := (*gotEvents[2].New)["_updated"].(int64)
420434
expectEvent := []etre.CDCEvent{
421435
{
422436
EntityId: id1.Hex(),
423437
EntityType: entityType,
424438
EntityRev: int64(1),
425439
Caller: username,
426440
Op: "u",
427-
Old: &etre.Entity{"y": "a"},
428-
New: &etre.Entity{"y": "y"},
441+
Old: &etre.Entity{"y": "a", "_updated": testNodes[0]["_updated"]},
442+
New: &etre.Entity{"y": "y", "_updated": upd1},
429443
SetId: "111",
430444
SetOp: "update-y1",
431445
SetSize: 1,
@@ -436,8 +450,8 @@ func TestUpdateEntities(t *testing.T) {
436450
EntityRev: int64(1),
437451
Caller: username,
438452
Op: "u",
439-
Old: &etre.Entity{"y": "b"},
440-
New: &etre.Entity{"y": "c"},
453+
Old: &etre.Entity{"y": "b", "_updated": testNodes[0]["_updated"]},
454+
New: &etre.Entity{"y": "c", "_updated": upd2},
441455
SetId: "222",
442456
SetOp: "update-y2",
443457
SetSize: 1,
@@ -448,14 +462,17 @@ func TestUpdateEntities(t *testing.T) {
448462
EntityRev: int64(1),
449463
Caller: username,
450464
Op: "u",
451-
Old: &etre.Entity{"y": "b"},
452-
New: &etre.Entity{"y": "c"},
465+
Old: &etre.Entity{"y": "b", "_updated": testNodes[0]["_updated"]},
466+
New: &etre.Entity{"y": "c", "_updated": upd3},
453467
SetId: "222",
454468
SetOp: "update-y2",
455469
SetSize: 1,
456470
},
457471
}
458472
assert.Equal(t, expectEvent, gotEvents)
473+
assert.Greater(t, upd1, testNodes[0]["_updated"].(int64), "expected _updated to be greater than original value")
474+
assert.Greater(t, upd2, testNodes[1]["_updated"].(int64), "expected _updated to be greater than original value")
475+
assert.Greater(t, upd3, testNodes[2]["_updated"].(int64), "expected _updated to be greater than original value")
459476
}
460477

461478
func TestUpdateEntitiesById(t *testing.T) {
@@ -488,10 +505,11 @@ func TestUpdateEntitiesById(t *testing.T) {
488505
require.NoError(t, err)
489506
expectDiffs := []etre.Entity{
490507
{
491-
"_id": testNodes[0]["_id"],
492-
"_type": entityType,
493-
"_rev": int64(0),
494-
"y": "a",
508+
"_id": testNodes[0]["_id"],
509+
"_type": entityType,
510+
"_rev": int64(0),
511+
"_updated": testNodes[0]["_updated"],
512+
"y": "a",
495513
},
496514
}
497515
assert.Equal(t, expectDiffs, gotDiffs)
@@ -513,16 +531,18 @@ func TestUpdateEntitiesById(t *testing.T) {
513531
require.NoError(t, err)
514532
expectDiffs = []etre.Entity{
515533
{
516-
"_id": testNodes[1]["_id"],
517-
"_type": entityType,
518-
"_rev": int64(0),
519-
"y": "b",
534+
"_id": testNodes[1]["_id"],
535+
"_type": entityType,
536+
"_rev": int64(0),
537+
"_updated": testNodes[1]["_updated"],
538+
"y": "b",
520539
},
521540
{
522-
"_id": testNodes[2]["_id"],
523-
"_type": entityType,
524-
"_rev": int64(0),
525-
"y": "b",
541+
"_id": testNodes[2]["_id"],
542+
"_type": entityType,
543+
"_rev": int64(0),
544+
"_updated": testNodes[2]["_updated"],
545+
"y": "b",
526546
},
527547
}
528548
assert.Equal(t, expectDiffs, gotDiffs)
@@ -536,15 +556,18 @@ func TestUpdateEntitiesById(t *testing.T) {
536556
id1, _ := testNodes[0]["_id"].(primitive.ObjectID)
537557
id2, _ := testNodes[1]["_id"].(primitive.ObjectID)
538558
id3, _ := testNodes[2]["_id"].(primitive.ObjectID)
559+
upd1 := (*gotEvents[0].New)["_updated"].(int64)
560+
upd2 := (*gotEvents[1].New)["_updated"].(int64)
561+
upd3 := (*gotEvents[2].New)["_updated"].(int64)
539562
expectEvent := []etre.CDCEvent{
540563
{
541564
EntityId: id1.Hex(),
542565
EntityType: entityType,
543566
EntityRev: int64(1),
544567
Caller: username,
545568
Op: "u",
546-
Old: &etre.Entity{"y": "a"},
547-
New: &etre.Entity{"y": "y"},
569+
Old: &etre.Entity{"y": "a", "_updated": testNodes[0]["_updated"]},
570+
New: &etre.Entity{"y": "y", "_updated": upd1},
548571
SetId: "111",
549572
SetOp: "update-y1",
550573
SetSize: 1,
@@ -555,8 +578,8 @@ func TestUpdateEntitiesById(t *testing.T) {
555578
EntityRev: int64(1),
556579
Caller: username,
557580
Op: "u",
558-
Old: &etre.Entity{"y": "b"},
559-
New: &etre.Entity{"y": "c"},
581+
Old: &etre.Entity{"y": "b", "_updated": testNodes[1]["_updated"]},
582+
New: &etre.Entity{"y": "c", "_updated": upd2},
560583
SetId: "222",
561584
SetOp: "update-y2",
562585
SetSize: 1,
@@ -567,13 +590,16 @@ func TestUpdateEntitiesById(t *testing.T) {
567590
EntityRev: int64(1),
568591
Caller: username,
569592
Op: "u",
570-
Old: &etre.Entity{"y": "b"},
571-
New: &etre.Entity{"y": "c"},
593+
Old: &etre.Entity{"y": "b", "_updated": testNodes[2]["_updated"]},
594+
New: &etre.Entity{"y": "c", "_updated": upd3},
572595
SetId: "222",
573596
SetOp: "update-y2",
574597
SetSize: 1,
575598
},
576599
}
600+
assert.Greater(t, upd1, testNodes[0]["_updated"].(int64), "expected _updated to increase after update")
601+
assert.Greater(t, upd2, testNodes[1]["_updated"].(int64), "expected _updated to increase after update")
602+
assert.Greater(t, upd3, testNodes[2]["_updated"].(int64), "expected _updated to increase after update")
577603
assert.Equal(t, expectEvent, gotEvents)
578604
}
579605

0 commit comments

Comments
 (0)