Skip to content

Add _created and _updated metalables #89

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 4, 2025
Merged

Add _created and _updated metalables #89

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions api/api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,9 @@ type server struct {
}

var testEntities = []etre.Entity{
{"_id": "59f10d2a5669fc79103a0000", "_type": "node", "_rev": int64(0), "x": "1", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a1111", "_type": "node", "_rev": int64(0), "x": "2", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a2222", "_type": "node", "_rev": int64(0), "x": "3", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a0000", "_type": "node", "_rev": int64(0), "_created": int64(1000), "_updated": int64(2000), "x": "1", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a1111", "_type": "node", "_rev": int64(0), "_created": int64(3000), "_updated": int64(4000), "x": "2", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a2222", "_type": "node", "_rev": int64(0), "_created": int64(5000), "_updated": int64(6000), "x": "3", "foo": "bar"},
}

var testEntityIds = []string{"59f10d2a5669fc79103a0000", "59f10d2a5669fc79103a1111", "59f10d2a5669fc79103a2222"}
Expand All @@ -66,9 +66,9 @@ var (
)

var testEntitiesWithObjectIDs = []etre.Entity{
{"_id": testEntityId0, "_type": "node", "_rev": int64(0), "x": "1", "foo": "bar"},
{"_id": testEntityId1, "_type": "node", "_rev": int64(0), "x": "2", "foo": "bar"},
{"_id": testEntityId2, "_type": "node", "_rev": int64(0), "x": "3", "foo": "bar"},
{"_id": testEntityId0, "_type": "node", "_rev": int64(0), "_created": int64(1000), "_updated": int64(2000), "x": "1", "foo": "bar"},
{"_id": testEntityId1, "_type": "node", "_rev": int64(0), "_created": int64(3000), "_updated": int64(4000), "x": "2", "foo": "bar"},
{"_id": testEntityId2, "_type": "node", "_rev": int64(0), "_created": int64(5000), "_updated": int64(6000), "x": "3", "foo": "bar"},
}

var defaultConfig = config.Config{
Expand Down Expand Up @@ -125,11 +125,13 @@ func uri(id string) string {
return addr + etre.API_ROOT + "/entity/" + id
}

func fixRev(e []etre.Entity) {
for i := range e {
f := e[i]["_rev"].(float64)
delete(e[i], "_rev")
e[i]["_rev"] = int64(f)
func fixInt64(e []etre.Entity) {
for _, label := range []string{"_rev", "_updated", "_created"} {
for i := range e {
f := e[i][label].(float64)
delete(e[i], label)
e[i][label] = int64(f)
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion api/query_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func TestQueryBasic(t *testing.T) {
expectFilter := etre.QueryFilter{}
assert.Equal(t, expectFilter, gotFilter)

fixRev(gotEntities) // JSON float64(_rev) ->, int64(_rev)
fixInt64(gotEntities) // JSON float64(_rev) ->, int64(_rev)
assert.Equal(t, testEntities, gotEntities)

// -- Metrics -----------------------------------------------------------
Expand Down
2 changes: 1 addition & 1 deletion api/single_entity_read_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func TestGetEntityBasic(t *testing.T) {
expectFilter := etre.QueryFilter{}
assert.Equal(t, expectFilter, gotFilter)

fixRev([]etre.Entity{gotEntity}) // JSON float64(_rev) ->, int64(_rev)
fixInt64([]etre.Entity{gotEntity}) // JSON float64(_rev) ->, int64(_rev)
assert.Equal(t, testEntities[0], gotEntity)

// -- Metrics -----------------------------------------------------------
Expand Down
6 changes: 6 additions & 0 deletions client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,9 @@ func TestQueryOK(t *testing.T) {
{
"_id": "abc",
"hostname": "localhost",
"_rev": float64(0), // json.Unmarshal will convert int64 to float64, so use float64 here so the asserts are okay
"_created": float64(1000),
"_updated": float64(2000),
},
}

Expand All @@ -162,6 +165,9 @@ func TestQueryOK(t *testing.T) {
assert.Equal(t, "query="+query, gotQuery)
assert.Equal(t, got, respData)
assert.Equal(t, ctx, httpRT.gotCtx)
assert.Equal(t, int64(0), got[0].Rev())
assert.Equal(t, time.Unix(0, 1000), got[0].Created())
assert.Equal(t, time.Unix(0, 2000), got[0].Updated())
}

func TestQueryNoResults(t *testing.T) {
Expand Down
6 changes: 5 additions & 1 deletion entity/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,13 @@ func (s store) CreateEntities(wo WriteOp, entities []etre.Entity) ([]string, err
// A slice of IDs we generate to insert along with entities into DB
newIds := make([]string, 0, len(entities))

now := time.Now().UnixNano()
for i := range entities {
entities[i]["_id"] = primitive.NewObjectID()
entities[i]["_type"] = wo.EntityType
entities[i]["_rev"] = int64(0)
entities[i]["_created"] = now
entities[i]["_updated"] = now

res, err := c.InsertOne(s.ctx, entities[i])
if err != nil {
Expand Down Expand Up @@ -187,14 +190,15 @@ func (s store) UpdateEntities(wo WriteOp, q query.Query, patch etre.Entity) ([]e
// diffs is a slice made up of a diff for each doc updated
diffs := []etre.Entity{}

patch["_updated"] = time.Now().UnixNano()
updates := bson.M{
"$set": patch,
"$inc": bson.M{
"_rev": 1, // increment the revision
},
}

p := bson.M{"_id": 1, "_type": 1, "_rev": 1}
p := bson.M{"_id": 1, "_type": 1, "_rev": 1, "_updated": 1}
for label := range patch {
p[label] = 1
}
Expand Down
116 changes: 71 additions & 45 deletions entity/store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package entity_test
import (
"context"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -62,10 +63,11 @@ func setup(t *testing.T, cdcm *mock.CDCStore) entity.Store {
_, err = iv.CreateOne(context.TODO(), idx)
require.NoError(t, err)

now := time.Now().UnixNano()
testNodes = []etre.Entity{
{"_type": entityType, "_rev": int64(0), "x": int64(2), "y": "a", "z": int64(9), "foo": ""},
{"_type": entityType, "_rev": int64(0), "x": int64(4), "y": "b", "bar": ""},
{"_type": entityType, "_rev": int64(0), "x": int64(6), "y": "b", "bar": ""},
{"_type": entityType, "_rev": int64(0), "x": int64(2), "y": "a", "z": int64(9), "foo": "", "_created": now, "_updated": now},
{"_type": entityType, "_rev": int64(0), "x": int64(4), "y": "b", "bar": "", "_created": now, "_updated": now},
{"_type": entityType, "_rev": int64(0), "x": int64(6), "y": "b", "bar": "", "_created": now, "_updated": now},
}
res, err := nodesColl.InsertMany(context.TODO(), docs(testNodes))
require.NoError(t, err)
Expand Down Expand Up @@ -210,11 +212,12 @@ func TestReadEntitiesFilterReturnMetalabels(t *testing.T) {
q, err := query.Translate("y=a")
require.NoError(t, err)

actual, err := store.ReadEntities(entityType, q, etre.QueryFilter{ReturnLabels: []string{"_id", "_type", "_rev", "y"}})
actual, err := store.ReadEntities(entityType, q, etre.QueryFilter{ReturnLabels: []string{"_id", "_type", "_rev", "y", "_created", "_updated"}})
require.NoError(t, err)

expect := []etre.Entity{
{"_id": testNodes[0]["_id"], "_type": entityType, "_rev": int64(0), "y": "a"},
{"_id": testNodes[0]["_id"], "_type": entityType, "_rev": int64(0), "y": "a",
"_created": testNodes[0]["_created"], "_updated": testNodes[0]["_updated"]},
}
assert.Equal(t, expect, actual)
}
Expand Down Expand Up @@ -248,6 +251,9 @@ func TestCreateEntitiesMultiple(t *testing.T) {
id1, _ := primitive.ObjectIDFromHex(ids[0])
id2, _ := primitive.ObjectIDFromHex(ids[1])
id3, _ := primitive.ObjectIDFromHex(ids[2])

createTime, ok := (*gotEvents[0].New)["_created"].(int64)
require.True(t, ok, "expected _created to be int64, got %T", (*gotEvents[0].New)["_created"])
expectEvents := []etre.CDCEvent{
{
Id: gotEvents[0].Id, // non-deterministic
Expand All @@ -258,7 +264,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 7},
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 7, "_created": createTime, "_updated": createTime},
},
{
Id: gotEvents[1].Id, // non-deterministic
Expand All @@ -269,7 +275,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "x": 8},
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "x": 8, "_created": createTime, "_updated": createTime},
},
{
Id: gotEvents[2].Id, // non-deterministic
Expand All @@ -280,7 +286,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "x": 9, "_setId": "343", "_setOp": "something", "_setSize": 1},
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "x": 9, "_setId": "343", "_setOp": "something", "_setSize": 1, "_created": createTime, "_updated": createTime},
SetId: "343",
SetOp: "something",
SetSize: 1,
Expand Down Expand Up @@ -317,6 +323,7 @@ func TestCreateEntitiesMultiplePartialSuccess(t *testing.T) {

// Only x=5 written/inserted, so only a CDC event for it
id1, _ := primitive.ObjectIDFromHex(ids[0])
upd1 := (*gotEvents[0].New)["_updated"].(int64)
expectEvents := []etre.CDCEvent{
{
Id: gotEvents[0].Id, // non-deterministic
Expand All @@ -327,9 +334,10 @@ func TestCreateEntitiesMultiplePartialSuccess(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 5},
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 5, "_created": upd1, "_updated": upd1},
},
}
assert.Greater(t, upd1, time.Now().Add(-10*time.Second).UnixNano(), "expected _created/_updated to be within the last 10 seconds")
assert.Equal(t, expectEvents, gotEvents)
}

Expand Down Expand Up @@ -368,10 +376,11 @@ func TestUpdateEntities(t *testing.T) {
assert.Len(t, gotDiffs, 1)
expectDiffs := []etre.Entity{
{
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "a",
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[0]["_updated"],
"y": "a",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
Expand All @@ -394,16 +403,18 @@ func TestUpdateEntities(t *testing.T) {
assert.Len(t, gotDiffs, 2)
expectDiffs = []etre.Entity{
{
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[1]["_updated"],
"y": "b",
},
{
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[2]["_updated"],
"y": "b",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
Expand All @@ -417,15 +428,18 @@ func TestUpdateEntities(t *testing.T) {
id1, _ := testNodes[0]["_id"].(primitive.ObjectID)
id2, _ := testNodes[1]["_id"].(primitive.ObjectID)
id3, _ := testNodes[2]["_id"].(primitive.ObjectID)
upd1 := (*gotEvents[0].New)["_updated"].(int64)
upd2 := (*gotEvents[1].New)["_updated"].(int64)
upd3 := (*gotEvents[2].New)["_updated"].(int64)
expectEvent := []etre.CDCEvent{
{
EntityId: id1.Hex(),
EntityType: entityType,
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "a"},
New: &etre.Entity{"y": "y"},
Old: &etre.Entity{"y": "a", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "y", "_updated": upd1},
SetId: "111",
SetOp: "update-y1",
SetSize: 1,
Expand All @@ -436,8 +450,8 @@ func TestUpdateEntities(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd2},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
Expand All @@ -448,14 +462,17 @@ func TestUpdateEntities(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd3},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
},
}
assert.Equal(t, expectEvent, gotEvents)
assert.Greater(t, upd1, testNodes[0]["_updated"].(int64), "expected _updated to be greater than original value")
assert.Greater(t, upd2, testNodes[1]["_updated"].(int64), "expected _updated to be greater than original value")
assert.Greater(t, upd3, testNodes[2]["_updated"].(int64), "expected _updated to be greater than original value")
}

func TestUpdateEntitiesById(t *testing.T) {
Expand Down Expand Up @@ -488,10 +505,11 @@ func TestUpdateEntitiesById(t *testing.T) {
require.NoError(t, err)
expectDiffs := []etre.Entity{
{
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "a",
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[0]["_updated"],
"y": "a",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
Expand All @@ -513,16 +531,18 @@ func TestUpdateEntitiesById(t *testing.T) {
require.NoError(t, err)
expectDiffs = []etre.Entity{
{
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[1]["_updated"],
"y": "b",
},
{
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[2]["_updated"],
"y": "b",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
Expand All @@ -536,15 +556,18 @@ func TestUpdateEntitiesById(t *testing.T) {
id1, _ := testNodes[0]["_id"].(primitive.ObjectID)
id2, _ := testNodes[1]["_id"].(primitive.ObjectID)
id3, _ := testNodes[2]["_id"].(primitive.ObjectID)
upd1 := (*gotEvents[0].New)["_updated"].(int64)
upd2 := (*gotEvents[1].New)["_updated"].(int64)
upd3 := (*gotEvents[2].New)["_updated"].(int64)
expectEvent := []etre.CDCEvent{
{
EntityId: id1.Hex(),
EntityType: entityType,
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "a"},
New: &etre.Entity{"y": "y"},
Old: &etre.Entity{"y": "a", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "y", "_updated": upd1},
SetId: "111",
SetOp: "update-y1",
SetSize: 1,
Expand All @@ -555,8 +578,8 @@ func TestUpdateEntitiesById(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[1]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd2},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
Expand All @@ -567,13 +590,16 @@ func TestUpdateEntitiesById(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[2]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd3},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
},
}
assert.Greater(t, upd1, testNodes[0]["_updated"].(int64), "expected _updated to increase after update")
assert.Greater(t, upd2, testNodes[1]["_updated"].(int64), "expected _updated to increase after update")
assert.Greater(t, upd3, testNodes[2]["_updated"].(int64), "expected _updated to increase after update")
assert.Equal(t, expectEvent, gotEvents)
}

Expand Down
Loading