Skip to content

Add _created and _updated metalables #89

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 4, 2025
+207 −94
Merged

Add _created and _updated metalables #89

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions api/api_test.go
View file
Original file line number Diff line number Diff line change
@@ -52,9 +52,9 @@ type server struct {
}

var testEntities = []etre.Entity{
{"_id": "59f10d2a5669fc79103a0000", "_type": "node", "_rev": int64(0), "x": "1", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a1111", "_type": "node", "_rev": int64(0), "x": "2", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a2222", "_type": "node", "_rev": int64(0), "x": "3", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a0000", "_type": "node", "_rev": int64(0), "_created": int64(1000), "_updated": int64(2000), "x": "1", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a1111", "_type": "node", "_rev": int64(0), "_created": int64(3000), "_updated": int64(4000), "x": "2", "foo": "bar"},
{"_id": "59f10d2a5669fc79103a2222", "_type": "node", "_rev": int64(0), "_created": int64(5000), "_updated": int64(6000), "x": "3", "foo": "bar"},
}

var testEntityIds = []string{"59f10d2a5669fc79103a0000", "59f10d2a5669fc79103a1111", "59f10d2a5669fc79103a2222"}
@@ -66,9 +66,9 @@ var (
)

var testEntitiesWithObjectIDs = []etre.Entity{
{"_id": testEntityId0, "_type": "node", "_rev": int64(0), "x": "1", "foo": "bar"},
{"_id": testEntityId1, "_type": "node", "_rev": int64(0), "x": "2", "foo": "bar"},
{"_id": testEntityId2, "_type": "node", "_rev": int64(0), "x": "3", "foo": "bar"},
{"_id": testEntityId0, "_type": "node", "_rev": int64(0), "_created": int64(1000), "_updated": int64(2000), "x": "1", "foo": "bar"},
{"_id": testEntityId1, "_type": "node", "_rev": int64(0), "_created": int64(3000), "_updated": int64(4000), "x": "2", "foo": "bar"},
{"_id": testEntityId2, "_type": "node", "_rev": int64(0), "_created": int64(5000), "_updated": int64(6000), "x": "3", "foo": "bar"},
}

var defaultConfig = config.Config{
@@ -125,11 +125,13 @@ func uri(id string) string {
return addr + etre.API_ROOT + "/entity/" + id
}

func fixRev(e []etre.Entity) {
for i := range e {
f := e[i]["_rev"].(float64)
delete(e[i], "_rev")
e[i]["_rev"] = int64(f)
func fixInt64(e []etre.Entity) {
for _, label := range []string{"_rev", "_updated", "_created"} {
for i := range e {
f := e[i][label].(float64)
delete(e[i], label)
e[i][label] = int64(f)
}
}
}

2 changes: 1 addition & 1 deletion api/query_test.go
View file
Original file line number Diff line number Diff line change
@@ -52,7 +52,7 @@ func TestQueryBasic(t *testing.T) {
expectFilter := etre.QueryFilter{}
assert.Equal(t, expectFilter, gotFilter)

fixRev(gotEntities) // JSON float64(_rev) ->, int64(_rev)
fixInt64(gotEntities) // JSON float64(_rev) ->, int64(_rev)
assert.Equal(t, testEntities, gotEntities)

// -- Metrics -----------------------------------------------------------
2 changes: 1 addition & 1 deletion api/single_entity_read_test.go
View file
Original file line number Diff line number Diff line change
@@ -55,7 +55,7 @@ func TestGetEntityBasic(t *testing.T) {
expectFilter := etre.QueryFilter{}
assert.Equal(t, expectFilter, gotFilter)

fixRev([]etre.Entity{gotEntity}) // JSON float64(_rev) ->, int64(_rev)
fixInt64([]etre.Entity{gotEntity}) // JSON float64(_rev) ->, int64(_rev)
assert.Equal(t, testEntities[0], gotEntity)

// -- Metrics -----------------------------------------------------------
6 changes: 6 additions & 0 deletions client_test.go
View file
Original file line number Diff line number Diff line change
@@ -145,6 +145,9 @@ func TestQueryOK(t *testing.T) {
{
"_id": "abc",
"hostname": "localhost",
"_rev": float64(0), // json.Unmarshal will convert int64 to float64, so use float64 here so the asserts are okay
"_created": float64(1000),
"_updated": float64(2000),
},
}

@@ -162,6 +165,9 @@ func TestQueryOK(t *testing.T) {
assert.Equal(t, "query="+query, gotQuery)
assert.Equal(t, got, respData)
assert.Equal(t, ctx, httpRT.gotCtx)
assert.Equal(t, int64(0), got[0].Rev())
assert.Equal(t, time.Unix(0, 1000), got[0].Created())
assert.Equal(t, time.Unix(0, 2000), got[0].Updated())
}

func TestQueryNoResults(t *testing.T) {
6 changes: 5 additions & 1 deletion entity/store.go
View file
Original file line number Diff line number Diff line change
@@ -129,10 +129,13 @@ func (s store) CreateEntities(wo WriteOp, entities []etre.Entity) ([]string, err
// A slice of IDs we generate to insert along with entities into DB
newIds := make([]string, 0, len(entities))

now := time.Now().UnixNano()
for i := range entities {
entities[i]["_id"] = primitive.NewObjectID()
entities[i]["_type"] = wo.EntityType
entities[i]["_rev"] = int64(0)
entities[i]["_created"] = now
entities[i]["_updated"] = now

res, err := c.InsertOne(s.ctx, entities[i])
if err != nil {
@@ -187,14 +190,15 @@ func (s store) UpdateEntities(wo WriteOp, q query.Query, patch etre.Entity) ([]e
// diffs is a slice made up of a diff for each doc updated
diffs := []etre.Entity{}

patch["_updated"] = time.Now().UnixNano()
updates := bson.M{
"$set": patch,
"$inc": bson.M{
"_rev": 1, // increment the revision
},
}

p := bson.M{"_id": 1, "_type": 1, "_rev": 1}
p := bson.M{"_id": 1, "_type": 1, "_rev": 1, "_updated": 1}
for label := range patch {
p[label] = 1
}
116 changes: 71 additions & 45 deletions entity/store_test.go
View file
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ package entity_test
import (
"context"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -62,10 +63,11 @@ func setup(t *testing.T, cdcm *mock.CDCStore) entity.Store {
_, err = iv.CreateOne(context.TODO(), idx)
require.NoError(t, err)

now := time.Now().UnixNano()
testNodes = []etre.Entity{
{"_type": entityType, "_rev": int64(0), "x": int64(2), "y": "a", "z": int64(9), "foo": ""},
{"_type": entityType, "_rev": int64(0), "x": int64(4), "y": "b", "bar": ""},
{"_type": entityType, "_rev": int64(0), "x": int64(6), "y": "b", "bar": ""},
{"_type": entityType, "_rev": int64(0), "x": int64(2), "y": "a", "z": int64(9), "foo": "", "_created": now, "_updated": now},
{"_type": entityType, "_rev": int64(0), "x": int64(4), "y": "b", "bar": "", "_created": now, "_updated": now},
{"_type": entityType, "_rev": int64(0), "x": int64(6), "y": "b", "bar": "", "_created": now, "_updated": now},
}
res, err := nodesColl.InsertMany(context.TODO(), docs(testNodes))
require.NoError(t, err)
@@ -210,11 +212,12 @@ func TestReadEntitiesFilterReturnMetalabels(t *testing.T) {
q, err := query.Translate("y=a")
require.NoError(t, err)

actual, err := store.ReadEntities(entityType, q, etre.QueryFilter{ReturnLabels: []string{"_id", "_type", "_rev", "y"}})
actual, err := store.ReadEntities(entityType, q, etre.QueryFilter{ReturnLabels: []string{"_id", "_type", "_rev", "y", "_created", "_updated"}})
require.NoError(t, err)

expect := []etre.Entity{
{"_id": testNodes[0]["_id"], "_type": entityType, "_rev": int64(0), "y": "a"},
{"_id": testNodes[0]["_id"], "_type": entityType, "_rev": int64(0), "y": "a",
"_created": testNodes[0]["_created"], "_updated": testNodes[0]["_updated"]},
}
assert.Equal(t, expect, actual)
}
@@ -248,6 +251,9 @@ func TestCreateEntitiesMultiple(t *testing.T) {
id1, _ := primitive.ObjectIDFromHex(ids[0])
id2, _ := primitive.ObjectIDFromHex(ids[1])
id3, _ := primitive.ObjectIDFromHex(ids[2])

createTime, ok := (*gotEvents[0].New)["_created"].(int64)
require.True(t, ok, "expected _created to be int64, got %T", (*gotEvents[0].New)["_created"])
expectEvents := []etre.CDCEvent{
{
Id: gotEvents[0].Id, // non-deterministic
@@ -258,7 +264,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 7},
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 7, "_created": createTime, "_updated": createTime},
},
{
Id: gotEvents[1].Id, // non-deterministic
@@ -269,7 +275,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "x": 8},
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "x": 8, "_created": createTime, "_updated": createTime},
},
{
Id: gotEvents[2].Id, // non-deterministic
@@ -280,7 +286,7 @@ func TestCreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "x": 9, "_setId": "343", "_setOp": "something", "_setSize": 1},
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "x": 9, "_setId": "343", "_setOp": "something", "_setSize": 1, "_created": createTime, "_updated": createTime},
SetId: "343",
SetOp: "something",
SetSize: 1,
@@ -317,6 +323,7 @@ func TestCreateEntitiesMultiplePartialSuccess(t *testing.T) {

// Only x=5 written/inserted, so only a CDC event for it
id1, _ := primitive.ObjectIDFromHex(ids[0])
upd1 := (*gotEvents[0].New)["_updated"].(int64)
expectEvents := []etre.CDCEvent{
{
Id: gotEvents[0].Id, // non-deterministic
@@ -327,9 +334,10 @@ func TestCreateEntitiesMultiplePartialSuccess(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 5},
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": 5, "_created": upd1, "_updated": upd1},
},
}
assert.Greater(t, upd1, time.Now().Add(-10*time.Second).UnixNano(), "expected _created/_updated to be within the last 10 seconds")
assert.Equal(t, expectEvents, gotEvents)
}

@@ -368,10 +376,11 @@ func TestUpdateEntities(t *testing.T) {
assert.Len(t, gotDiffs, 1)
expectDiffs := []etre.Entity{
{
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "a",
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[0]["_updated"],
"y": "a",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
@@ -394,16 +403,18 @@ func TestUpdateEntities(t *testing.T) {
assert.Len(t, gotDiffs, 2)
expectDiffs = []etre.Entity{
{
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[1]["_updated"],
"y": "b",
},
{
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[2]["_updated"],
"y": "b",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
@@ -417,15 +428,18 @@ func TestUpdateEntities(t *testing.T) {
id1, _ := testNodes[0]["_id"].(primitive.ObjectID)
id2, _ := testNodes[1]["_id"].(primitive.ObjectID)
id3, _ := testNodes[2]["_id"].(primitive.ObjectID)
upd1 := (*gotEvents[0].New)["_updated"].(int64)
upd2 := (*gotEvents[1].New)["_updated"].(int64)
upd3 := (*gotEvents[2].New)["_updated"].(int64)
expectEvent := []etre.CDCEvent{
{
EntityId: id1.Hex(),
EntityType: entityType,
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "a"},
New: &etre.Entity{"y": "y"},
Old: &etre.Entity{"y": "a", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "y", "_updated": upd1},
SetId: "111",
SetOp: "update-y1",
SetSize: 1,
@@ -436,8 +450,8 @@ func TestUpdateEntities(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd2},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
@@ -448,14 +462,17 @@ func TestUpdateEntities(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd3},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
},
}
assert.Equal(t, expectEvent, gotEvents)
assert.Greater(t, upd1, testNodes[0]["_updated"].(int64), "expected _updated to be greater than original value")
assert.Greater(t, upd2, testNodes[1]["_updated"].(int64), "expected _updated to be greater than original value")
assert.Greater(t, upd3, testNodes[2]["_updated"].(int64), "expected _updated to be greater than original value")
}

func TestUpdateEntitiesById(t *testing.T) {
@@ -488,10 +505,11 @@ func TestUpdateEntitiesById(t *testing.T) {
require.NoError(t, err)
expectDiffs := []etre.Entity{
{
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "a",
"_id": testNodes[0]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[0]["_updated"],
"y": "a",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
@@ -513,16 +531,18 @@ func TestUpdateEntitiesById(t *testing.T) {
require.NoError(t, err)
expectDiffs = []etre.Entity{
{
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[1]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[1]["_updated"],
"y": "b",
},
{
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"y": "b",
"_id": testNodes[2]["_id"],
"_type": entityType,
"_rev": int64(0),
"_updated": testNodes[2]["_updated"],
"y": "b",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
@@ -536,15 +556,18 @@ func TestUpdateEntitiesById(t *testing.T) {
id1, _ := testNodes[0]["_id"].(primitive.ObjectID)
id2, _ := testNodes[1]["_id"].(primitive.ObjectID)
id3, _ := testNodes[2]["_id"].(primitive.ObjectID)
upd1 := (*gotEvents[0].New)["_updated"].(int64)
upd2 := (*gotEvents[1].New)["_updated"].(int64)
upd3 := (*gotEvents[2].New)["_updated"].(int64)
expectEvent := []etre.CDCEvent{
{
EntityId: id1.Hex(),
EntityType: entityType,
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "a"},
New: &etre.Entity{"y": "y"},
Old: &etre.Entity{"y": "a", "_updated": testNodes[0]["_updated"]},
New: &etre.Entity{"y": "y", "_updated": upd1},
SetId: "111",
SetOp: "update-y1",
SetSize: 1,
@@ -555,8 +578,8 @@ func TestUpdateEntitiesById(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[1]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd2},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
@@ -567,13 +590,16 @@ func TestUpdateEntitiesById(t *testing.T) {
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "b"},
New: &etre.Entity{"y": "c"},
Old: &etre.Entity{"y": "b", "_updated": testNodes[2]["_updated"]},
New: &etre.Entity{"y": "c", "_updated": upd3},
SetId: "222",
SetOp: "update-y2",
SetSize: 1,
},
}
assert.Greater(t, upd1, testNodes[0]["_updated"].(int64), "expected _updated to increase after update")
assert.Greater(t, upd2, testNodes[1]["_updated"].(int64), "expected _updated to increase after update")
assert.Greater(t, upd3, testNodes[2]["_updated"].(int64), "expected _updated to increase after update")
assert.Equal(t, expectEvent, gotEvents)
}

38 changes: 23 additions & 15 deletions entity/v09_test.go
View file
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ package entity_test
import (
"context"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -52,14 +53,14 @@ func setupV09(t *testing.T, cdcm *mock.CDCStore) entity.Store {
}

v09testNodes = []etre.Entity{
{"_type": entityType, "_rev": int(0), "x": "a", "y": "a"},
{"_type": entityType, "_rev": int(0), "x": "b", "y": "a"},
{"_type": entityType, "_rev": int(0), "x": "c", "y": "a"},
{"_type": entityType, "_rev": int(0), "_updated": int(0), "_created": int(0), "x": "a", "y": "a"},
{"_type": entityType, "_rev": int(0), "_updated": int(0), "_created": int(0), "x": "b", "y": "a"},
{"_type": entityType, "_rev": int(0), "_updated": int(0), "_created": int(0), "x": "c", "y": "a"},
}
v09testNodes_int32 = []etre.Entity{
{"_type": entityType, "_rev": int32(0), "x": "a", "y": "a"},
{"_type": entityType, "_rev": int32(0), "x": "b", "y": "a"},
{"_type": entityType, "_rev": int32(0), "x": "c", "y": "a"},
{"_type": entityType, "_rev": int32(0), "_updated": int32(0), "_created": int32(0), "x": "a", "y": "a"},
{"_type": entityType, "_rev": int32(0), "_updated": int32(0), "_created": int32(0), "x": "b", "y": "a"},
{"_type": entityType, "_rev": int32(0), "_updated": int32(0), "_created": int32(0), "x": "c", "y": "a"},
}
res, err := nodesColl.InsertMany(context.TODO(), docs(v09testNodes))
require.NoError(t, err)
@@ -101,6 +102,7 @@ func TestV09CreateEntitiesMultiple(t *testing.T) {
id1, _ := primitive.ObjectIDFromHex(ids[0])
id2, _ := primitive.ObjectIDFromHex(ids[1])
id3, _ := primitive.ObjectIDFromHex(ids[2])
upd := (*gotEvents[0].New)["_updated"].(int64)
expectEvents := []etre.CDCEvent{
{
Id: gotEvents[0].Id, // non-deterministic
@@ -111,7 +113,7 @@ func TestV09CreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "x": "d"},
New: &etre.Entity{"_id": id1, "_type": entityType, "_rev": int64(0), "_created": upd, "_updated": upd, "x": "d"},
},
{
Id: gotEvents[1].Id, // non-deterministic
@@ -122,7 +124,7 @@ func TestV09CreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "x": "e"},
New: &etre.Entity{"_id": id2, "_type": entityType, "_rev": int64(0), "_created": upd, "_updated": upd, "x": "e"},
},
{
Id: gotEvents[2].Id, // non-deterministic
@@ -133,12 +135,13 @@ func TestV09CreateEntitiesMultiple(t *testing.T) {
Caller: username,
Op: "i",
Old: nil,
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "x": "f", "_setId": "343", "_setOp": "something", "_setSize": 1},
New: &etre.Entity{"_id": id3, "_type": entityType, "_rev": int64(0), "_created": upd, "_updated": upd, "x": "f", "_setId": "343", "_setOp": "something", "_setSize": 1},
SetId: "343",
SetOp: "something",
SetSize: 1,
},
}
assert.Greater(t, upd, time.Now().Add(-10*time.Second).UnixNano(), "expected _created/_updated to be within the last 10 seconds")
assert.Equal(t, expectEvents, gotEvents)
}

@@ -168,10 +171,11 @@ func TestV09UpdateEntities(t *testing.T) {
require.NoError(t, err)
expectDiffs := []etre.Entity{
{
"_id": v09testNodes[0]["_id"],
"_type": entityType,
"_rev": int32(0),
"y": "a",
"_id": v09testNodes[0]["_id"],
"_type": entityType,
"_rev": int32(0),
"_updated": int32(0),
"y": "a",
},
}
assert.Equal(t, expectDiffs, gotDiffs)
@@ -181,20 +185,22 @@ func TestV09UpdateEntities(t *testing.T) {
gotEvents[i].Ts = 0
}
id1, _ := v09testNodes[0]["_id"].(primitive.ObjectID)
upd := (*gotEvents[0].New)["_updated"].(int64)
expectEvent := []etre.CDCEvent{
{
EntityId: id1.Hex(),
EntityType: entityType,
EntityRev: int64(1),
Caller: username,
Op: "u",
Old: &etre.Entity{"y": "a"},
New: &etre.Entity{"y": "y"},
Old: &etre.Entity{"y": "a", "_updated": int32(0)},
New: &etre.Entity{"y": "y", "_updated": upd},
SetId: "111",
SetOp: "update-y1",
SetSize: 1,
},
}
assert.Greater(t, upd, time.Now().Add(-10*time.Second).UnixNano(), "expected _created/_updated to be within the last 10 seconds")
assert.Equal(t, expectEvent, gotEvents)
}

@@ -297,6 +303,8 @@ func TestV09DeleteLabel(t *testing.T) {
}
delete(e, "y") // because we deleted the label
e["_rev"] = int32(1) // because we deleted the label
e["_created"] = int32(0)
e["_updated"] = int32(0)
expectNew := []etre.Entity{e}
assert.Equal(t, expectNew, gotNew)

2 changes: 1 addition & 1 deletion entity/validate.go
View file
Original file line number Diff line number Diff line change
@@ -86,7 +86,7 @@ func (v validator) Entities(entities []etre.Entity, op byte) error {
switch op {
case VALIDATE_ON_CREATE:
// User cannot set these metalabels on create
for _, ml := range []string{"_id", "_type", "_rev", "_ts"} {
for _, ml := range []string{"_id", "_type", "_rev", "_created", "_updated"} {
if label != ml {
continue
}
2 changes: 2 additions & 0 deletions entity/validate_test.go
View file
Original file line number Diff line number Diff line change
@@ -29,6 +29,8 @@ func TestValidateCreateEntitiesErrorsMetalabels(t *testing.T) {
{"a": "b", "_id": "59f10d2a5669fc79103a1111"}, // _id not allowed
{"a": "b", "_type": "node"}, // _type not allowed
{"a": "b", "_rev": int64(0)}, // _rev not allowed
{"a": "b", "_created": int64(0)}, // _created not allowed
{"a": "b", "_updated": int64(0)}, // _updated not allowed
}

for _, e := range invalid {
12 changes: 12 additions & 0 deletions es/es.go
View file
Original file line number Diff line number Diff line change
@@ -453,6 +453,18 @@ func Run(ctx app.Context) {
return
}

// For the CLI, convert _created and _updated labels in place to RFC3339Nano format so they print human readable.
// Note that this will break entity.Created() and entity.Updated() since they expect this to be a numeric timestamp,
// but that's okay since the CLI just prints the entities as a map and exits.
for _, entity := range entities {
if entity["_created"] != nil {
entity["_created"] = entity.Created().Format(time.RFC3339Nano)
}
if entity["_updated"] != nil {
entity["_updated"] = entity.Updated().Format(time.RFC3339Nano)
}
}

if ctx.Options.JSON {
bytes, err := json.Marshal(entities)
if err != nil {
91 changes: 72 additions & 19 deletions etre.go
View file
Original file line number Diff line number Diff line change
@@ -11,15 +11,18 @@ import (
"path"
"runtime"
"sort"
"time"
)

const (
VERSION = "0.12.0"
API_ROOT string = "/api/v1"
META_LABEL_ID = "_id"
META_LABEL_TYPE = "_type"
META_LABEL_REV = "_rev"
CDC_WRITE_TIMEOUT int = 5 // seconds
VERSION = "0.12.0"
API_ROOT string = "/api/v1"
META_LABEL_ID = "_id"
META_LABEL_TYPE = "_type"
META_LABEL_REV = "_rev"
META_LABEL_CREATED = "_created"
META_LABEL_UPDATED = "_updated"
CDC_WRITE_TIMEOUT int = 5 // seconds

VERSION_HEADER = "X-Etre-Version"
TRACE_HEADER = "X-Etre-Trace"
@@ -56,34 +59,83 @@ func (e Entity) Id() string {
return e[META_LABEL_ID].(string)
}

// Created returns the entities creation time from the META_LABEL_CREATED label.
// If the label is not present, returns the zero value of time.Time.
func (e Entity) Created() time.Time {
v := e[META_LABEL_CREATED]
if v == nil {
// Allowed, since old entities might not have this field
return time.Time{}
}
nanos, err := toInt64(v)
if err != nil {
// Since they do have this field, it must be the right type
panic(fmt.Sprintf("entity %s has invalid _created data type: %T; expected int64 (or int/int32 before v0.11)",
e.Id(), v))
}
return time.Unix(0, nanos)
}

// Updated returns the entities last update time from the META_LABEL_UPDATED label.
// If the label is not present, returns the zero value of time.Time.
func (e Entity) Updated() time.Time {
v := e[META_LABEL_UPDATED]
if v == nil {
// Allowed, since old entities might not have this field
return time.Time{}
}
nanos, err := toInt64(v)
if err != nil {
// Since they do have this field, it must be the right type
panic(fmt.Sprintf("entity %s has invalid _updated data type: %T; expected int64 (or int/int32 before v0.11)",
e.Id(), v))
}
return time.Unix(0, nanos)
}

func (e Entity) Type() string {
return e[META_LABEL_TYPE].(string)
}

func (e Entity) Rev() int64 {
v := e[META_LABEL_REV]
rev, err := toInt64(v)
if err != nil {
panic(fmt.Sprintf("entity %s has invalid _rev data type: %T; expected int64 (or int/int32 before v0.11)",
e.Id(), v))
}
return rev
}

// Has returns true of the entity has the label, regardless of its value.
func (e Entity) Has(label string) bool {
_, ok := e[label]
return ok
}

func toInt64(v any) (int64, error) {
// See "Some other useful marshalling mappings are:" at https://pkg.go.dev/go.mongodb.org/mongo-driver/bson?tab=doc
// TL;DR: only int32 and int64 map 1:1 Go:BSON. Before v0.11, we used int
// but that is magical in BSON: "int marshals to a BSON int32 if the value
// is between math.MinInt32 and math.MaxInt32, inclusive, and a BSON int64
// otherwise." As of v0.11 _rev is int64 everywhere, but for backwards-compat
// we check for int and int32.
v := e[META_LABEL_REV]
//
// We also need to check for float64 and float32, since json.Unmarshal might use this type.
switch v.(type) {
case int64:
return v.(int64)
return v.(int64), nil
case int32:
return int64(v.(int32))
return int64(v.(int32)), nil
case int:
return int64(v.(int))
return int64(v.(int)), nil
case float64:
return int64(v.(float64)), nil
case float32:
return int64(v.(float32)), nil
default:
return 0, fmt.Errorf("invalid type %T for int64 conversion", v)
}
panic(fmt.Sprintf("entity %s has invalid _rev data type: %T; expected int64 (or int/int32 before v0.11)",
e.Id(), v))
}

// Has returns true of the entity has the label, regardless of its value.
func (e Entity) Has(label string) bool {
_, ok := e[label]
return ok
}

// A Set is a user-defined logical grouping of writes (insert, update, delete).
@@ -122,7 +174,8 @@ var metaLabels = map[string]bool{
"_setId": true,
"_setOp": true,
"_setSize": true,
"_ts": true,
"_created": true,
"_updated": true,
"_type": true,
}