Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: sourcenetwork/defradb
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 5d1691285d9739e99e2d5ffcac41968dbf8222d7
Choose a base ref
..
head repository: sourcenetwork/defradb
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: c6037996c55d3e6fb1ef1380d2ebdeab5716448a
Choose a head ref
Showing with 1,365 additions and 512 deletions.
  1. +1 −1 .github/workflows/lint.yml
  2. +1 −1 Makefile
  3. +28 −2 acp/acp_local_test.go
  4. +8 −0 acp/dpi.go
  5. +2 −0 acp/errors.go
  6. +71 −23 acp/source_hub_client.go
  7. +64 −10 client/document.go
  8. +24 −6 client/document_test.go
  9. +1 −1 client/errors.go
  10. +2 −0 client/normal_new.go
  11. +24 −0 client/normal_scalar.go
  12. +3 −0 client/normal_value.go
  13. +59 −0 client/normal_value_test.go
  14. +4 −0 client/normal_void.go
  15. +4 −1 cmd/genopenapi/main.go
  16. +3 −0 docs/data_format_changes/i3097-json-type-coercion.md
  17. +12 −12 go.mod
  18. +24 −28 go.sum
  19. +1 −2 http/client_collection.go
  20. +4 −1 http/handler_collection.go
  21. +4 −1 http/handler_store.go
  22. +55 −18 internal/connor/connor.go
  23. +25 −1 internal/connor/eq.go
  24. +42 −0 internal/core/encoding.go
  25. +13 −6 internal/core/key.go
  26. +3 −5 internal/db/db.go
  27. +1 −0 internal/db/errors.go
  28. +4 −1 internal/db/messages.go
  29. +117 −76 internal/db/p2p_replicator.go
  30. +5 −0 internal/planner/mapper/mapper.go
  31. +2 −2 internal/request/graphql/schema/collection.go
  32. +40 −39 internal/request/graphql/schema/types/scalars.go
  33. +31 −72 internal/request/graphql/schema/types/scalars_test.go
  34. +6 −9 net/server.go
  35. +99 −118 playground/package-lock.json
  36. +4 −4 playground/package.json
  37. +1 −2 tests/clients/cli/wrapper_collection.go
  38. +11 −7 tests/integration/acp/relationship/doc_actor/add/with_only_write_gql_test.go
  39. +39 −13 tests/integration/acp/relationship/doc_actor/add/with_only_write_test.go
  40. +268 −19 tests/integration/mutation/create/field_kinds/field_kind_json_test.go
  41. +36 −0 tests/integration/mutation/create/with_variables_test.go
  42. +1 −1 tests/integration/mutation/update/field_kinds/json_test.go
  43. +80 −0 tests/integration/net/simple/replicator/with_update_test.go
  44. +115 −10 tests/integration/query/simple/with_filter/with_eq_json_test.go
  45. +15 −10 tests/integration/query/simple/with_filter/with_in_json_test.go
  46. +4 −4 tests/integration/schema/updates/add/field/kind/json_test.go
  47. +4 −6 tools/configs/golangci.yaml
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
@@ -50,7 +50,7 @@ jobs:
# Required: the version of golangci-lint is required.
# Note: The version should not pick the patch version as the latest patch
# version is what will always be used.
version: v1.54
version: v1.61

# Optional: working directory, useful for monorepos or if we wanted to run this
# on a non-root directory.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -124,7 +124,7 @@ client\:add-schema:

.PHONY: deps\:lint-go
deps\:lint-go:
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61

.PHONY: deps\:lint-yaml
deps\:lint-yaml:
30 changes: 28 additions & 2 deletions acp/acp_local_test.go
Original file line number Diff line number Diff line change
@@ -482,7 +482,7 @@ func Test_LocalACP_InMemory_CheckDocAccess_TrueIfHaveAccessFalseIfNotErrorOtherw
policyID,
)

// Invalid empty arguments such that we can't check doc access.
// Invalid empty arguments such that we can't check doc access (read).
hasAccess, errCheckDocAccess := localACP.CheckDocAccess(
ctx,
ReadPermission,
@@ -495,6 +495,19 @@ func Test_LocalACP_InMemory_CheckDocAccess_TrueIfHaveAccessFalseIfNotErrorOtherw
require.ErrorIs(t, errCheckDocAccess, ErrFailedToVerifyDocAccessWithACP)
require.False(t, hasAccess)

// Invalid empty arguments such that we can't check doc access (write).
hasAccess, errCheckDocAccess = localACP.CheckDocAccess(
ctx,
WritePermission,
identity1.DID,
validPolicyID,
"",
"",
)
require.Error(t, errCheckDocAccess)
require.ErrorIs(t, errCheckDocAccess, ErrFailedToVerifyDocAccessWithACP)
require.False(t, hasAccess)

// Check document accesss for a document that does not exist.
hasAccess, errCheckDocAccess = localACP.CheckDocAccess(
ctx,
@@ -568,7 +581,7 @@ func Test_LocalACP_PersistentMemory_CheckDocAccess_TrueIfHaveAccessFalseIfNotErr
policyID,
)

// Invalid empty arguments such that we can't check doc access.
// Invalid empty arguments such that we can't check doc access (read).
hasAccess, errCheckDocAccess := localACP.CheckDocAccess(
ctx,
ReadPermission,
@@ -581,6 +594,19 @@ func Test_LocalACP_PersistentMemory_CheckDocAccess_TrueIfHaveAccessFalseIfNotErr
require.ErrorIs(t, errCheckDocAccess, ErrFailedToVerifyDocAccessWithACP)
require.False(t, hasAccess)

// Invalid empty arguments such that we can't check doc access (write).
hasAccess, errCheckDocAccess = localACP.CheckDocAccess(
ctx,
WritePermission,
identity1.DID,
validPolicyID,
"",
"",
)
require.Error(t, errCheckDocAccess)
require.ErrorIs(t, errCheckDocAccess, ErrFailedToVerifyDocAccessWithACP)
require.False(t, hasAccess)

// Check document accesss for a document that does not exist.
hasAccess, errCheckDocAccess = localACP.CheckDocAccess(
ctx,
8 changes: 8 additions & 0 deletions acp/dpi.go
Original file line number Diff line number Diff line change
@@ -22,6 +22,14 @@ const (
WritePermission
)

// permissionsThatImplyRead is a list of any permissions that if we have, we assume that the user can read.
// This is because for DefraDB's purposes if an identity has access to the write permission, then they don't
// need to explicitly have read permission inorder to read, we can just imply that they have read access.
var permissionsThatImplyRead = []DPIPermission{
ReadPermission,
WritePermission,
}

// List of all valid DPI permissions, the order of permissions in this list must match
// the above defined ordering such that iota matches the index position within the list.
var dpiRequiredPermissions = []string{
2 changes: 2 additions & 0 deletions acp/errors.go
Original file line number Diff line number Diff line change
@@ -129,6 +129,7 @@ func NewErrFailedToCheckIfDocIsRegisteredWithACP(
func NewErrFailedToVerifyDocAccessWithACP(
inner error,
Type string,
permission string,
policyID string,
actorID string,
resourceName string,
@@ -138,6 +139,7 @@ func NewErrFailedToVerifyDocAccessWithACP(
errFailedToVerifyDocAccessWithACP,
inner,
errors.NewKV("Type", Type),
errors.NewKV("Permission", permission),
errors.NewKV("PolicyID", policyID),
errors.NewKV("ActorID", actorID),
errors.NewKV("ResourceName", resourceName),
94 changes: 71 additions & 23 deletions acp/source_hub_client.go
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ package acp

import (
"context"
"strconv"

protoTypes "github.com/cosmos/gogoproto/types"
"github.com/sourcenetwork/corelog"
@@ -342,39 +343,86 @@ func (a *sourceHubBridge) CheckDocAccess(
resourceName string,
docID string,
) (bool, error) {
isValid, err := a.client.VerifyAccessRequest(
// We grant "read" access even if the identity does not explicitly have the "read" permission,
// as long as they have any of the permissions that imply read access.
if permission == ReadPermission {
var canRead bool = false
var withPermission string
var err error

for _, permissionThatImpliesRead := range permissionsThatImplyRead {
canRead, err = a.client.VerifyAccessRequest(
ctx,
permissionThatImpliesRead,
actorID,
policyID,
resourceName,
docID,
)

if err != nil {
return false, NewErrFailedToVerifyDocAccessWithACP(
err,
"Local",
permissionThatImpliesRead.String(),
policyID,
actorID,
resourceName,
docID,
)
}

if canRead {
withPermission = permissionThatImpliesRead.String()
break
}
}

log.InfoContext(
ctx,
"Document readable="+strconv.FormatBool(canRead),
corelog.Any("Permission", withPermission),
corelog.Any("PolicyID", policyID),
corelog.Any("Resource", resourceName),
corelog.Any("ActorID", actorID),
corelog.Any("DocID", docID),
)

return canRead, nil
}

hasAccess, err := a.client.VerifyAccessRequest(
ctx,
permission,
actorID,
policyID,
resourceName,
docID,
)
if err != nil {
return false, NewErrFailedToVerifyDocAccessWithACP(err, "Local", policyID, actorID, resourceName, docID)
}

if isValid {
log.InfoContext(
ctx,
"Document accessible",
corelog.Any("PolicyID", policyID),
corelog.Any("ActorID", actorID),
corelog.Any("Resource", resourceName),
corelog.Any("DocID", docID),
)
return true, nil
} else {
log.InfoContext(
ctx,
"Document inaccessible",
corelog.Any("PolicyID", policyID),
corelog.Any("ActorID", actorID),
corelog.Any("Resource", resourceName),
corelog.Any("DocID", docID),
if err != nil {
return false, NewErrFailedToVerifyDocAccessWithACP(
err,
"Local",
permission.String(),
policyID,
actorID,
resourceName,
docID,
)
return false, nil
}

log.InfoContext(
ctx,
"Document accessible="+strconv.FormatBool(hasAccess),
corelog.Any("Permission", permission),
corelog.Any("PolicyID", policyID),
corelog.Any("Resource", resourceName),
corelog.Any("ActorID", actorID),
corelog.Any("DocID", docID),
)

return hasAccess, nil
}

func (a *sourceHubBridge) AddDocActorRelationship(
74 changes: 64 additions & 10 deletions client/document.go
Original file line number Diff line number Diff line change
@@ -341,7 +341,7 @@ func validateFieldSchema(val any, field FieldDefinition) (NormalValue, error) {
if err != nil {
return nil, err
}
return NewNormalString(v), nil
return NewNormalJSON(&JSON{v}), nil
}

return nil, NewErrUnhandledType("FieldKind", field.Kind)
@@ -417,16 +417,70 @@ func getDateTime(v any) (time.Time, error) {
return time.Parse(time.RFC3339, s)
}

func getJSON(v any) (string, error) {
s, err := getString(v)
if err != nil {
return "", err
}
val, err := fastjson.Parse(s)
if err != nil {
return "", NewErrInvalidJSONPaylaod(s)
// getJSON converts the given value to a valid JSON value.
//
// If the value is of type *fastjson.Value it needs to be
// manually parsed. All other values are valid JSON.
func getJSON(v any) (any, error) {
val, ok := v.(*fastjson.Value)
if !ok {
return v, nil
}
switch val.Type() {
case fastjson.TypeArray:
arr, err := val.Array()
if err != nil {
return nil, err
}
out := make([]any, len(arr))
for i, v := range arr {
c, err := getJSON(v)
if err != nil {
return nil, err
}
out[i] = c
}
return out, nil

case fastjson.TypeObject:
obj, err := val.Object()
if err != nil {
return nil, err
}
out := make(map[string]any)
obj.Visit(func(key []byte, v *fastjson.Value) {
c, e := getJSON(v)
out[string(key)] = c
err = errors.Join(err, e)
})
return out, err

case fastjson.TypeFalse:
return false, nil

case fastjson.TypeTrue:
return true, nil

case fastjson.TypeNumber:
out, err := val.Int64()
if err == nil {
return out, nil
}
return val.Float64()

case fastjson.TypeString:
out, err := val.StringBytes()
if err != nil {
return nil, err
}
return string(out), nil

case fastjson.TypeNull:
return nil, nil

default:
return nil, NewErrInvalidJSONPayload(v)
}
return val.String(), nil
}

func getArray[T any](
30 changes: 24 additions & 6 deletions client/document_test.go
Original file line number Diff line number Diff line change
@@ -161,7 +161,16 @@ func TestNewFromJSON_WithValidJSONFieldValue_NoError(t *testing.T) {
objWithJSONField := []byte(`{
"Name": "John",
"Age": 26,
"Custom": "{\"tree\":\"maple\", \"age\": 260}"
"Custom": {
"string": "maple",
"int": 260,
"float": 3.14,
"false": false,
"true": true,
"null": null,
"array": ["one", 1],
"object": {"one": 1}
}
}`)
doc, err := NewDocFromJSON(objWithJSONField, def)
if err != nil {
@@ -183,28 +192,37 @@ func TestNewFromJSON_WithValidJSONFieldValue_NoError(t *testing.T) {
assert.Equal(t, doc.values[doc.fields["Name"]].IsDocument(), false)
assert.Equal(t, doc.values[doc.fields["Age"]].Value(), int64(26))
assert.Equal(t, doc.values[doc.fields["Age"]].IsDocument(), false)
assert.Equal(t, doc.values[doc.fields["Custom"]].Value(), "{\"tree\":\"maple\",\"age\":260}")
assert.Equal(t, doc.values[doc.fields["Custom"]].Value(), map[string]any{
"string": "maple",
"int": int64(260),
"float": float64(3.14),
"false": false,
"true": true,
"null": nil,
"array": []any{"one", int64(1)},
"object": map[string]any{"one": int64(1)},
})
assert.Equal(t, doc.values[doc.fields["Custom"]].IsDocument(), false)
}

func TestNewFromJSON_WithInvalidJSONFieldValue_Error(t *testing.T) {
objWithJSONField := []byte(`{
"Name": "John",
"Age": 26,
"Custom": "{\"tree\":\"maple, \"age\": 260}"
"Custom": {"tree":"maple, "age": 260}
}`)
_, err := NewDocFromJSON(objWithJSONField, def)
require.ErrorContains(t, err, "invalid JSON payload. Payload: {\"tree\":\"maple, \"age\": 260}")
require.ErrorContains(t, err, "cannot parse JSON")
}

func TestNewFromJSON_WithInvalidJSONFieldValueSimpleString_Error(t *testing.T) {
func TestNewFromJSON_WithJSONFieldValueSimpleString_Succeed(t *testing.T) {
objWithJSONField := []byte(`{
"Name": "John",
"Age": 26,
"Custom": "blah"
}`)
_, err := NewDocFromJSON(objWithJSONField, def)
require.ErrorContains(t, err, "invalid JSON payload. Payload: blah")
require.NoError(t, err)
}

func TestIsJSONArray(t *testing.T) {
2 changes: 1 addition & 1 deletion client/errors.go
Original file line number Diff line number Diff line change
@@ -166,7 +166,7 @@ func NewErrCRDTKindMismatch(cType, kind string) error {
return errors.New(fmt.Sprintf(errCRDTKindMismatch, cType, kind))
}

func NewErrInvalidJSONPaylaod(payload string) error {
func NewErrInvalidJSONPayload(payload any) error {
return errors.New(errInvalidJSONPayload, errors.NewKV("Payload", payload))
}

2 changes: 2 additions & 0 deletions client/normal_new.go
Original file line number Diff line number Diff line change
@@ -64,6 +64,8 @@ func NewNormalValue(val any) (NormalValue, error) {
return NewNormalTime(v), nil
case *Document:
return NewNormalDocument(v), nil
case *JSON:
return NewNormalJSON(v), nil

case immutable.Option[bool]:
return NewNormalNillableBool(v), nil
Loading