Skip to content

Commit 370663e

Browse files
committed
🌅
0 parents  commit 370663e

File tree

8 files changed

+487
-0
lines changed

8 files changed

+487
-0
lines changed

.editorconfig

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# http://editorconfig.org
2+
root = true
3+
4+
[*]
5+
indent_style = space
6+
indent_size = 4
7+
end_of_line = lf
8+
charset = utf-8
9+
trim_trailing_whitespace = true
10+
insert_final_newline = true
11+
12+
[*.md]
13+
trim_trailing_whitespace = false

.gitignore

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#ignore these from source control
2+
_ReSharper*/
3+
_tmp/
4+
[Tt]est[Rr]esult*/
5+
[Pp]ackages/
6+
.paket/
7+
AutoTest.Net*/
8+
node_modules/
9+
.tmp/
10+
.sass-cache/
11+
bower_components/
12+
dist/
13+
lib/
14+
*.log
15+
paket-files/
16+
report/
17+
.vs/
18+
.fable/
19+
.idea/
20+
*.orig
21+
*.DotSettings.user
22+
.ionide/
23+
.fake/
24+
[Tt]humbs.db
25+
*.obj
26+
*.exe
27+
*.pdb
28+
*.user
29+
*.aps
30+
*.pch
31+
*.vspscc
32+
*_i.c
33+
*_p.c
34+
*.ncb
35+
*.jfm
36+
*.suo
37+
*.tlb
38+
*.tlh
39+
*.bak
40+
*.cache
41+
*.ilk
42+
*.log
43+
[Bb]in
44+
[Dd]ebug*/
45+
*.lib
46+
*.sbr
47+
obj/
48+
[Rr]elease*/
49+
.idea/
50+
temp/
51+
*.jfm
52+
database/database.dbmdl
53+
/.vscode/settings.json
54+
tests/Benchmarks/BenchmarkDotNet.Artifacts/*
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.8.34309.116
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.JsonProvider.Serializer", "src\FSharp.Data.JsonProvider.Serializer.fsproj", "{D8750E90-6B62-4FB9-8452-FD133F50B013}"
7+
EndProject
8+
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "BenchmarkTests", "tests\Benchmarks\BenchmarkTests.fsproj", "{929730F2-89FD-4514-9A02-81319C4E9628}"
9+
EndProject
10+
Global
11+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
12+
Debug|Any CPU = Debug|Any CPU
13+
Release|Any CPU = Release|Any CPU
14+
EndGlobalSection
15+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
16+
{D8750E90-6B62-4FB9-8452-FD133F50B013}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
17+
{D8750E90-6B62-4FB9-8452-FD133F50B013}.Debug|Any CPU.Build.0 = Debug|Any CPU
18+
{D8750E90-6B62-4FB9-8452-FD133F50B013}.Release|Any CPU.ActiveCfg = Release|Any CPU
19+
{D8750E90-6B62-4FB9-8452-FD133F50B013}.Release|Any CPU.Build.0 = Release|Any CPU
20+
{929730F2-89FD-4514-9A02-81319C4E9628}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
21+
{929730F2-89FD-4514-9A02-81319C4E9628}.Debug|Any CPU.Build.0 = Debug|Any CPU
22+
{929730F2-89FD-4514-9A02-81319C4E9628}.Release|Any CPU.ActiveCfg = Release|Any CPU
23+
{929730F2-89FD-4514-9A02-81319C4E9628}.Release|Any CPU.Build.0 = Release|Any CPU
24+
EndGlobalSection
25+
GlobalSection(SolutionProperties) = preSolution
26+
HideSolutionNode = FALSE
27+
EndGlobalSection
28+
GlobalSection(ExtensibilityGlobals) = postSolution
29+
SolutionGuid = {E7A7708B-CD79-421F-B890-3F224B41C646}
30+
EndGlobalSection
31+
EndGlobal

Readme.md

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
2+
# FSharp.Data.JsonProvider.Serializer
3+
4+
NuGet package: https://www.nuget.org/packages/FSharp.Data.JsonProvider.Serializer/
5+
6+
This will provide utilities to use the fast `System.Text.Json` library to serialize the `FSharp.Data.JsonProvider`` items.
7+
8+
- FSharp.Data: https://fsprojects.github.io/FSharp.Data/
9+
- System.Text.Json: https://www.nuget.org/packages/System.Text.Json
10+
11+
Motivation: Serialization speed.
12+
Typically JSON Serialization is used either so that the user is watching a progress-bar, or in a big batch-process.
13+
14+
Current FSharp.Data is using custom Json-serializer.
15+
16+
We need a compromise having the convinience of F# JsonProvider, but speed of System.Text.Json.
17+
18+
The idea is to be in-replacement for current functions:
19+
20+
## Reading values to type provider (Deserialization)
21+
22+
Current FSharp.Data.JsonProvider:
23+
24+
```fsharp
25+
type MyJsonType = FSharp.Data.JsonProvider<"""{ "model": "..." } """>
26+
27+
let fromJson (response:string) = MyJsonType.Parse response
28+
29+
```
30+
31+
Using this library:
32+
33+
```fsharp
34+
type MyJsonType = FSharp.Data.JsonProvider<"""{ "model": "..." } """>
35+
36+
let fromJson (response:string) = MyJsonType.Load (Serializer.Deserialize response)
37+
38+
```
39+
40+
41+
## Saving values from type provider (Serialization)
42+
43+
Current FSharp.Data.JsonProvider:
44+
45+
```fsharp
46+
type MyJson = FSharp.Data.JsonProvider<"""{ "model": "..." } """>
47+
48+
let toJson mymodel = mymodel.JsonValue.ToString()
49+
50+
```
51+
52+
Using this library:
53+
54+
```fsharp
55+
type MyJson = FSharp.Data.JsonProvider<"""{ "model": "..." } """>
56+
57+
let toJson mymodel = Serializer.Serialize (mymodel.JsonValue)
58+
59+
```
60+
61+
Besides of this, you continue using your existing JsonProvider implementation as is.
62+
63+
## Initial Benchmarks
64+
65+
Test-case:
66+
- Read JSON to JsonProvider. (Serialization)
67+
- Verify a property
68+
- Save JSON back to a string. (Deserialization)
69+
70+
Tested with small JSON file, and with Stripe OpenAPI spec file (+5MB of JSON).
71+
72+
BenchmarkDotNet v0.13.10, Windows 11 (10.0.22621.2715/22H2/2022Update/SunValley2)
73+
13th Gen Intel Core i9-13900H, 1 CPU, 20 logical and 14 physical cores
74+
75+
### .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2 DEBUG
76+
77+
| Method | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated |
78+
|---------------------------------------- |---------------:|--------------:|--------------:|---------------:|----------:|----------:|----------:|------------:|
79+
| Serialization_SmallJson_JsonProvider | 7.050 us | 0.2193 us | 0.6465 us | 6.616 us | 0.8545 | - | - | 5.32 KB |
80+
| Serialization_SmallJson_SystemTextJson | 5.000 us | 0.0416 us | 0.0369 us | 4.998 us | 0.5798 | - | - | 3.59 KB |
81+
| Serialization_StripeJson_JsonProvider | 110,650.331 us | 1,939.4994 us | 1,619.5699 us | 111,154.500 us | 8200.0000 | 3800.0000 | 1400.0000 | 61615.76 KB |
82+
| Serialization_StripeJson_SystemTextJson | 79,382.557 us | 1,570.6070 us | 1,469.1468 us | 79,597.929 us | 6857.1429 | 2857.1429 | 1000.0000 | 61057.1 KB |
83+
84+
### .NET Framework 4.8 : .NET Framework 4.8.1 (4.8.9181.0), X64 RyuJIT VectorSize=256
85+
86+
| Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated |
87+
|---------------------------------------- |---------------:|--------------:|--------------:|----------:|----------:|----------:|------------:|
88+
| Serialization_SmallJson_JsonProvider | 7.132 us | 0.0773 us | 0.0685 us | 0.8621 | - | - | 5.32 KB |
89+
| Serialization_SmallJson_SystemTextJson | 4.938 us | 0.0664 us | 0.0621 us | 0.5798 | - | - | 3.59 KB |
90+
| Serialization_StripeJson_JsonProvider | 107,985.207 us | 1,474.9305 us | 1,151.5288 us | 8200.0000 | 3800.0000 | 1400.0000 | 61615.82 KB |
91+
| Serialization_StripeJson_SystemTextJson | 81,017.954 us | 1,565.8860 us | 2,295.2554 us | 6857.1429 | 2857.1429 | 1000.0000 | 61057.03 KB |
92+
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFrameworks>net48;netstandard2.0;netstandard2.1;net8.0</TargetFrameworks>
5+
<GenerateDocumentationFile>true</GenerateDocumentationFile>
6+
<PackageId>FSharp.Data.JsonProvider.Serializer</PackageId>
7+
<AssemblyName>FSharp.Data.JsonProvider.Serializer</AssemblyName>
8+
<Version>0.0.1</Version>
9+
<Authors>Tuomas</Authors>
10+
<PackageTags>FSharp Data JsonProvider System Text Json serialization serialisation deserialization deserialisation</PackageTags>
11+
<Description>
12+
Replace FSharp.Data.JsonProvider default serialization with System.Text.Json.
13+
</Description>
14+
<RepositoryUrl>https://github.com/Thorium/FSharp.Data.JsonProvider.Serializer.git</RepositoryUrl>
15+
<RepositoryType>git</RepositoryType>
16+
<PackageLicenseExpression>Unlicense</PackageLicenseExpression>
17+
<IsPackable>true</IsPackable>
18+
<IsTestProject>false</IsTestProject>
19+
<PublishRepositoryUrl>true</PublishRepositoryUrl>
20+
<EmbedUntrackedSources>true</EmbedUntrackedSources>
21+
</PropertyGroup>
22+
23+
<ItemGroup>
24+
<Compile Include="Serializer.fs" />
25+
</ItemGroup>
26+
27+
<ItemGroup>
28+
<PackageReference Include="FSharp.Data.Json.Core" Version="6.3.0" />
29+
<PackageReference Include="System.Text.Json" Version="8.0.0" />
30+
</ItemGroup>
31+
32+
</Project>

src/Serializer.fs

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
namespace FSharp.Data.JsonProvider
2+
3+
open System.Runtime.CompilerServices
4+
open System
5+
open System.Text
6+
open System.Text.Json
7+
open FSharp.Data
8+
open System.IO
9+
10+
module internal SerializationFunctions =
11+
12+
let defaultoptions =
13+
JsonReaderOptions(
14+
AllowTrailingCommas = true,
15+
CommentHandling = JsonCommentHandling.Skip)
16+
17+
let rec constructArray (reader: byref<Utf8JsonReader>) (built:JsonValue[]) =
18+
if not (reader.Read()) then built
19+
else
20+
match reader.TokenType with
21+
| JsonTokenType.EndArray ->
22+
built
23+
| _ ->
24+
match handleToken &reader with
25+
| ValueSome thisToken ->
26+
let nextGen = Array.append built [|thisToken|]
27+
constructArray &reader nextGen
28+
| ValueNone -> built
29+
30+
and constructRecord (reader: byref<Utf8JsonReader>) (nextName:string) (built:(string*JsonValue)[]) =
31+
if not (reader.Read()) then built
32+
else
33+
match reader.TokenType with
34+
| JsonTokenType.EndObject ->
35+
built
36+
| JsonTokenType.PropertyName ->
37+
let name = reader.GetString()
38+
// todo, combine name and value
39+
constructRecord &reader name built
40+
| _ ->
41+
match handleToken &reader with
42+
| ValueSome thisToken ->
43+
44+
let nextGen = Array.append built [|nextName,thisToken|]
45+
constructRecord &reader nextName nextGen
46+
| ValueNone -> built
47+
48+
and handleToken (reader: byref<Utf8JsonReader>) =
49+
match reader.TokenType with
50+
| JsonTokenType.Number ->
51+
match reader.TryGetDecimal() with
52+
| true, deci -> ValueSome (JsonValue.Number (reader.GetDecimal()))
53+
| false, floa -> ValueSome (JsonValue.Float (reader.GetDouble()))
54+
| JsonTokenType.Null -> ValueSome JsonValue.Null
55+
| JsonTokenType.String ->
56+
ValueSome (JsonValue.String (reader.GetString()))
57+
| JsonTokenType.Comment
58+
| JsonTokenType.None -> ValueNone
59+
| JsonTokenType.True -> ValueSome (JsonValue.Boolean true)
60+
| JsonTokenType.False -> ValueSome (JsonValue.Boolean false)
61+
| JsonTokenType.StartArray ->
62+
let array = constructArray &reader [||]
63+
ValueSome (JsonValue.Array array)
64+
| JsonTokenType.StartObject ->
65+
let record = constructRecord &reader "" [||]
66+
ValueSome (JsonValue.Record record)
67+
68+
69+
| x -> failwithf $"Unexpected json: {x}" // Other token types elided for brevity
70+
71+
72+
let read (contentBytes:ReadOnlySpan<byte>) (options:JsonReaderOptions) =
73+
let mutable fullreader = Utf8JsonReader(contentBytes, options)
74+
let nextToken = fullreader.Read()
75+
if not nextToken then JsonValue.Null
76+
else
77+
78+
match handleToken &fullreader with
79+
| ValueSome token -> token
80+
| ValueNone -> JsonValue.Null
81+
82+
83+
let write (item:JsonValue) (options:JsonWriterOptions voption) =
84+
use stream = new MemoryStream();
85+
use writer =
86+
match options with
87+
| ValueSome opt -> new Utf8JsonWriter(stream, opt)
88+
| ValueNone -> new Utf8JsonWriter(stream)
89+
90+
let rec write' content =
91+
match content with
92+
| JsonValue.Null -> writer.WriteNullValue()
93+
| JsonValue.Boolean b -> writer.WriteBooleanValue b
94+
| JsonValue.String s -> writer.WriteStringValue s
95+
| JsonValue.Float f -> writer.WriteNumberValue f
96+
| JsonValue.Number n -> writer.WriteNumberValue n
97+
| JsonValue.Array a ->
98+
writer.WriteStartArray()
99+
for itm in a do
100+
write' itm
101+
writer.WriteEndArray()
102+
| JsonValue.Record r ->
103+
writer.WriteStartObject()
104+
for (name,value) in r do
105+
writer.WritePropertyName name
106+
write' value
107+
writer.WriteEndObject()
108+
write' item
109+
writer.Flush();
110+
stream.ToArray()
111+
112+
module Serializer =
113+
114+
/// Deserialize UTF8 stream to FSharp.Data.JsonValue using System.Text.Json
115+
[<Extension>]
116+
let DeserializeBytes (jsonUtf8Bytes:ReadOnlySpan<byte>) =
117+
SerializationFunctions.read jsonUtf8Bytes SerializationFunctions.defaultoptions
118+
119+
/// Deserialize UTF8 stream to FSharp.Data.JsonValue using System.Text.Json
120+
[<Extension>]
121+
let DeserializeBytesWith (jsonUtf8Bytes:ReadOnlySpan<byte>, options:JsonReaderOptions) =
122+
SerializationFunctions.read jsonUtf8Bytes options
123+
124+
/// Deserialize string to FSharp.Data.JsonValue using System.Text.Json
125+
[<Extension>]
126+
let Deserialize (item:string) =
127+
SerializationFunctions.read (System.ReadOnlySpan (Encoding.UTF8.GetBytes item)) SerializationFunctions.defaultoptions
128+
129+
/// Deserialize string to FSharp.Data.JsonValue using System.Text.Json
130+
[<Extension>]
131+
let DeserializeWith (item:string, options:JsonReaderOptions) =
132+
SerializationFunctions.read (System.ReadOnlySpan (Encoding.UTF8.GetBytes item)) options
133+
134+
135+
/// Serialize FSharp.Data.JsonValue to byte array using System.Text.Json
136+
[<Extension>]
137+
let SerializeBytes (item:JsonValue) =
138+
SerializationFunctions.write item ValueNone
139+
140+
/// Serialize FSharp.Data.JsonValue to byte array using System.Text.Json
141+
[<Extension>]
142+
let SerializeBytesWith (item:JsonValue, options:JsonWriterOptions) =
143+
SerializationFunctions.write item (ValueSome options)
144+
145+
/// Serialize FSharp.Data.JsonValue to string using System.Text.Json
146+
[<Extension>]
147+
let Serialize (item:JsonValue) =
148+
Encoding.UTF8.GetString(SerializationFunctions.write item ValueNone)
149+
150+
/// Serialize FSharp.Data.JsonValue to string using System.Text.Json
151+
[<Extension>]
152+
let SerializeWith (item:JsonValue, options:JsonWriterOptions) =
153+
Encoding.UTF8.GetString(SerializationFunctions.write item (ValueSome options))
154+

0 commit comments

Comments
 (0)