Setup project and implement first draft of audit file reader

This commit is contained in:
Peter 2021-02-15 08:46:19 +01:00
parent e6959915c6
commit 3fc1eaf0df
Signed by: prskr
GPG key ID: C1DB5D2E8DB512F9
18 changed files with 639 additions and 0 deletions

10
.editorconfig Normal file
View file

@ -0,0 +1,10 @@
root = true
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.cs]
indent_style = space
indent_size = 4

6
.gitignore vendored Normal file
View file

@ -0,0 +1,6 @@
bin/
obj/
.idea/
.vscode/
INetMock.sln.DotSettings.user

36
INetMock.sln Normal file
View file

@ -0,0 +1,36 @@

Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "INetMock.Client", "src\INetMock.Client\INetMock.Client.csproj", "{4B1CFFB0-23B5-4238-BFCC-4155F459FF8E}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionItems", "SolutionItems", "{DF7A15B0-72D0-4BA1-9C9E-424343F6D530}"
ProjectSection(SolutionItems) = preProject
.editorconfig = .editorconfig
.gitignore = .gitignore
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{07100561-E3C0-4B95-92E1-D2D3BA12C3A6}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{E7144CE4-3E55-431B-8835-E804C664A694}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "INetMock.Client.Test", "test\INetMock.Client.Test\INetMock.Client.Test.csproj", "{BFC06300-C82F-4A24-A10E-5C7D66208280}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{4B1CFFB0-23B5-4238-BFCC-4155F459FF8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4B1CFFB0-23B5-4238-BFCC-4155F459FF8E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4B1CFFB0-23B5-4238-BFCC-4155F459FF8E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4B1CFFB0-23B5-4238-BFCC-4155F459FF8E}.Release|Any CPU.Build.0 = Release|Any CPU
{BFC06300-C82F-4A24-A10E-5C7D66208280}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BFC06300-C82F-4A24-A10E-5C7D66208280}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BFC06300-C82F-4A24-A10E-5C7D66208280}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BFC06300-C82F-4A24-A10E-5C7D66208280}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{4B1CFFB0-23B5-4238-BFCC-4155F459FF8E} = {07100561-E3C0-4B95-92E1-D2D3BA12C3A6}
{BFC06300-C82F-4A24-A10E-5C7D66208280} = {E7144CE4-3E55-431B-8835-E804C664A694}
EndGlobalSection
EndGlobal

View file

@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using Google.Protobuf.WellKnownTypes;
using INetMock.Client.Audit.Details;
namespace INetMock.Client.Audit
{
public abstract record EventDetails;
public record EmptyDetails : EventDetails;
public record GenericDetails : EventDetails
{
private readonly Any? _detailsAny;
public GenericDetails()
{
_detailsAny = null;
}
public GenericDetails(Any? any)
{
_detailsAny = any;
}
public static implicit operator HttpDetails(GenericDetails gd)
{
if (gd._detailsAny == null || gd._detailsAny.Value == null) return new();
//if(!gd._detailsAny.TypeUrl.EndsWith(HTTPDetailsEntity.Descriptor.FullName)) throw new InvalidOperationException();
return new HttpDetails(gd._detailsAny);
}
public static implicit operator DnsDetails(GenericDetails gd)
{
if (gd._detailsAny == null || gd._detailsAny.Value == null) return new();
//if(!gd._detailsAny.TypeUrl.EndsWith(HTTPDetailsEntity.Descriptor.FullName)) throw new InvalidOperationException();
return new DnsDetails(gd._detailsAny);
}
}
public record HttpDetails : EventDetails
{
public HttpDetails()
{
}
public HttpDetails(Any? any)
{
if (any == null || any.Value == null) return;
var detailsEntity = HTTPDetailsEntity.Parser.ParseFrom(any.Value);
Method = new HttpMethod(detailsEntity.Method.ToString());
Host = detailsEntity.Host;
Uri = detailsEntity.Uri;
Proto = detailsEntity.Proto;
Headers = new INetMockHttpHeaders(detailsEntity.Headers);
}
public HttpMethod Method { get; init; }
public string Host { get; init; }
public string Uri { get; init; }
public string Proto { get; init; }
public HttpHeaders Headers { get; init; }
}
public record DnsDetails : EventDetails
{
public DnsDetails()
{
}
public DNSOpCode OpCode { get; init; }
public IReadOnlyList<DNSQuestionEntity> Questions { get; init; }
public DnsDetails(Any? any)
{
if(any == null || any.Value == null) return;
var entity = DNSDetailsEntity.Parser.ParseFrom(any.Value);
OpCode = entity.Opcode;
Questions = entity.Questions;
}
}
}

View file

@ -0,0 +1,91 @@
using System;
using System.Net;
namespace INetMock.Client.Audit
{
public record Event<T> where T : EventDetails, new()
{
public Event()
{
}
public Event(EventEntity entity)
{
if (entity == null) throw new ArgumentNullException(nameof(entity));
var details = (new T(), entity.Application) switch
{
(GenericDetails, _) => new GenericDetails(entity.ProtocolDetails) as T,
(_, AppProtocol.Dns) => new DnsDetails(entity.ProtocolDetails) as T,
(_, AppProtocol.Http or AppProtocol.HttpProxy) => new HttpDetails(entity.ProtocolDetails) as T,
(_, _) => new EmptyDetails() as T
};
Id = entity.Id;
Timestamp = entity.Timestamp.ToDateTimeOffset();
Transport = entity.Transport;
SourceIp = new IPAddress(entity.SourceIP.Span);
SourcePort = Convert.ToUInt16(entity.SourcePort);
DestinationIp = new IPAddress(entity.DestinationIP.Span);
DestinationPort = Convert.ToUInt16(entity.DestinationPort);
TlsDetails = entity.Tls;
Application = entity.Application;
Details = details;
}
public long Id { get; init; }
public DateTimeOffset Timestamp { get; init; }
public TransportProtocol Transport { get; init; }
public AppProtocol Application { get; init; }
public IPAddress SourceIp { get; init; }
public IPAddress DestinationIp { get; init; }
public ushort SourcePort { get; init; }
public ushort DestinationPort { get; init; }
public TLSDetailsEntity TlsDetails { get; init; }
public T? Details { get; init; }
public bool CanConvert<TTarget>() where TTarget : EventDetails, new()
{
if (Details == null) return false;
return (new TTarget(), Application) switch
{
(DnsDetails, AppProtocol.Dns) => true,
(HttpDetails, AppProtocol.Http or AppProtocol.HttpProxy) => true,
(_, _) => false
};
}
public static explicit operator Event<T>?(Event<GenericDetails> ge)
{
T? details = null;
if (ge.Details != null)
{
details = ge.Application switch
{
AppProtocol.Dns => (DnsDetails) ge.Details as T,
AppProtocol.Http or AppProtocol.HttpProxy => (HttpDetails) ge.Details as T,
_ => new EmptyDetails() as T
};
}
if (details == null) return null;
return new()
{
Id = ge.Id,
Timestamp = ge.Timestamp,
Transport = ge.Transport,
SourceIp = ge.SourceIp,
SourcePort = ge.SourcePort,
DestinationIp = ge.DestinationIp,
DestinationPort = ge.DestinationPort,
TlsDetails = ge.TlsDetails,
Application = ge.Application,
Details = details
};
}
}
}

View file

@ -0,0 +1,28 @@
using System.Threading;
using System.Threading.Tasks;
using Grpc.Core;
namespace INetMock.Client.Audit
{
public sealed class EventServerStreamReader : IProtoEventReader
{
private readonly IAsyncStreamReader<EventEntity> _asyncEventStream;
public EventServerStreamReader(IAsyncStreamReader<EventEntity> asyncEventStream)
{
_asyncEventStream = asyncEventStream;
}
public async Task<EventEntity?> ReadAsync(CancellationToken token = default)
{
if (!await _asyncEventStream.MoveNext(token)) return null;
return _asyncEventStream.Current;
}
public void Dispose()
{
}
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}

View file

@ -0,0 +1,45 @@
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace INetMock.Client.Audit
{
public sealed class GenericReader : IEventReader<GenericDetails>
{
private readonly IProtoEventReader _reader;
public GenericReader(IProtoEventReader reader)
{
_reader = reader;
}
public async IAsyncEnumerable<Event<GenericDetails>> ReadAllAsync([EnumeratorCancellation] CancellationToken token = default)
{
while (true)
{
var ev = await ReadAsync(token);
if (ev == null)
{
yield break;
}
yield return ev;
}
}
public async Task<Event<GenericDetails>?> ReadAsync(CancellationToken token = default)
{
var entity = await _reader.ReadAsync(token);
if (entity == null) return null;
return new Event<GenericDetails>(entity);
}
public ValueTask DisposeAsync() => _reader.DisposeAsync();
public void Dispose()
{
_reader.Dispose();
}
}
}

View file

@ -0,0 +1,22 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace INetMock.Client.Audit
{
public interface IEventReader<T>: IDisposable, IAsyncDisposable where T : EventDetails, new()
{
IAsyncEnumerable<Event<T>> ReadAllAsync(CancellationToken token = default);
/// <summary>
///
/// </summary>
/// <param name="token"></param>
/// <returns>
/// An event as long as underlying stream has data.
/// When the end of the stream has been reached it will return null.
/// </returns>
Task<Event<T>?> ReadAsync(CancellationToken token = default);
}
}

View file

@ -0,0 +1,18 @@
using System.Net.Http.Headers;
using Google.Protobuf.Collections;
using INetMock.Client.Audit.Details;
namespace INetMock.Client.Audit
{
internal class INetMockHttpHeaders : HttpHeaders
{
internal INetMockHttpHeaders(MapField<string, HTTPHeaderValue> headers)
{
foreach (var (key, values) in headers)
{
if (string.IsNullOrEmpty(key) || values == null) continue;
Add(key, values.Values);
}
}
}
}

View file

@ -0,0 +1,11 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace INetMock.Client.Audit
{
public interface IProtoEventReader : IDisposable, IAsyncDisposable
{
Task<EventEntity?> ReadAsync(CancellationToken token = default);
}
}

View file

@ -0,0 +1,68 @@
using System.Buffers;
using System.Buffers.Binary;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Google.Protobuf;
namespace INetMock.Client.Audit
{
public sealed class ProtoReader : IProtoEventReader
{
private readonly MemoryPool<byte> _memoryPool;
private readonly Stream _sourceStream;
private readonly bool _keepStreamOpen;
public ProtoReader(Stream sourceStream, bool keepStreamOpen = false)
{
_memoryPool = MemoryPool<byte>.Shared;
_sourceStream = sourceStream;
_keepStreamOpen = keepStreamOpen;
}
public async Task<EventEntity?> ReadAsync(CancellationToken token = default)
{
using var rentedLengthMem = _memoryPool.Rent(4);
var lengthMem = rentedLengthMem.Memory.Slice(0, 4);
var read = await _sourceStream.ReadAsync(lengthMem, token);
if (read != 4)
{
return null;
}
var messageLength = BinaryPrimitives.ReadInt32BigEndian(lengthMem.Span);
using var rentedMsgMem = _memoryPool.Rent(messageLength);
var msgMem = rentedMsgMem.Memory.Slice(0, messageLength);
read = await _sourceStream.ReadAsync(msgMem, token);
if (read != messageLength)
{
return null;
}
var entity = new EventEntity();
entity.MergeFrom(msgMem.ToArray());
return entity;
}
public async ValueTask DisposeAsync()
{
if (!_keepStreamOpen)
{
await _sourceStream.DisposeAsync();
}
_memoryPool.Dispose();
}
public void Dispose()
{
if (!_keepStreamOpen)
{
_sourceStream?.Dispose();
}
_memoryPool?.Dispose();
}
}
}

View file

@ -0,0 +1,47 @@
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace INetMock.Client.Audit
{
public sealed class TypedReader<T> : IEventReader<T> where T : EventDetails, new()
{
private readonly IProtoEventReader _reader;
public TypedReader(IProtoEventReader reader)
{
_reader = reader;
}
public async IAsyncEnumerable<Event<T>> ReadAllAsync([EnumeratorCancellation] CancellationToken token = default)
{
while (true)
{
var ev = await ReadAsync(token);
if (ev == null)
{
yield break;
}
yield return ev;
}
}
public async Task<Event<T>?> ReadAsync(CancellationToken token = default)
{
do
{
var entity = await _reader.ReadAsync(token);
if (entity == null) return null;
var parsed = new Event<T>(entity);
if (parsed.Details == null) continue;
return parsed;
} while (true);
}
public void Dispose() => _reader.Dispose();
public ValueTask DisposeAsync() => _reader.DisposeAsync();
}
}

View file

@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<Nullable>enable</Nullable>
<LangVersion>latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.14.0" />
<PackageReference Include="Grpc.Net.Client" Version="2.35.0" />
<PackageReference Include="Grpc.Tools" Version="2.35.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<Protobuf Include="proto\rpc\audit.proto" GrpcServices="Client" />
<Protobuf Include="proto\rpc\health.proto" GrpcServices="Client" />
<Protobuf Include="proto\audit\event_entity.proto" GrpcServices="Client" />
<Protobuf Include="proto\audit\details\dns_details.proto" GrpcServices="Client" />
<Protobuf Include="proto\audit\details\http_details.proto" GrpcServices="Client" />
</ItemGroup>
</Project>

1
src/INetMock.Client/proto Symbolic link
View file

@ -0,0 +1 @@
../../api/proto/

View file

@ -0,0 +1,57 @@
using System.IO;
using INetMock.Client.Audit;
using INetMock.Client.Test.Hex;
using Xunit;
namespace INetMock.Client.Test.Audit
{
public class GenericReaderTest
{
private const string HttpEventPayload = "000000dd120b088092b8c398feffffff01180120022a047f00000132047f00000138d8fc0140504a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73745282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e";
private const string DnsEventPayload = "0000003b120b088092b8c398feffffff01180120012a100000000000000000000000000000000132100000000000000000000000000000000138d8fc014050";
private readonly byte[] _httpEventPayloadBytes;
private readonly byte[] _dnsEventPayloadBytes;
public GenericReaderTest()
{
_httpEventPayloadBytes = HttpEventPayload.HexToByteArray();
_dnsEventPayloadBytes = DnsEventPayload.HexToByteArray();
}
[Fact]
public async void TestRead_HttpEvent_Success()
{
await using var protoReader = new ProtoReader(new MemoryStream(_httpEventPayloadBytes));
await using var reader = new GenericReader(protoReader);
await foreach (var ev in reader.ReadAllAsync())
{
Assert.NotNull(ev);
Assert.True(ev.CanConvert<HttpDetails>());
Assert.False(ev.CanConvert<DnsDetails>());
var httpEvent = (Event<HttpDetails>?) ev;
var dnsEvent = (Event<DnsDetails>?) ev;
Assert.NotNull(httpEvent);
Assert.Null(dnsEvent);
}
}
[Fact]
public async void TestRead_DnsEvent_Success()
{
await using var memStream = new MemoryStream(_dnsEventPayloadBytes);
await using var protoReader = new ProtoReader(new MemoryStream(_dnsEventPayloadBytes));
await using var reader = new GenericReader(protoReader);
await foreach (var ev in reader.ReadAllAsync())
{
Assert.NotNull(ev);
Assert.True(ev.CanConvert<DnsDetails>());
var dnsEvent = (Event<DnsDetails>?) ev;
Assert.NotNull(dnsEvent);
}
}
}
}

View file

@ -0,0 +1,45 @@
using System.IO;
using INetMock.Client.Audit;
using INetMock.Client.Test.Hex;
using Xunit;
namespace INetMock.Client.Test.Audit
{
public class TypedReaderTest
{
private const string HttpEventPayload = "000000dd120b088092b8c398feffffff01180120022a047f00000132047f00000138d8fc0140504a3308041224544c535f45434448455f45434453415f574954485f4145535f3235365f4342435f5348411a096c6f63616c686f73745282010a34747970652e676f6f676c65617069732e636f6d2f696e65746d6f636b2e61756469742e4854545044657461696c73456e74697479124a12096c6f63616c686f73741a15687474703a2f2f6c6f63616c686f73742f6173646622084854545020312e312a1c0a0641636365707412120a106170706c69636174696f6e2f6a736f6e";
private const string DnsEventPayload = "0000003b120b088092b8c398feffffff01180120012a100000000000000000000000000000000132100000000000000000000000000000000138d8fc014050";
private readonly byte[] _httpEventPayloadBytes;
private readonly byte[] _dnsEventPayloadBytes;
public TypedReaderTest()
{
_httpEventPayloadBytes = HttpEventPayload.HexToByteArray();
_dnsEventPayloadBytes = DnsEventPayload.HexToByteArray();
}
[Fact]
public async void TestRead_HttpEvent_Success()
{
await using var protoReader = new ProtoReader(new MemoryStream(_httpEventPayloadBytes));
await using var reader = new TypedReader<HttpDetails>(protoReader);
await foreach (var ev in reader.ReadAllAsync())
{
Assert.NotNull(ev);
}
}
[Fact]
public async void TestRead_DnsEvent_Success()
{
await using var memStream = new MemoryStream(_dnsEventPayloadBytes);
await using var protoReader = new ProtoReader(new MemoryStream(_dnsEventPayloadBytes));
await using var reader = new TypedReader<DnsDetails>(protoReader);
await foreach (var ev in reader.ReadAllAsync())
{
Assert.NotNull(ev);
}
}
}
}

View file

@ -0,0 +1,15 @@
using System;
using System.Linq;
namespace INetMock.Client.Test.Hex
{
public static class Converter
{
public static byte[] HexToByteArray(this string hex) {
return Enumerable.Range(0, hex.Length)
.Where(x => x % 2 == 0)
.Select(x => Convert.ToByte(hex.Substring(x, 2), 16))
.ToArray();
}
}
}

View file

@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<IsPackable>false</IsPackable>
<LangVersion>latest</LangVersion>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="xunit" Version="2.4.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
<PackageReference Include="coverlet.collector" Version="1.2.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\INetMock.Client\INetMock.Client.csproj" />
</ItemGroup>
</Project>